Source code

Download source archive ↓

src/SparkML.scala

1package org.apache.spark.sql.piflow
2
3import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
4import java.nio.file.{Files, Path}
5import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
6
7import com.fasterxml.jackson.databind.JsonNode
8import org.apache.arrow.memory.RootAllocator
9import org.apache.arrow.vector.ipc.ArrowStreamReader
10import org.apache.hadoop.fs.{LocalFileSystem, RawLocalFileSystem, Path => HadoopPath}
11import org.apache.hadoop.fs.permission.FsPermission
12import org.apache.spark.ml.{Estimator, Model}
13import org.apache.spark.ml.classification._
14import org.apache.spark.ml.clustering._
15import org.apache.spark.ml.feature.Word2Vec
16import org.apache.spark.ml.functions.{array_to_vector, vector_to_array}
17import org.apache.spark.ml.util.MLWritable
18import org.apache.spark.sql.{DataFrame, SparkSession}
19import org.apache.spark.sql.catalyst.InternalRow
20import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
21import org.apache.spark.sql.execution.arrow.ArrowConverters
22import org.apache.spark.sql.functions.{col, split, trim}
23import org.apache.spark.sql.util.ArrowUtils
24import org.apache.spark.sql.vectorized.{ArrowColumnVector, ColumnarBatch}
25import xstream.execution.{InferenceResult, OutputSample, PreviewLimits, StreamDescriptor}
26import xstream.operator.{OperatorContext, TransformOperator}
27import scala.collection.mutable.ArrayBuffer
28import scala.jdk.CollectionConverters._
29
30/** Use JVM filesystem permissions in the distroless Spark runtime. */
31final class ModelFileSystem extends LocalFileSystem(new RawLocalFileSystem {
32  override def setPermission(path: HadoopPath, permission: FsPermission): Unit = {
33    val mode = permission.toString
34    Files.setPosixFilePermissions(pathToFile(path).toPath, java.nio.file.attribute.PosixFilePermissions.fromString(mode))
35  }
36})
37
38abstract class SparkML(algorithm: String, phase: String) extends TransformOperator {
39  private val limit = 32L * 1024 * 1024
40  private val isTraining = phase == "training"
41  private val isPrediction = phase == "prediction"
42
43  override def inferOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode): InferenceResult = {
44    require(inputs.keySet == (if (isPrediction) Set("source", "model") else Set("source")), "unexpected input ports")
45    val view = if (isTraining) "bytes" else "records"
46    val format = if (isTraining) "binary" else "arrow"
47    InferenceResult(Map("result" -> StreamDescriptor(Set(view), Set(format),
48      certainty = if (isTraining) "known" else "unknown")))
49  }
50
51  override def previewOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode, limits: PreviewLimits):
52      Map[String, OutputSample] = Map.empty
53
54  private def sparkSession(): SparkSession = {
55    // The SDK has already captured the framed output stream.
56    val output = System.out
57    System.setOut(System.err)
58    try createSparkSession() finally System.setOut(output)
59  }
60
61  private def createSparkSession(): SparkSession = {
62    val spark = SparkSession.builder().master("local[2]").appName("PiFlow Spark ML")
63      .config("spark.ui.enabled", "false")
64      .config("spark.driver.host", "127.0.0.1")
65      .config("spark.driver.bindAddress", "127.0.0.1")
66      .config("spark.hadoop.fs.file.impl", classOf[ModelFileSystem].getName)
67      .config("spark.sql.shuffle.partitions", "2")
68      .config("spark.sql.adaptive.enabled", "false")
69      .config("spark.rpc.netty.dispatcher.numThreads", "2")
70      .config("spark.shuffle.io.serverThreads", "2")
71      .config("spark.shuffle.io.clientThreads", "2")
72      .getOrCreate()
73    spark.sparkContext.setLogLevel("ERROR")
74    spark
75  }
76
77  private def readRecords(ctx: OperatorContext, spark: SparkSession): DataFrame = {
78    val input = ctx.input("source")
79    val schema = ArrowUtils.fromArrowSchema(input.schema)
80    val project = UnsafeProjection.create(schema)
81    val rows = ArrayBuffer.empty[InternalRow]
82    var bytes = 0L
83    input.foreachRecords { root =>
84      bytes += root.getFieldVectors.asScala.map(_.getBufferSize.toLong).sum
85      require(bytes <= limit && rows.size.toLong + root.getRowCount <= 200000, "Spark input exceeds 32 MiB or 200,000 rows")
86      val batch = new ColumnarBatch(root.getFieldVectors.asScala.map(v => new ArrowColumnVector(v)).toArray, root.getRowCount)
87      batch.rowIterator().asScala.foreach(row => rows += project(row).copy())
88    }
89    spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession]
90      .internalCreateDataFrame(spark.sparkContext.parallelize(rows.toVector, 2), schema)
91  }
92
93  private def emitRecords(ctx: OperatorContext, frame: DataFrame): Unit = {
94    val schema = ArrowUtils.toArrowSchema(frame.schema, "UTC", true, false)
95    val output = ctx.recordsOutput("result", schema)
96    val allocator = new RootAllocator()
97    val batches = ArrowConverters.toBatchWithSchemaIterator(frame.queryExecution.toRdd.map(_.copy()).toLocalIterator,
98      frame.schema, 1024, 1024 * 1024, "UTC", true, false)
99    try while (batches.hasNext) {
100      val reader = new ArrowStreamReader(new ByteArrayInputStream(batches.next()), allocator)
101      try while (reader.loadNextBatch()) output.write(reader.getVectorSchemaRoot)
102      finally reader.close()
103    } finally {
104      batches.close()
105      allocator.close()
106    }
107  }
108
109  private def saveModel(model: MLWritable, dir: Path, ctx: OperatorContext): Unit = {
110    model.save(dir.toUri.toString)
111    val files = Files.walk(dir)
112    val data = new ByteArrayOutputStream()
113    val zip = new ZipOutputStream(data)
114    var size = 0L
115    try files.iterator().asScala.filter { file =>
116      Files.isRegularFile(file) && !file.getFileName.toString.startsWith(".") &&
117        !file.getFileName.toString.startsWith("_")
118    }.foreach { file =>
119      size += Files.size(file)
120      require(size <= limit, "Spark model exceeds 32 MiB")
121      zip.putNextEntry(new ZipEntry(dir.relativize(file).toString))
122      Files.copy(file, zip)
123      zip.closeEntry()
124    } finally {
125      files.close()
126      zip.close()
127    }
128    val output = ctx.bytesOutput("result")
129    data.toByteArray.grouped(64 * 1024).foreach(output.write(_))
130  }
131
132  private def unpackModel(ctx: OperatorContext, dir: Path): Unit = {
133    val input = new ByteArrayOutputStream()
134    ctx.input("model").foreachBytes { chunk =>
135      require(input.size().toLong + chunk.length <= limit, "compressed Spark model exceeds 32 MiB")
136      input.write(chunk)
137    }
138    val zip = new ZipInputStream(new ByteArrayInputStream(input.toByteArray))
139    val buffer = new Array[Byte](16 * 1024)
140    var size = 0L
141    var count = 0
142    try Iterator.continually(zip.getNextEntry).takeWhile(_ != null).foreach { entry =>
143      val name = entry.getName
144      require(!entry.isDirectory && name.matches("(?:[A-Za-z][A-Za-z0-9_-]{0,50}/){1,5}part-[A-Za-z0-9_.-]+"), "unexpected Spark model entry")
145      count += 1
146      require(count <= 128, "Spark model contains too many entries")
147      val file = dir.resolve(name)
148      Files.createDirectories(file.getParent)
149      val output = Files.newOutputStream(file, java.nio.file.StandardOpenOption.CREATE_NEW, java.nio.file.StandardOpenOption.WRITE)
150      try Iterator.continually(zip.read(buffer)).takeWhile(_ != -1).foreach { n =>
151        size += n
152        require(size <= limit, "expanded Spark model exceeds 32 MiB")
153        output.write(buffer, 0, n)
154      } finally output.close()
155    } finally zip.close()
156    require(Files.isDirectory(dir.resolve("metadata")) && Files.isDirectory(dir.resolve("data")), "incomplete Spark model")
157  }
158
159  private def estimator(params: JsonNode, inputWidth: Int): Estimator[_] = {
160    val seed = 17L
161    val maxIter = params.path("maxIter").asInt(8)
162    val maxDepth = params.path("maxDepth").asInt(5)
163    val k = params.path("k").asInt(2)
164    require(maxIter >= 1 && maxIter <= 100 && maxDepth >= 1 && maxDepth <= 15 && k >= 2 && k <= 32,
165      "expected maxIter in 1..100, maxDepth in 1..15 and k in 2..32")
166    algorithm match {
167      case "decision_tree" => new DecisionTreeClassifier().setMaxDepth(maxDepth).setSeed(seed)
168      case "gbt" => new GBTClassifier().setMaxDepth(maxDepth).setMaxIter(maxIter).setSeed(seed)
169      case "logistic_regression" => new LogisticRegression().setMaxIter(maxIter)
170      case "multilayer_perceptron" =>
171        val hidden = params.path("hiddenLayerSize").asInt(4)
172        require(hidden >= 1 && hidden <= 256, "hiddenLayerSize must be in 1..256")
173        new MultilayerPerceptronClassifier().setLayers(Array(inputWidth, hidden, 2)).setMaxIter(maxIter).setSeed(seed)
174      case "naive_bayes" =>
175        val smoothing = params.path("smoothing").asDouble(1)
176        require(smoothing.isFinite && smoothing >= 0, "smoothing must be finite and nonnegative")
177        new NaiveBayes().setSmoothing(smoothing)
178      case "random_forest" =>
179        val trees = params.path("numTrees").asInt(8)
180        require(trees >= 1 && trees <= 100, "numTrees must be in 1..100")
181        new RandomForestClassifier().setNumTrees(trees).setMaxDepth(maxDepth).setSeed(seed)
182      case "bisecting_kmeans" => new BisectingKMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
183      case "gaussian_mixture" => new GaussianMixture().setK(k).setMaxIter(maxIter).setSeed(seed)
184      case "kmeans" => new KMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
185      case "lda" => new LDA().setOptimizer("online").setK(k).setMaxIter(maxIter).setSeed(seed)
186    }
187  }
188
189  private def load(uri: String): Model[_] = algorithm match {
190    case "decision_tree" => DecisionTreeClassificationModel.load(uri)
191    case "gbt" => GBTClassificationModel.load(uri)
192    case "logistic_regression" => LogisticRegressionModel.load(uri)
193    case "multilayer_perceptron" => MultilayerPerceptronClassificationModel.load(uri)
194    case "naive_bayes" => NaiveBayesModel.load(uri)
195    case "random_forest" => RandomForestClassificationModel.load(uri)
196    case "bisecting_kmeans" => BisectingKMeansModel.load(uri)
197    case "gaussian_mixture" => GaussianMixtureModel.load(uri)
198    case "kmeans" => KMeansModel.load(uri)
199    case "lda" => LocalLDAModel.load(uri)
200  }
201
202  private def features(frame: DataFrame, training: Boolean): DataFrame = {
203    require(frame.columns.contains("features"), "expected a numeric list column named features")
204    require(!frame.columns.contains("prediction") && !frame.columns.contains("topicDistribution"),
205      "prediction and topicDistribution are reserved output fields")
206    val classification = Set("decision_tree", "gbt", "logistic_regression", "multilayer_perceptron", "naive_bayes", "random_forest")
207    val input = frame.withColumn("features", array_to_vector(col("features")))
208    if (training && classification(algorithm)) input.withColumn("label", col("label").cast("double")) else input
209  }
210
211  private def embedding(frame: DataFrame, params: JsonNode): DataFrame = {
212    val column = params.path("column").asText("text")
213    val dimensions = params.path("vectorSize").asInt(4)
214    require(dimensions >= 1 && dimensions <= 256, "vectorSize must be in 1..256")
215    require(column.matches("[a-zA-Z_][a-zA-Z0-9_]*") && frame.columns.contains(column), "unknown text column")
216    require(!frame.columns.contains("tokens") && !frame.columns.contains("embedding"), "tokens and embedding are reserved")
217    val words = frame.withColumn("tokens", split(trim(col(column)), "\\s+"))
218    val model = new Word2Vec().setInputCol("tokens").setOutputCol("embedding")
219      .setVectorSize(dimensions).setMinCount(1).setMaxIter(5).setNumPartitions(1).setSeed(17L).fit(words)
220    model.transform(words).drop("tokens").withColumn("embedding", vector_to_array(col("embedding")))
221  }
222
223  override def process(ctx: OperatorContext): Unit = {
224    val temporary = Files.createTempDirectory("piflow-spark-ml-")
225    val modelDir = temporary.resolve("model")
226    try {
227      if (isPrediction) unpackModel(ctx, modelDir)
228      val spark = sparkSession()
229      try {
230        val frame = readRecords(ctx, spark)
231        if (algorithm == "word_to_vec") emitRecords(ctx, embedding(frame, ctx.params))
232        else if (isTraining) {
233          val data = features(frame, training = true)
234          val width = frame.select("features").first().getSeq[Any](0).size
235          val model = estimator(ctx.params, width).fit(data).asInstanceOf[MLWritable]
236          saveModel(model, modelDir, ctx)
237        } else {
238          val input = features(frame, training = false)
239          val predicted = load(modelDir.toUri.toString).transform(input)
240          val name = if (algorithm == "lda") "topicDistribution" else "prediction"
241          val value = if (algorithm == "lda") vector_to_array(col(name)) else col(name).cast("double")
242          val scalarResult = predicted.withColumn(name, value).withColumn("features", vector_to_array(col("features")))
243          val columns = frame.columns.map(scalarResult.col).toIndexedSeq :+ scalarResult.col(name)
244          emitRecords(ctx, scalarResult.select(columns: _*))
245        }
246      } finally spark.stop()
247    } finally {
248      val paths = Files.walk(temporary)
249      try paths.iterator().asScala.toVector.reverse.foreach(Files.deleteIfExists)
250      finally paths.close()
251    }
252  }
253}