Source code

Download source archive ↓

src/SparkML.scala

1package org.apache.spark.sql.piflow
2
3import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
4import java.nio.file.{Files, Path}
5import java.nio.file.attribute.{PosixFileAttributes, PosixFilePermissions}
6import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
7
8import com.fasterxml.jackson.databind.JsonNode
9import org.apache.arrow.memory.RootAllocator
10import org.apache.arrow.vector.ipc.ArrowStreamReader
11import org.apache.hadoop.fs.{FileStatus, LocalFileSystem, RawLocalFileSystem, Path => HadoopPath}
12import org.apache.hadoop.fs.permission.FsPermission
13import org.apache.spark.ml.{Estimator, Model}
14import org.apache.spark.ml.classification._
15import org.apache.spark.ml.clustering._
16import org.apache.spark.ml.feature.Word2Vec
17import org.apache.spark.ml.functions.{array_to_vector, vector_to_array}
18import org.apache.spark.ml.util.MLWritable
19import org.apache.spark.sql.{DataFrame, SparkSession}
20import org.apache.spark.sql.catalyst.InternalRow
21import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
22import org.apache.spark.sql.execution.arrow.ArrowConverters
23import org.apache.spark.sql.functions.{col, split, trim}
24import org.apache.spark.sql.util.ArrowUtils
25import org.apache.spark.sql.vectorized.{ArrowColumnVector, ColumnarBatch}
26import xstream.execution.{InferenceResult, OutputSample, PreviewLimits, StreamDescriptor}
27import xstream.operator.{OperatorContext, TransformOperator}
28import scala.collection.mutable.ArrayBuffer
29import scala.jdk.CollectionConverters._
30
31/** Use JVM filesystem permissions in the distroless Spark runtime. */
32final class ModelFileSystem extends LocalFileSystem(new RawLocalFileSystem {
33  override def setPermission(path: HadoopPath, permission: FsPermission): Unit = {
34    Files.setPosixFilePermissions(pathToFile(path).toPath, PosixFilePermissions.fromString(permission.toString))
35  }
36
37  override def getFileStatus(path: HadoopPath): FileStatus = {
38    val original = super.getFileStatus(path)
39    val attributes = Files.readAttributes(pathToFile(path).toPath, classOf[PosixFileAttributes])
40    val permissions = FsPermission.valueOf("-" + PosixFilePermissions.toString(attributes.permissions()))
41    new FileStatus(original.getLen, original.isDirectory, original.getReplication, original.getBlockSize,
42      attributes.lastModifiedTime().toMillis, attributes.lastAccessTime().toMillis, permissions,
43      attributes.owner().getName, attributes.group().getName, original.getPath)
44  }
45
46  override def listStatus(path: HadoopPath): Array[FileStatus] =
47    super.listStatus(path).map(status => getFileStatus(status.getPath))
48})
49
50abstract class SparkML(algorithm: String, phase: String) extends TransformOperator {
51  private val limit = 32L * 1024 * 1024
52  private val isTraining = phase == "training"
53  private val isPrediction = phase == "prediction"
54
55  override def inferOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode): InferenceResult = {
56    require(inputs.keySet == (if (isPrediction) Set("source", "model") else Set("source")), "unexpected input ports")
57    val view = if (isTraining) "bytes" else "records"
58    val format = if (isTraining) "binary" else "arrow"
59    InferenceResult(Map("result" -> StreamDescriptor(Set(view), Set(format),
60      certainty = if (isTraining) "known" else "unknown")))
61  }
62
63  override def previewOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode, limits: PreviewLimits):
64      Map[String, OutputSample] = Map.empty
65
66  private def sparkSession(): SparkSession = {
67    // The SDK has already captured the framed output stream.
68    val output = System.out
69    System.setOut(System.err)
70    try createSparkSession() finally System.setOut(output)
71  }
72
73  private def createSparkSession(): SparkSession = {
74    val spark = SparkSession.builder().master("local[2]").appName("PiFlow Spark ML")
75      .config("spark.ui.enabled", "false")
76      .config("spark.driver.host", "127.0.0.1")
77      .config("spark.driver.bindAddress", "127.0.0.1")
78      .config("spark.hadoop.fs.file.impl", classOf[ModelFileSystem].getName)
79      .config("spark.sql.shuffle.partitions", "2")
80      .config("spark.sql.adaptive.enabled", "false")
81      .config("spark.rpc.netty.dispatcher.numThreads", "2")
82      .config("spark.shuffle.io.serverThreads", "2")
83      .config("spark.shuffle.io.clientThreads", "2")
84      .getOrCreate()
85    spark.sparkContext.setLogLevel("ERROR")
86    spark
87  }
88
89  private def readRecords(ctx: OperatorContext, spark: SparkSession): DataFrame = {
90    val input = ctx.input("source")
91    val schema = ArrowUtils.fromArrowSchema(input.schema)
92    val project = UnsafeProjection.create(schema)
93    val rows = ArrayBuffer.empty[InternalRow]
94    var bytes = 0L
95    input.foreachRecords { root =>
96      bytes += root.getFieldVectors.asScala.map(_.getBufferSize.toLong).sum
97      require(bytes <= limit && rows.size.toLong + root.getRowCount <= 200000, "Spark input exceeds 32 MiB or 200,000 rows")
98      val batch = new ColumnarBatch(root.getFieldVectors.asScala.map(v => new ArrowColumnVector(v)).toArray, root.getRowCount)
99      batch.rowIterator().asScala.foreach(row => rows += project(row).copy())
100    }
101    spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession]
102      .internalCreateDataFrame(spark.sparkContext.parallelize(rows.toVector, 2), schema)
103  }
104
105  private def emitRecords(ctx: OperatorContext, frame: DataFrame): Unit = {
106    val schema = ArrowUtils.toArrowSchema(frame.schema, "UTC", true, false)
107    val output = ctx.recordsOutput("result", schema)
108    val allocator = new RootAllocator()
109    val batches = ArrowConverters.toBatchWithSchemaIterator(frame.queryExecution.toRdd.map(_.copy()).toLocalIterator,
110      frame.schema, 1024, 1024 * 1024, "UTC", true, false)
111    try while (batches.hasNext) {
112      val reader = new ArrowStreamReader(new ByteArrayInputStream(batches.next()), allocator)
113      try while (reader.loadNextBatch()) output.write(reader.getVectorSchemaRoot)
114      finally reader.close()
115    } finally {
116      batches.close()
117      allocator.close()
118    }
119  }
120
121  private def saveModel(model: MLWritable, dir: Path, ctx: OperatorContext): Unit = {
122    model.save(dir.toUri.toString)
123    val files = Files.walk(dir)
124    val data = new ByteArrayOutputStream()
125    val zip = new ZipOutputStream(data)
126    var size = 0L
127    try files.iterator().asScala.filter { file =>
128      Files.isRegularFile(file) && !file.getFileName.toString.startsWith(".") &&
129        !file.getFileName.toString.startsWith("_")
130    }.foreach { file =>
131      size += Files.size(file)
132      require(size <= limit, "Spark model exceeds 32 MiB")
133      zip.putNextEntry(new ZipEntry(dir.relativize(file).toString))
134      Files.copy(file, zip)
135      zip.closeEntry()
136    } finally {
137      files.close()
138      zip.close()
139    }
140    val output = ctx.bytesOutput("result")
141    data.toByteArray.grouped(64 * 1024).foreach(output.write(_))
142  }
143
144  private def unpackModel(ctx: OperatorContext, dir: Path): Unit = {
145    val input = new ByteArrayOutputStream()
146    ctx.input("model").foreachBytes { chunk =>
147      require(input.size().toLong + chunk.length <= limit, "compressed Spark model exceeds 32 MiB")
148      input.write(chunk)
149    }
150    val zip = new ZipInputStream(new ByteArrayInputStream(input.toByteArray))
151    val buffer = new Array[Byte](16 * 1024)
152    var size = 0L
153    var count = 0
154    try Iterator.continually(zip.getNextEntry).takeWhile(_ != null).foreach { entry =>
155      val name = entry.getName
156      require(!entry.isDirectory && name.matches("(?:[A-Za-z][A-Za-z0-9_-]{0,50}/){1,5}part-[A-Za-z0-9_.-]+"), "unexpected Spark model entry")
157      count += 1
158      require(count <= 128, "Spark model contains too many entries")
159      val file = dir.resolve(name)
160      Files.createDirectories(file.getParent)
161      val output = Files.newOutputStream(file, java.nio.file.StandardOpenOption.CREATE_NEW, java.nio.file.StandardOpenOption.WRITE)
162      try Iterator.continually(zip.read(buffer)).takeWhile(_ != -1).foreach { n =>
163        size += n
164        require(size <= limit, "expanded Spark model exceeds 32 MiB")
165        output.write(buffer, 0, n)
166      } finally output.close()
167    } finally zip.close()
168    require(Files.isDirectory(dir.resolve("metadata")) && Files.isDirectory(dir.resolve("data")), "incomplete Spark model")
169  }
170
171  private def estimator(params: JsonNode, inputWidth: Int): Estimator[_] = {
172    val seed = 17L
173    val maxIter = params.path("maxIter").asInt(8)
174    val maxDepth = params.path("maxDepth").asInt(5)
175    val k = params.path("k").asInt(2)
176    require(maxIter >= 1 && maxIter <= 100 && maxDepth >= 1 && maxDepth <= 15 && k >= 2 && k <= 32,
177      "expected maxIter in 1..100, maxDepth in 1..15 and k in 2..32")
178    algorithm match {
179      case "decision_tree" => new DecisionTreeClassifier().setMaxDepth(maxDepth).setSeed(seed)
180      case "gbt" => new GBTClassifier().setMaxDepth(maxDepth).setMaxIter(maxIter).setSeed(seed)
181      case "logistic_regression" => new LogisticRegression().setMaxIter(maxIter)
182      case "multilayer_perceptron" =>
183        val hidden = params.path("hiddenLayerSize").asInt(4)
184        require(hidden >= 1 && hidden <= 256, "hiddenLayerSize must be in 1..256")
185        new MultilayerPerceptronClassifier().setLayers(Array(inputWidth, hidden, 2)).setMaxIter(maxIter).setSeed(seed)
186      case "naive_bayes" =>
187        val smoothing = params.path("smoothing").asDouble(1)
188        require(smoothing.isFinite && smoothing >= 0, "smoothing must be finite and nonnegative")
189        new NaiveBayes().setSmoothing(smoothing)
190      case "random_forest" =>
191        val trees = params.path("numTrees").asInt(8)
192        require(trees >= 1 && trees <= 100, "numTrees must be in 1..100")
193        new RandomForestClassifier().setNumTrees(trees).setMaxDepth(maxDepth).setSeed(seed)
194      case "bisecting_kmeans" => new BisectingKMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
195      case "gaussian_mixture" => new GaussianMixture().setK(k).setMaxIter(maxIter).setSeed(seed)
196      case "kmeans" => new KMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
197      case "lda" => new LDA().setOptimizer("online").setK(k).setMaxIter(maxIter).setSeed(seed)
198    }
199  }
200
201  private def load(uri: String): Model[_] = algorithm match {
202    case "decision_tree" => DecisionTreeClassificationModel.load(uri)
203    case "gbt" => GBTClassificationModel.load(uri)
204    case "logistic_regression" => LogisticRegressionModel.load(uri)
205    case "multilayer_perceptron" => MultilayerPerceptronClassificationModel.load(uri)
206    case "naive_bayes" => NaiveBayesModel.load(uri)
207    case "random_forest" => RandomForestClassificationModel.load(uri)
208    case "bisecting_kmeans" => BisectingKMeansModel.load(uri)
209    case "gaussian_mixture" => GaussianMixtureModel.load(uri)
210    case "kmeans" => KMeansModel.load(uri)
211    case "lda" => LocalLDAModel.load(uri)
212  }
213
214  private def features(frame: DataFrame, training: Boolean): DataFrame = {
215    require(frame.columns.contains("features"), "expected a numeric list column named features")
216    require(!frame.columns.contains("prediction") && !frame.columns.contains("topicDistribution"),
217      "prediction and topicDistribution are reserved output fields")
218    val classification = Set("decision_tree", "gbt", "logistic_regression", "multilayer_perceptron", "naive_bayes", "random_forest")
219    val input = frame.withColumn("features", array_to_vector(col("features")))
220    if (training && classification(algorithm)) input.withColumn("label", col("label").cast("double")) else input
221  }
222
223  private def embedding(frame: DataFrame, params: JsonNode): DataFrame = {
224    val column = params.path("column").asText("text")
225    val dimensions = params.path("vectorSize").asInt(4)
226    require(dimensions >= 1 && dimensions <= 256, "vectorSize must be in 1..256")
227    require(column.matches("[a-zA-Z_][a-zA-Z0-9_]*") && frame.columns.contains(column), "unknown text column")
228    require(!frame.columns.contains("tokens") && !frame.columns.contains("embedding"), "tokens and embedding are reserved")
229    val words = frame.withColumn("tokens", split(trim(col(column)), "\\s+"))
230    val model = new Word2Vec().setInputCol("tokens").setOutputCol("embedding")
231      .setVectorSize(dimensions).setMinCount(1).setMaxIter(5).setNumPartitions(1).setSeed(17L).fit(words)
232    model.transform(words).drop("tokens").withColumn("embedding", vector_to_array(col("embedding")))
233  }
234
235  override def process(ctx: OperatorContext): Unit = {
236    val temporary = Files.createTempDirectory("piflow-spark-ml-")
237    val modelDir = temporary.resolve("model")
238    try {
239      if (isPrediction) unpackModel(ctx, modelDir)
240      val spark = sparkSession()
241      try {
242        val frame = readRecords(ctx, spark)
243        if (algorithm == "word_to_vec") emitRecords(ctx, embedding(frame, ctx.params))
244        else if (isTraining) {
245          val data = features(frame, training = true)
246          val width = frame.select("features").first().getSeq[Any](0).size
247          val model = estimator(ctx.params, width).fit(data).asInstanceOf[MLWritable]
248          saveModel(model, modelDir, ctx)
249        } else {
250          val input = features(frame, training = false)
251          val predicted = load(modelDir.toUri.toString).transform(input)
252          val name = if (algorithm == "lda") "topicDistribution" else "prediction"
253          val value = if (algorithm == "lda") vector_to_array(col(name)) else col(name).cast("double")
254          val scalarResult = predicted.withColumn(name, value).withColumn("features", vector_to_array(col("features")))
255          val columns = frame.columns.map(scalarResult.col).toIndexedSeq :+ scalarResult.col(name)
256          emitRecords(ctx, scalarResult.select(columns: _*))
257        }
258      } finally spark.stop()
259    } finally {
260      val paths = Files.walk(temporary)
261      try paths.iterator().asScala.toVector.reverse.foreach(Files.deleteIfExists)
262      finally paths.close()
263    }
264  }
265}