src/SparkML.scala
1package org.apache.spark.sql.piflow
2
3import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
4import java.nio.file.{Files, Path}
5import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
6
7import com.fasterxml.jackson.databind.JsonNode
8import org.apache.arrow.memory.RootAllocator
9import org.apache.arrow.vector.ipc.ArrowStreamReader
10import org.apache.hadoop.fs.{LocalFileSystem, RawLocalFileSystem, Path => HadoopPath}
11import org.apache.hadoop.fs.permission.FsPermission
12import org.apache.spark.ml.{Estimator, Model}
13import org.apache.spark.ml.classification._
14import org.apache.spark.ml.clustering._
15import org.apache.spark.ml.feature.Word2Vec
16import org.apache.spark.ml.functions.{array_to_vector, vector_to_array}
17import org.apache.spark.ml.util.MLWritable
18import org.apache.spark.sql.{DataFrame, SparkSession}
19import org.apache.spark.sql.catalyst.InternalRow
20import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
21import org.apache.spark.sql.execution.arrow.ArrowConverters
22import org.apache.spark.sql.functions.{col, split, trim}
23import org.apache.spark.sql.util.ArrowUtils
24import org.apache.spark.sql.vectorized.{ArrowColumnVector, ColumnarBatch}
25import xstream.execution.{InferenceResult, OutputSample, PreviewLimits, StreamDescriptor}
26import xstream.operator.{OperatorContext, TransformOperator}
27import scala.collection.mutable.ArrayBuffer
28import scala.jdk.CollectionConverters._
29
30/** Use JVM filesystem permissions in the distroless Spark runtime. */
31final class ModelFileSystem extends LocalFileSystem(new RawLocalFileSystem {
32 override def setPermission(path: HadoopPath, permission: FsPermission): Unit = {
33 val mode = permission.toString
34 Files.setPosixFilePermissions(pathToFile(path).toPath, java.nio.file.attribute.PosixFilePermissions.fromString(mode))
35 }
36})
37
38abstract class SparkML(algorithm: String, phase: String) extends TransformOperator {
39 private val limit = 32L * 1024 * 1024
40 private val isTraining = phase == "training"
41 private val isPrediction = phase == "prediction"
42
43 override def inferOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode): InferenceResult = {
44 require(inputs.keySet == (if (isPrediction) Set("source", "model") else Set("source")), "unexpected input ports")
45 val view = if (isTraining) "bytes" else "records"
46 val format = if (isTraining) "binary" else "arrow"
47 InferenceResult(Map("result" -> StreamDescriptor(Set(view), Set(format),
48 certainty = if (isTraining) "known" else "unknown")))
49 }
50
51 override def previewOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode, limits: PreviewLimits):
52 Map[String, OutputSample] = Map.empty
53
54 private def sparkSession(): SparkSession = {
55 // The SDK has already captured the framed output stream.
56 val output = System.out
57 System.setOut(System.err)
58 try createSparkSession() finally System.setOut(output)
59 }
60
61 private def createSparkSession(): SparkSession = {
62 val spark = SparkSession.builder().master("local[2]").appName("PiFlow Spark ML")
63 .config("spark.ui.enabled", "false")
64 .config("spark.driver.host", "127.0.0.1")
65 .config("spark.driver.bindAddress", "127.0.0.1")
66 .config("spark.hadoop.fs.file.impl", classOf[ModelFileSystem].getName)
67 .config("spark.sql.shuffle.partitions", "2")
68 .config("spark.sql.adaptive.enabled", "false")
69 .config("spark.rpc.netty.dispatcher.numThreads", "2")
70 .config("spark.shuffle.io.serverThreads", "2")
71 .config("spark.shuffle.io.clientThreads", "2")
72 .getOrCreate()
73 spark.sparkContext.setLogLevel("ERROR")
74 spark
75 }
76
77 private def readRecords(ctx: OperatorContext, spark: SparkSession): DataFrame = {
78 val input = ctx.input("source")
79 val schema = ArrowUtils.fromArrowSchema(input.schema)
80 val project = UnsafeProjection.create(schema)
81 val rows = ArrayBuffer.empty[InternalRow]
82 var bytes = 0L
83 input.foreachRecords { root =>
84 bytes += root.getFieldVectors.asScala.map(_.getBufferSize.toLong).sum
85 require(bytes <= limit && rows.size.toLong + root.getRowCount <= 200000, "Spark input exceeds 32 MiB or 200,000 rows")
86 val batch = new ColumnarBatch(root.getFieldVectors.asScala.map(v => new ArrowColumnVector(v)).toArray, root.getRowCount)
87 batch.rowIterator().asScala.foreach(row => rows += project(row).copy())
88 }
89 spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession]
90 .internalCreateDataFrame(spark.sparkContext.parallelize(rows.toVector, 2), schema)
91 }
92
93 private def emitRecords(ctx: OperatorContext, frame: DataFrame): Unit = {
94 val schema = ArrowUtils.toArrowSchema(frame.schema, "UTC", true, false)
95 val output = ctx.recordsOutput("result", schema)
96 val allocator = new RootAllocator()
97 val batches = ArrowConverters.toBatchWithSchemaIterator(frame.queryExecution.toRdd.map(_.copy()).toLocalIterator,
98 frame.schema, 1024, 1024 * 1024, "UTC", true, false)
99 try while (batches.hasNext) {
100 val reader = new ArrowStreamReader(new ByteArrayInputStream(batches.next()), allocator)
101 try while (reader.loadNextBatch()) output.write(reader.getVectorSchemaRoot)
102 finally reader.close()
103 } finally {
104 batches.close()
105 allocator.close()
106 }
107 }
108
109 private def saveModel(model: MLWritable, dir: Path, ctx: OperatorContext): Unit = {
110 model.save(dir.toUri.toString)
111 val files = Files.walk(dir)
112 val data = new ByteArrayOutputStream()
113 val zip = new ZipOutputStream(data)
114 var size = 0L
115 try files.iterator().asScala.filter { file =>
116 Files.isRegularFile(file) && !file.getFileName.toString.startsWith(".") &&
117 !file.getFileName.toString.startsWith("_")
118 }.foreach { file =>
119 size += Files.size(file)
120 require(size <= limit, "Spark model exceeds 32 MiB")
121 zip.putNextEntry(new ZipEntry(dir.relativize(file).toString))
122 Files.copy(file, zip)
123 zip.closeEntry()
124 } finally {
125 files.close()
126 zip.close()
127 }
128 val output = ctx.bytesOutput("result")
129 data.toByteArray.grouped(64 * 1024).foreach(output.write(_))
130 }
131
132 private def unpackModel(ctx: OperatorContext, dir: Path): Unit = {
133 val input = new ByteArrayOutputStream()
134 ctx.input("model").foreachBytes { chunk =>
135 require(input.size().toLong + chunk.length <= limit, "compressed Spark model exceeds 32 MiB")
136 input.write(chunk)
137 }
138 val zip = new ZipInputStream(new ByteArrayInputStream(input.toByteArray))
139 val buffer = new Array[Byte](16 * 1024)
140 var size = 0L
141 var count = 0
142 try Iterator.continually(zip.getNextEntry).takeWhile(_ != null).foreach { entry =>
143 val name = entry.getName
144 require(!entry.isDirectory && name.matches("(?:[A-Za-z][A-Za-z0-9_-]{0,50}/){1,5}part-[A-Za-z0-9_.-]+"), "unexpected Spark model entry")
145 count += 1
146 require(count <= 128, "Spark model contains too many entries")
147 val file = dir.resolve(name)
148 Files.createDirectories(file.getParent)
149 val output = Files.newOutputStream(file, java.nio.file.StandardOpenOption.CREATE_NEW, java.nio.file.StandardOpenOption.WRITE)
150 try Iterator.continually(zip.read(buffer)).takeWhile(_ != -1).foreach { n =>
151 size += n
152 require(size <= limit, "expanded Spark model exceeds 32 MiB")
153 output.write(buffer, 0, n)
154 } finally output.close()
155 } finally zip.close()
156 require(Files.isDirectory(dir.resolve("metadata")) && Files.isDirectory(dir.resolve("data")), "incomplete Spark model")
157 }
158
159 private def estimator(params: JsonNode, inputWidth: Int): Estimator[_] = {
160 val seed = 17L
161 val maxIter = params.path("maxIter").asInt(8)
162 val maxDepth = params.path("maxDepth").asInt(5)
163 val k = params.path("k").asInt(2)
164 require(maxIter >= 1 && maxIter <= 100 && maxDepth >= 1 && maxDepth <= 15 && k >= 2 && k <= 32,
165 "expected maxIter in 1..100, maxDepth in 1..15 and k in 2..32")
166 algorithm match {
167 case "decision_tree" => new DecisionTreeClassifier().setMaxDepth(maxDepth).setSeed(seed)
168 case "gbt" => new GBTClassifier().setMaxDepth(maxDepth).setMaxIter(maxIter).setSeed(seed)
169 case "logistic_regression" => new LogisticRegression().setMaxIter(maxIter)
170 case "multilayer_perceptron" =>
171 val hidden = params.path("hiddenLayerSize").asInt(4)
172 require(hidden >= 1 && hidden <= 256, "hiddenLayerSize must be in 1..256")
173 new MultilayerPerceptronClassifier().setLayers(Array(inputWidth, hidden, 2)).setMaxIter(maxIter).setSeed(seed)
174 case "naive_bayes" =>
175 val smoothing = params.path("smoothing").asDouble(1)
176 require(smoothing.isFinite && smoothing >= 0, "smoothing must be finite and nonnegative")
177 new NaiveBayes().setSmoothing(smoothing)
178 case "random_forest" =>
179 val trees = params.path("numTrees").asInt(8)
180 require(trees >= 1 && trees <= 100, "numTrees must be in 1..100")
181 new RandomForestClassifier().setNumTrees(trees).setMaxDepth(maxDepth).setSeed(seed)
182 case "bisecting_kmeans" => new BisectingKMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
183 case "gaussian_mixture" => new GaussianMixture().setK(k).setMaxIter(maxIter).setSeed(seed)
184 case "kmeans" => new KMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
185 case "lda" => new LDA().setOptimizer("online").setK(k).setMaxIter(maxIter).setSeed(seed)
186 }
187 }
188
189 private def load(uri: String): Model[_] = algorithm match {
190 case "decision_tree" => DecisionTreeClassificationModel.load(uri)
191 case "gbt" => GBTClassificationModel.load(uri)
192 case "logistic_regression" => LogisticRegressionModel.load(uri)
193 case "multilayer_perceptron" => MultilayerPerceptronClassificationModel.load(uri)
194 case "naive_bayes" => NaiveBayesModel.load(uri)
195 case "random_forest" => RandomForestClassificationModel.load(uri)
196 case "bisecting_kmeans" => BisectingKMeansModel.load(uri)
197 case "gaussian_mixture" => GaussianMixtureModel.load(uri)
198 case "kmeans" => KMeansModel.load(uri)
199 case "lda" => LocalLDAModel.load(uri)
200 }
201
202 private def features(frame: DataFrame, training: Boolean): DataFrame = {
203 require(frame.columns.contains("features"), "expected a numeric list column named features")
204 require(!frame.columns.contains("prediction") && !frame.columns.contains("topicDistribution"),
205 "prediction and topicDistribution are reserved output fields")
206 val classification = Set("decision_tree", "gbt", "logistic_regression", "multilayer_perceptron", "naive_bayes", "random_forest")
207 val input = frame.withColumn("features", array_to_vector(col("features")))
208 if (training && classification(algorithm)) input.withColumn("label", col("label").cast("double")) else input
209 }
210
211 private def embedding(frame: DataFrame, params: JsonNode): DataFrame = {
212 val column = params.path("column").asText("text")
213 val dimensions = params.path("vectorSize").asInt(4)
214 require(dimensions >= 1 && dimensions <= 256, "vectorSize must be in 1..256")
215 require(column.matches("[a-zA-Z_][a-zA-Z0-9_]*") && frame.columns.contains(column), "unknown text column")
216 require(!frame.columns.contains("tokens") && !frame.columns.contains("embedding"), "tokens and embedding are reserved")
217 val words = frame.withColumn("tokens", split(trim(col(column)), "\\s+"))
218 val model = new Word2Vec().setInputCol("tokens").setOutputCol("embedding")
219 .setVectorSize(dimensions).setMinCount(1).setMaxIter(5).setNumPartitions(1).setSeed(17L).fit(words)
220 model.transform(words).drop("tokens").withColumn("embedding", vector_to_array(col("embedding")))
221 }
222
223 override def process(ctx: OperatorContext): Unit = {
224 val temporary = Files.createTempDirectory("piflow-spark-ml-")
225 val modelDir = temporary.resolve("model")
226 try {
227 if (isPrediction) unpackModel(ctx, modelDir)
228 val spark = sparkSession()
229 try {
230 val frame = readRecords(ctx, spark)
231 if (algorithm == "word_to_vec") emitRecords(ctx, embedding(frame, ctx.params))
232 else if (isTraining) {
233 val data = features(frame, training = true)
234 val width = frame.select("features").first().getSeq[Any](0).size
235 val model = estimator(ctx.params, width).fit(data).asInstanceOf[MLWritable]
236 saveModel(model, modelDir, ctx)
237 } else {
238 val input = features(frame, training = false)
239 val predicted = load(modelDir.toUri.toString).transform(input)
240 val name = if (algorithm == "lda") "topicDistribution" else "prediction"
241 val value = if (algorithm == "lda") vector_to_array(col(name)) else col(name).cast("double")
242 val scalarResult = predicted.withColumn(name, value).withColumn("features", vector_to_array(col("features")))
243 val columns = frame.columns.map(scalarResult.col).toIndexedSeq :+ scalarResult.col(name)
244 emitRecords(ctx, scalarResult.select(columns: _*))
245 }
246 } finally spark.stop()
247 } finally {
248 val paths = Files.walk(temporary)
249 try paths.iterator().asScala.toVector.reverse.foreach(Files.deleteIfExists)
250 finally paths.close()
251 }
252 }
253}