src/SparkML.scala
1package org.apache.spark.sql.piflow
2
3import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
4import java.nio.file.{Files, Path}
5import java.nio.file.attribute.{PosixFileAttributes, PosixFilePermissions}
6import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream}
7
8import com.fasterxml.jackson.databind.JsonNode
9import org.apache.arrow.memory.RootAllocator
10import org.apache.arrow.vector.ipc.ArrowStreamReader
11import org.apache.hadoop.fs.{FileStatus, LocalFileSystem, RawLocalFileSystem, Path => HadoopPath}
12import org.apache.hadoop.fs.permission.FsPermission
13import org.apache.spark.ml.{Estimator, Model}
14import org.apache.spark.ml.classification._
15import org.apache.spark.ml.clustering._
16import org.apache.spark.ml.feature.Word2Vec
17import org.apache.spark.ml.functions.{array_to_vector, vector_to_array}
18import org.apache.spark.ml.util.MLWritable
19import org.apache.spark.sql.{DataFrame, SparkSession}
20import org.apache.spark.sql.catalyst.InternalRow
21import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
22import org.apache.spark.sql.execution.arrow.ArrowConverters
23import org.apache.spark.sql.functions.{col, split, trim}
24import org.apache.spark.sql.util.ArrowUtils
25import org.apache.spark.sql.vectorized.{ArrowColumnVector, ColumnarBatch}
26import xstream.execution.{InferenceResult, OutputSample, PreviewLimits, StreamDescriptor}
27import xstream.operator.{OperatorContext, TransformOperator}
28import scala.collection.mutable.ArrayBuffer
29import scala.jdk.CollectionConverters._
30
31/** Use JVM filesystem permissions in the distroless Spark runtime. */
32final class ModelFileSystem extends LocalFileSystem(new RawLocalFileSystem {
33 override def setPermission(path: HadoopPath, permission: FsPermission): Unit = {
34 Files.setPosixFilePermissions(pathToFile(path).toPath, PosixFilePermissions.fromString(permission.toString))
35 }
36
37 override def getFileStatus(path: HadoopPath): FileStatus = {
38 val original = super.getFileStatus(path)
39 val attributes = Files.readAttributes(pathToFile(path).toPath, classOf[PosixFileAttributes])
40 val permissions = FsPermission.valueOf("-" + PosixFilePermissions.toString(attributes.permissions()))
41 new FileStatus(original.getLen, original.isDirectory, original.getReplication, original.getBlockSize,
42 attributes.lastModifiedTime().toMillis, attributes.lastAccessTime().toMillis, permissions,
43 attributes.owner().getName, attributes.group().getName, original.getPath)
44 }
45
46 override def listStatus(path: HadoopPath): Array[FileStatus] =
47 super.listStatus(path).map(status => getFileStatus(status.getPath))
48})
49
50abstract class SparkML(algorithm: String, phase: String) extends TransformOperator {
51 private val limit = 32L * 1024 * 1024
52 private val isTraining = phase == "training"
53 private val isPrediction = phase == "prediction"
54
55 override def inferOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode): InferenceResult = {
56 require(inputs.keySet == (if (isPrediction) Set("source", "model") else Set("source")), "unexpected input ports")
57 val view = if (isTraining) "bytes" else "records"
58 val format = if (isTraining) "binary" else "arrow"
59 InferenceResult(Map("result" -> StreamDescriptor(Set(view), Set(format),
60 certainty = if (isTraining) "known" else "unknown")))
61 }
62
63 override def previewOutputs(inputs: Map[String, StreamDescriptor], params: JsonNode, limits: PreviewLimits):
64 Map[String, OutputSample] = Map.empty
65
66 private def sparkSession(): SparkSession = {
67 // The SDK has already captured the framed output stream.
68 val output = System.out
69 System.setOut(System.err)
70 try createSparkSession() finally System.setOut(output)
71 }
72
73 private def createSparkSession(): SparkSession = {
74 val spark = SparkSession.builder().master("local[2]").appName("PiFlow Spark ML")
75 .config("spark.ui.enabled", "false")
76 .config("spark.driver.host", "127.0.0.1")
77 .config("spark.driver.bindAddress", "127.0.0.1")
78 .config("spark.hadoop.fs.file.impl", classOf[ModelFileSystem].getName)
79 .config("spark.sql.shuffle.partitions", "2")
80 .config("spark.sql.adaptive.enabled", "false")
81 .config("spark.rpc.netty.dispatcher.numThreads", "2")
82 .config("spark.shuffle.io.serverThreads", "2")
83 .config("spark.shuffle.io.clientThreads", "2")
84 .getOrCreate()
85 spark.sparkContext.setLogLevel("ERROR")
86 spark
87 }
88
89 private def readRecords(ctx: OperatorContext, spark: SparkSession): DataFrame = {
90 val input = ctx.input("source")
91 val schema = ArrowUtils.fromArrowSchema(input.schema)
92 val project = UnsafeProjection.create(schema)
93 val rows = ArrayBuffer.empty[InternalRow]
94 var bytes = 0L
95 input.foreachRecords { root =>
96 bytes += root.getFieldVectors.asScala.map(_.getBufferSize.toLong).sum
97 require(bytes <= limit && rows.size.toLong + root.getRowCount <= 200000, "Spark input exceeds 32 MiB or 200,000 rows")
98 val batch = new ColumnarBatch(root.getFieldVectors.asScala.map(v => new ArrowColumnVector(v)).toArray, root.getRowCount)
99 batch.rowIterator().asScala.foreach(row => rows += project(row).copy())
100 }
101 spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession]
102 .internalCreateDataFrame(spark.sparkContext.parallelize(rows.toVector, 2), schema)
103 }
104
105 private def emitRecords(ctx: OperatorContext, frame: DataFrame): Unit = {
106 val schema = ArrowUtils.toArrowSchema(frame.schema, "UTC", true, false)
107 val output = ctx.recordsOutput("result", schema)
108 val allocator = new RootAllocator()
109 val batches = ArrowConverters.toBatchWithSchemaIterator(frame.queryExecution.toRdd.map(_.copy()).toLocalIterator,
110 frame.schema, 1024, 1024 * 1024, "UTC", true, false)
111 try while (batches.hasNext) {
112 val reader = new ArrowStreamReader(new ByteArrayInputStream(batches.next()), allocator)
113 try while (reader.loadNextBatch()) output.write(reader.getVectorSchemaRoot)
114 finally reader.close()
115 } finally {
116 batches.close()
117 allocator.close()
118 }
119 }
120
121 private def saveModel(model: MLWritable, dir: Path, ctx: OperatorContext): Unit = {
122 model.save(dir.toUri.toString)
123 val files = Files.walk(dir)
124 val data = new ByteArrayOutputStream()
125 val zip = new ZipOutputStream(data)
126 var size = 0L
127 try files.iterator().asScala.filter { file =>
128 Files.isRegularFile(file) && !file.getFileName.toString.startsWith(".") &&
129 !file.getFileName.toString.startsWith("_")
130 }.foreach { file =>
131 size += Files.size(file)
132 require(size <= limit, "Spark model exceeds 32 MiB")
133 zip.putNextEntry(new ZipEntry(dir.relativize(file).toString))
134 Files.copy(file, zip)
135 zip.closeEntry()
136 } finally {
137 files.close()
138 zip.close()
139 }
140 val output = ctx.bytesOutput("result")
141 data.toByteArray.grouped(64 * 1024).foreach(output.write(_))
142 }
143
144 private def unpackModel(ctx: OperatorContext, dir: Path): Unit = {
145 val input = new ByteArrayOutputStream()
146 ctx.input("model").foreachBytes { chunk =>
147 require(input.size().toLong + chunk.length <= limit, "compressed Spark model exceeds 32 MiB")
148 input.write(chunk)
149 }
150 val zip = new ZipInputStream(new ByteArrayInputStream(input.toByteArray))
151 val buffer = new Array[Byte](16 * 1024)
152 var size = 0L
153 var count = 0
154 try Iterator.continually(zip.getNextEntry).takeWhile(_ != null).foreach { entry =>
155 val name = entry.getName
156 require(!entry.isDirectory && name.matches("(?:[A-Za-z][A-Za-z0-9_-]{0,50}/){1,5}part-[A-Za-z0-9_.-]+"), "unexpected Spark model entry")
157 count += 1
158 require(count <= 128, "Spark model contains too many entries")
159 val file = dir.resolve(name)
160 Files.createDirectories(file.getParent)
161 val output = Files.newOutputStream(file, java.nio.file.StandardOpenOption.CREATE_NEW, java.nio.file.StandardOpenOption.WRITE)
162 try Iterator.continually(zip.read(buffer)).takeWhile(_ != -1).foreach { n =>
163 size += n
164 require(size <= limit, "expanded Spark model exceeds 32 MiB")
165 output.write(buffer, 0, n)
166 } finally output.close()
167 } finally zip.close()
168 require(Files.isDirectory(dir.resolve("metadata")) && Files.isDirectory(dir.resolve("data")), "incomplete Spark model")
169 }
170
171 private def estimator(params: JsonNode, inputWidth: Int): Estimator[_] = {
172 val seed = 17L
173 val maxIter = params.path("maxIter").asInt(8)
174 val maxDepth = params.path("maxDepth").asInt(5)
175 val k = params.path("k").asInt(2)
176 require(maxIter >= 1 && maxIter <= 100 && maxDepth >= 1 && maxDepth <= 15 && k >= 2 && k <= 32,
177 "expected maxIter in 1..100, maxDepth in 1..15 and k in 2..32")
178 algorithm match {
179 case "decision_tree" => new DecisionTreeClassifier().setMaxDepth(maxDepth).setSeed(seed)
180 case "gbt" => new GBTClassifier().setMaxDepth(maxDepth).setMaxIter(maxIter).setSeed(seed)
181 case "logistic_regression" => new LogisticRegression().setMaxIter(maxIter)
182 case "multilayer_perceptron" =>
183 val hidden = params.path("hiddenLayerSize").asInt(4)
184 require(hidden >= 1 && hidden <= 256, "hiddenLayerSize must be in 1..256")
185 new MultilayerPerceptronClassifier().setLayers(Array(inputWidth, hidden, 2)).setMaxIter(maxIter).setSeed(seed)
186 case "naive_bayes" =>
187 val smoothing = params.path("smoothing").asDouble(1)
188 require(smoothing.isFinite && smoothing >= 0, "smoothing must be finite and nonnegative")
189 new NaiveBayes().setSmoothing(smoothing)
190 case "random_forest" =>
191 val trees = params.path("numTrees").asInt(8)
192 require(trees >= 1 && trees <= 100, "numTrees must be in 1..100")
193 new RandomForestClassifier().setNumTrees(trees).setMaxDepth(maxDepth).setSeed(seed)
194 case "bisecting_kmeans" => new BisectingKMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
195 case "gaussian_mixture" => new GaussianMixture().setK(k).setMaxIter(maxIter).setSeed(seed)
196 case "kmeans" => new KMeans().setK(k).setMaxIter(maxIter).setSeed(seed)
197 case "lda" => new LDA().setOptimizer("online").setK(k).setMaxIter(maxIter).setSeed(seed)
198 }
199 }
200
201 private def load(uri: String): Model[_] = algorithm match {
202 case "decision_tree" => DecisionTreeClassificationModel.load(uri)
203 case "gbt" => GBTClassificationModel.load(uri)
204 case "logistic_regression" => LogisticRegressionModel.load(uri)
205 case "multilayer_perceptron" => MultilayerPerceptronClassificationModel.load(uri)
206 case "naive_bayes" => NaiveBayesModel.load(uri)
207 case "random_forest" => RandomForestClassificationModel.load(uri)
208 case "bisecting_kmeans" => BisectingKMeansModel.load(uri)
209 case "gaussian_mixture" => GaussianMixtureModel.load(uri)
210 case "kmeans" => KMeansModel.load(uri)
211 case "lda" => LocalLDAModel.load(uri)
212 }
213
214 private def features(frame: DataFrame, training: Boolean): DataFrame = {
215 require(frame.columns.contains("features"), "expected a numeric list column named features")
216 require(!frame.columns.contains("prediction") && !frame.columns.contains("topicDistribution"),
217 "prediction and topicDistribution are reserved output fields")
218 val classification = Set("decision_tree", "gbt", "logistic_regression", "multilayer_perceptron", "naive_bayes", "random_forest")
219 val input = frame.withColumn("features", array_to_vector(col("features")))
220 if (training && classification(algorithm)) input.withColumn("label", col("label").cast("double")) else input
221 }
222
223 private def embedding(frame: DataFrame, params: JsonNode): DataFrame = {
224 val column = params.path("column").asText("text")
225 val dimensions = params.path("vectorSize").asInt(4)
226 require(dimensions >= 1 && dimensions <= 256, "vectorSize must be in 1..256")
227 require(column.matches("[a-zA-Z_][a-zA-Z0-9_]*") && frame.columns.contains(column), "unknown text column")
228 require(!frame.columns.contains("tokens") && !frame.columns.contains("embedding"), "tokens and embedding are reserved")
229 val words = frame.withColumn("tokens", split(trim(col(column)), "\\s+"))
230 val model = new Word2Vec().setInputCol("tokens").setOutputCol("embedding")
231 .setVectorSize(dimensions).setMinCount(1).setMaxIter(5).setNumPartitions(1).setSeed(17L).fit(words)
232 model.transform(words).drop("tokens").withColumn("embedding", vector_to_array(col("embedding")))
233 }
234
235 override def process(ctx: OperatorContext): Unit = {
236 val temporary = Files.createTempDirectory("piflow-spark-ml-")
237 val modelDir = temporary.resolve("model")
238 try {
239 if (isPrediction) unpackModel(ctx, modelDir)
240 val spark = sparkSession()
241 try {
242 val frame = readRecords(ctx, spark)
243 if (algorithm == "word_to_vec") emitRecords(ctx, embedding(frame, ctx.params))
244 else if (isTraining) {
245 val data = features(frame, training = true)
246 val width = frame.select("features").first().getSeq[Any](0).size
247 val model = estimator(ctx.params, width).fit(data).asInstanceOf[MLWritable]
248 saveModel(model, modelDir, ctx)
249 } else {
250 val input = features(frame, training = false)
251 val predicted = load(modelDir.toUri.toString).transform(input)
252 val name = if (algorithm == "lda") "topicDistribution" else "prediction"
253 val value = if (algorithm == "lda") vector_to_array(col(name)) else col(name).cast("double")
254 val scalarResult = predicted.withColumn(name, value).withColumn("features", vector_to_array(col("features")))
255 val columns = frame.columns.map(scalarResult.col).toIndexedSeq :+ scalarResult.col(name)
256 emitRecords(ctx, scalarResult.select(columns: _*))
257 }
258 } finally spark.stop()
259 } finally {
260 val paths = Files.walk(temporary)
261 try paths.iterator().asScala.toVector.reverse.foreach(Files.deleteIfExists)
262 finally paths.close()
263 }
264 }
265}