src/EcoFlowOperator.scala
1package org.apache.spark.sql.dacp
2
3import _root_.xstream.execution.{InferenceDiagnostic,InferenceResult,OutputSample,PreviewLimits,StreamDescriptor}
4
5
6import cn.piflow.{JobContext,JobInputStream,JobOutputStream}
7import cn.piflow.conf.ConfigurableStop
8import com.fasterxml.jackson.databind.JsonNode
9import xstream.operator.{TransformOperator,OperatorContext}
10import org.apache.arrow.memory.RootAllocator
11import org.apache.arrow.vector.ipc.ArrowStreamReader
12import org.apache.spark.sql.{DataFrame,SparkSession}
13import org.apache.spark.sql.catalyst.InternalRow
14import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
15import org.apache.spark.sql.execution.arrow.ArrowConverters
16import org.apache.spark.sql.util.ArrowUtils
17import org.apache.spark.sql.vectorized.{ArrowColumnVector,ColumnarBatch}
18import java.io.ByteArrayInputStream
19import scala.collection.mutable.ArrayBuffer
20import scala.jdk.CollectionConverters._
21
22/** Shared Spark environment, thin wrappers, explicit bounded input materialization. */
23abstract class EcoFlowOperator(className:String,ports:Map[String,String],required:Set[String]) extends TransformOperator {
24 private val number = "[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?"
25 private val field = "[A-Za-z_][A-Za-z0-9_]{0,99}"
26 private val numeric = Set("longitude","latitude","maxValue","minValue","flagValue","qualityControlMarkerValue",
27 "multiple","para_H","equipmentHeightDifference","time_interval","partitionNum","mdv_num")
28 private val comparison = s"$field\\s*(?:<=|>=|<>|!=|=|<|>)\\s*$number"
29
30 private def validated(params:JsonNode):Map[String,Any]={
31 require(params.isObject && params.fieldNames.asScala.toSet==required,"missing or unexpected parameters")
32 params.fields.asScala.map { entry =>
33 val key=entry.getKey
34 val value=entry.getValue
35 require(value.isTextual,"parameter must be a string: "+key)
36 val text=value.asText
37 require(text.length<=2048,"parameter is too long: "+key)
38 if (key=="additionalCondition")
39 require(text.isEmpty || text.matches(s"(?i)$comparison(?:\\s+(?:AND|OR)\\s+$comparison)*"),"expected numeric column comparisons")
40 else if (key=="statisticalType") require(Set("stddev","variance")(text),"expected stddev or variance")
41 else if (numeric(key)) {
42 val parts=text.split(",",-1).map(_.trim)
43 require((parts.length==1 || key=="equipmentHeightDifference") &&
44 parts.forall(p=>p.matches(number) && p.toDouble.isFinite),"expected finite numeric value: "+key)
45 if (key=="partitionNum") require(text.toInt>=1 && text.toInt<=8,"partitionNum must be between 1 and 8")
46 if (key=="mdv_num") require(text.toInt>=1 && text.toInt<=365,"mdv_num must be between 1 and 365")
47 } else require(text.split(",",-1).forall(_.trim.matches(field)),"expected column identifiers: "+key)
48 key->text
49 }.toMap
50 }
51
52 final def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult={
53 require(inputs.keySet==ports.keySet,"unexpected input ports")
54 validated(params)
55 InferenceResult(Map("result"->StreamDescriptor(Set("records"),Set("arrow"),certainty="unknown")),
56 Vector(InferenceDiagnostic("warning","spark_dynamic_schema","Spark output schema requires execution")))
57 }
58
59 override def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=
60 Map.empty
61
62 def process(ctx:OperatorContext):Unit={
63 require(ctx.inputNames.toSet==ports.keySet,"unexpected input ports")
64 val p=validated(ctx.params)
65 val stop=Class.forName(className).getDeclaredConstructor().newInstance().asInstanceOf[ConfigurableStop]
66 stop.setProperties(p)
67 if(className.contains("StorageItemPM_")){
68 val heights=p("equipmentHeightDifference").toString.split(",").map(_.trim.toDouble)
69 require(heights.length==7 && heights.forall(x=>x.isFinite && x>=0),"seven finite nonnegative layer heights required")
70 require(p("everyTaFields").toString.split(",").length==7,"seven temperature fields required")
71 p.get("everyPvaporFields").foreach(x=>require(x.toString.split(",").length==7,"seven vapor fields required"))
72 require(p("time_interval").toString.toInt>0,"positive interval in seconds required")
73 }
74 if(className.contains("WildSpotCulling")){
75 require(!p("fields").toString.contains(",") && p("fields").toString.nonEmpty,"this outlier adapter accepts one field per step")
76 require(Set("stddev","variance").contains(p("statisticalType").toString),"statisticalType must be stddev or variance")
77 require(p("multiple").toString.toDouble>=0,"multiple must be nonnegative")
78 require(p("flagValue").toString.toDouble.isFinite,"flagValue must be finite")
79 }
80 p.get("mdv_num").foreach { value=>
81 val window=value.toString.toInt
82 require(window>0 && (!className.endsWith("MDV_G") || window%2==1),"positive window required; sliding MDV requires odd window")
83 }
84 val spark=SparkSession.builder().master("local[2]").appName("DACP EcoFlow")
85 .config("spark.ui.enabled","false").config("spark.driver.host","127.0.0.1")
86 .config("spark.driver.bindAddress","127.0.0.1").config("spark.sql.shuffle.partitions","2")
87 .config("spark.sql.adaptive.enabled","false").config("spark.sql.autoBroadcastJoinThreshold","-1")
88 .config("spark.rpc.netty.dispatcher.numThreads","2").config("spark.shuffle.io.serverThreads","2")
89 .config("spark.shuffle.io.clientThreads","2")
90 .config("spark.sql.session.timeZone","UTC").config("spark.sql.ansi.enabled","false")
91 .config("spark.sql.legacy.allowUntypedScalaUDF","true").getOrCreate()
92 spark.sparkContext.setLogLevel("ERROR")
93 val allocator=new RootAllocator()
94 try {
95 val inputs=ctx.inputNames.map { name=>
96 val in=ctx.input(name);require(in.representation=="records","records input required")
97 val schema=ArrowUtils.fromArrowSchema(in.schema)
98 val projection=UnsafeProjection.create(schema)
99 val rows=ArrayBuffer.empty[InternalRow];var bytes=0L
100 in.foreachRecords { root=>
101 bytes+=root.getFieldVectors.asScala.map(_.getBufferSize.toLong).sum
102 require(bytes<=128L*1024*1024 && rows.size+root.getRowCount<=1000000,"Spark input exceeds 128 MiB or one million rows")
103 val batch=new ColumnarBatch(root.getFieldVectors.asScala.map(v=>new ArrowColumnVector(v)).toArray,root.getRowCount)
104 batch.rowIterator().asScala.foreach(row=>rows+=projection(row).copy())
105 // The SDK owns and closes the vectors; do not close the borrowed ColumnarBatch.
106 }
107 val classic=spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession]
108 ports(name)->classic.internalCreateDataFrame(spark.sparkContext.parallelize(rows.toVector,2),schema)
109 }.toMap
110 val input=new JobInputStream {
111 def read():DataFrame={require(inputs.size==1,"named input required");inputs.values.head}
112 def read(port:String):DataFrame=inputs.getOrElse(port,throw new IllegalArgumentException("unknown upstream input: "+port))
113 }
114 var result:Option[DataFrame]=None
115 val output=new JobOutputStream { def write(df:DataFrame):Unit={require(result.isEmpty,"single output expected");result=Some(df)} }
116 val context=new JobContext { def get[T]()(implicit m:Manifest[T]):T={require(m.runtimeClass==classOf[SparkSession]);spark.asInstanceOf[T]} }
117 stop.perform(input,output,context)
118 val df=result.getOrElse(throw new IllegalStateException("upstream operator produced no result"))
119 val schema=ArrowUtils.toArrowSchema(df.schema,"UTC",true,false)
120 val out=ctx.recordsOutput("result",schema)
121 // Spark execution iterators reuse UnsafeRow storage; copy before partition collection.
122 val batches=ArrowConverters.toBatchWithSchemaIterator(df.queryExecution.toRdd.map(_.copy()).toLocalIterator,
123 df.schema,1024,1024*1024,"UTC",true,false)
124 try while(batches.hasNext){
125 val reader=new ArrowStreamReader(new ByteArrayInputStream(batches.next()),allocator)
126 try while(reader.loadNextBatch())out.write(reader.getVectorSchemaRoot)finally reader.close()
127 }finally batches.close()
128 }finally {allocator.close();spark.stop()}
129 }
130}