Source code

Download source archive ↓

src/EcoFlowOperator.scala

1package org.apache.spark.sql.dacp
2
3import _root_.xstream.execution.{InferenceDiagnostic,InferenceResult,OutputSample,PreviewLimits,StreamDescriptor}
4
5
6import cn.piflow.{JobContext,JobInputStream,JobOutputStream}
7import cn.piflow.conf.ConfigurableStop
8import com.fasterxml.jackson.databind.JsonNode
9import xstream.operator.{TransformOperator,OperatorContext}
10import org.apache.arrow.memory.RootAllocator
11import org.apache.arrow.vector.ipc.ArrowStreamReader
12import org.apache.spark.sql.{DataFrame,SparkSession}
13import org.apache.spark.sql.catalyst.InternalRow
14import org.apache.spark.sql.catalyst.expressions.UnsafeProjection
15import org.apache.spark.sql.execution.arrow.ArrowConverters
16import org.apache.spark.sql.util.ArrowUtils
17import org.apache.spark.sql.vectorized.{ArrowColumnVector,ColumnarBatch}
18import java.io.ByteArrayInputStream
19import scala.collection.mutable.ArrayBuffer
20import scala.jdk.CollectionConverters._
21
22/** Shared Spark environment, thin wrappers, explicit bounded input materialization. */
23abstract class EcoFlowOperator(className:String,ports:Map[String,String],required:Set[String]) extends TransformOperator {
24  private val number = "[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?"
25  private val field = "[A-Za-z_][A-Za-z0-9_]{0,99}"
26  private val numeric = Set("longitude","latitude","maxValue","minValue","flagValue","qualityControlMarkerValue",
27    "multiple","para_H","equipmentHeightDifference","time_interval","partitionNum","mdv_num")
28  private val comparison = s"$field\\s*(?:<=|>=|<>|!=|=|<|>)\\s*$number"
29
30  private def validated(params:JsonNode):Map[String,Any]={
31    require(params.isObject && params.fieldNames.asScala.toSet==required,"missing or unexpected parameters")
32    params.fields.asScala.map { entry =>
33      val key=entry.getKey
34      val value=entry.getValue
35      require(value.isTextual,"parameter must be a string: "+key)
36      val text=value.asText
37      require(text.length<=2048,"parameter is too long: "+key)
38      if (key=="additionalCondition")
39        require(text.isEmpty || text.matches(s"(?i)$comparison(?:\\s+(?:AND|OR)\\s+$comparison)*"),"expected numeric column comparisons")
40      else if (key=="statisticalType") require(Set("stddev","variance")(text),"expected stddev or variance")
41      else if (numeric(key)) {
42        val parts=text.split(",",-1).map(_.trim)
43        require((parts.length==1 || key=="equipmentHeightDifference") &&
44          parts.forall(p=>p.matches(number) && p.toDouble.isFinite),"expected finite numeric value: "+key)
45        if (key=="partitionNum") require(text.toInt>=1 && text.toInt<=8,"partitionNum must be between 1 and 8")
46        if (key=="mdv_num") require(text.toInt>=1 && text.toInt<=365,"mdv_num must be between 1 and 365")
47      } else require(text.split(",",-1).forall(_.trim.matches(field)),"expected column identifiers: "+key)
48      key->text
49    }.toMap
50  }
51
52  final def inferOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode):InferenceResult={
53    require(inputs.keySet==ports.keySet,"unexpected input ports")
54    validated(params)
55    InferenceResult(Map("result"->StreamDescriptor(Set("records"),Set("arrow"),certainty="unknown")),
56      Vector(InferenceDiagnostic("warning","spark_dynamic_schema","Spark output schema requires execution")))
57  }
58
59  override def previewOutputs(inputs:Map[String,StreamDescriptor],params:JsonNode,limits:PreviewLimits):Map[String,OutputSample]=
60    Map.empty
61
62  def process(ctx:OperatorContext):Unit={
63    require(ctx.inputNames.toSet==ports.keySet,"unexpected input ports")
64    val p=validated(ctx.params)
65    val stop=Class.forName(className).getDeclaredConstructor().newInstance().asInstanceOf[ConfigurableStop]
66    stop.setProperties(p)
67    if(className.contains("StorageItemPM_")){
68      val heights=p("equipmentHeightDifference").toString.split(",").map(_.trim.toDouble)
69      require(heights.length==7 && heights.forall(x=>x.isFinite && x>=0),"seven finite nonnegative layer heights required")
70      require(p("everyTaFields").toString.split(",").length==7,"seven temperature fields required")
71      p.get("everyPvaporFields").foreach(x=>require(x.toString.split(",").length==7,"seven vapor fields required"))
72      require(p("time_interval").toString.toInt>0,"positive interval in seconds required")
73    }
74    if(className.contains("WildSpotCulling")){
75      require(!p("fields").toString.contains(",") && p("fields").toString.nonEmpty,"this outlier adapter accepts one field per step")
76      require(Set("stddev","variance").contains(p("statisticalType").toString),"statisticalType must be stddev or variance")
77      require(p("multiple").toString.toDouble>=0,"multiple must be nonnegative")
78      require(p("flagValue").toString.toDouble.isFinite,"flagValue must be finite")
79    }
80    p.get("mdv_num").foreach { value=>
81      val window=value.toString.toInt
82      require(window>0 && (!className.endsWith("MDV_G") || window%2==1),"positive window required; sliding MDV requires odd window")
83    }
84    val spark=SparkSession.builder().master("local[2]").appName("DACP EcoFlow")
85      .config("spark.ui.enabled","false").config("spark.driver.host","127.0.0.1")
86      .config("spark.driver.bindAddress","127.0.0.1").config("spark.sql.shuffle.partitions","2")
87      .config("spark.sql.adaptive.enabled","false").config("spark.sql.autoBroadcastJoinThreshold","-1")
88      .config("spark.rpc.netty.dispatcher.numThreads","2").config("spark.shuffle.io.serverThreads","2")
89      .config("spark.shuffle.io.clientThreads","2")
90      .config("spark.sql.session.timeZone","UTC").config("spark.sql.ansi.enabled","false")
91      .config("spark.sql.legacy.allowUntypedScalaUDF","true").getOrCreate()
92    spark.sparkContext.setLogLevel("ERROR")
93    val allocator=new RootAllocator()
94    try {
95      val inputs=ctx.inputNames.map { name=>
96        val in=ctx.input(name);require(in.representation=="records","records input required")
97        val schema=ArrowUtils.fromArrowSchema(in.schema)
98        val projection=UnsafeProjection.create(schema)
99        val rows=ArrayBuffer.empty[InternalRow];var bytes=0L
100        in.foreachRecords { root=>
101          bytes+=root.getFieldVectors.asScala.map(_.getBufferSize.toLong).sum
102          require(bytes<=128L*1024*1024 && rows.size+root.getRowCount<=1000000,"Spark input exceeds 128 MiB or one million rows")
103          val batch=new ColumnarBatch(root.getFieldVectors.asScala.map(v=>new ArrowColumnVector(v)).toArray,root.getRowCount)
104          batch.rowIterator().asScala.foreach(row=>rows+=projection(row).copy())
105          // The SDK owns and closes the vectors; do not close the borrowed ColumnarBatch.
106        }
107        val classic=spark.asInstanceOf[org.apache.spark.sql.classic.SparkSession]
108        ports(name)->classic.internalCreateDataFrame(spark.sparkContext.parallelize(rows.toVector,2),schema)
109      }.toMap
110      val input=new JobInputStream {
111        def read():DataFrame={require(inputs.size==1,"named input required");inputs.values.head}
112        def read(port:String):DataFrame=inputs.getOrElse(port,throw new IllegalArgumentException("unknown upstream input: "+port))
113      }
114      var result:Option[DataFrame]=None
115      val output=new JobOutputStream { def write(df:DataFrame):Unit={require(result.isEmpty,"single output expected");result=Some(df)} }
116      val context=new JobContext { def get[T]()(implicit m:Manifest[T]):T={require(m.runtimeClass==classOf[SparkSession]);spark.asInstanceOf[T]} }
117      stop.perform(input,output,context)
118      val df=result.getOrElse(throw new IllegalStateException("upstream operator produced no result"))
119      val schema=ArrowUtils.toArrowSchema(df.schema,"UTC",true,false)
120      val out=ctx.recordsOutput("result",schema)
121      // Spark execution iterators reuse UnsafeRow storage; copy before partition collection.
122      val batches=ArrowConverters.toBatchWithSchemaIterator(df.queryExecution.toRdd.map(_.copy()).toLocalIterator,
123        df.schema,1024,1024*1024,"UTC",true,false)
124      try while(batches.hasNext){
125        val reader=new ArrowStreamReader(new ByteArrayInputStream(batches.next()),allocator)
126        try while(reader.loadNextBatch())out.write(reader.getVectorSchemaRoot)finally reader.close()
127      }finally batches.close()
128    }finally {allocator.close();spark.stop()}
129  }
130}