src/Support.scala
1package xstream.examples.piflow
2
3import java.io.{InputStream,ByteArrayInputStream,ByteArrayOutputStream}
4import java.nio.charset.StandardCharsets.UTF_8
5import com.fasterxml.jackson.databind.{JsonNode,ObjectMapper}
6import org.apache.arrow.memory.RootAllocator
7import org.apache.arrow.vector._
8import org.apache.arrow.vector.types.pojo.{ArrowType,Field,FieldType,Schema}
9import org.apache.arrow.vector.types.FloatingPointPrecision
10import xstream.operator.{Operator,OperatorContext}
11import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
12import scala.jdk.CollectionConverters._
13import scala.collection.mutable
14
15private[piflow] trait Source {
16 def schema:Schema
17 def records(f:VectorSchemaRoot=>Unit):Unit
18 def bytes:InputStream
19}
20private[piflow] trait Sink {
21 def records(schema:Schema):VectorSchemaRoot=>Unit
22 def bytes():java.io.OutputStream
23}
24private[piflow] object Support {
25 val mapper=new ObjectMapper()
26 val MaxBytes=128L*1024*1024
27 val BatchSize=128
28 def fields(s:Schema):Vector[Field]=s.getFields.asScala.toVector
29 def names(s:Schema):Vector[String]=fields(s).map(_.getName)
30 def unique(s:Schema):Unit=require(names(s).nonEmpty && names(s).distinct.size==names(s).size,"unique, nonempty columns required")
31 def field(name:String,t:ArrowType,nullable:Boolean=true):Field=new Field(name,new FieldType(nullable,t,null),null)
32 def nullable(f:Field,b:Boolean):Field=new Field(f.getName,new FieldType(b,f.getType,f.getDictionary,f.getMetadata),f.getChildren)
33 def rename(f:Field,n:String):Field=new Field(n,f.getFieldType,f.getChildren)
34 def schema(fs:Vector[Field],base:Schema=null):Schema=new Schema(fs.asJava,if(base==null)null else base.getCustomMetadata)
35 def columns(s:String):Vector[String]={val ns=s.split(",",-1).toVector.map(_.trim);require(ns.forall(_.nonEmpty) && ns.distinct.size==ns.size,"unique, nonempty column names required");ns}
36 def text(p:JsonNode,k:String,default:String=null):String={val n=p.get(k);if(n==null){require(default!=null,s"missing parameter: $k");default}else{require(n.isTextual,s"$k must be text");n.asText}}
37 def bool(p:JsonNode,k:String,default:Boolean):Boolean={val n=p.get(k);if(n==null)default else {require(n.isBoolean,s"$k must be boolean");n.asBoolean}}
38 def params(p:JsonNode,required:Set[String],optional:Set[String]=Set.empty):Unit={require(p.isObject,"parameters must be an object");val ks=p.fieldNames.asScala.toSet;require(required.subsetOf(ks) && ks.subsetOf(required++optional),"missing or unexpected parameters")}
39 def check():Unit=if(Thread.currentThread.isInterrupted)throw new java.util.concurrent.CancellationException()
40 def scalar(f:Field):Unit=require(f.getChildren.isEmpty && f.getDictionary==null && !f.getType.isInstanceOf[ArrowType.Union],"scalar, non-dictionary columns required")
41 def value(v:FieldVector,i:Int):AnyRef=v.getObject(i) match {
42 case null=>null
43 case t:org.apache.arrow.vector.util.Text=>t.toString
44 case x=>x
45 }
46 def set(v:FieldVector,i:Int,x:AnyRef):Unit={
47 if(x==null){v.setNull(i);return}
48 v match {
49 case z:VarCharVector=>z.setSafe(i,x.toString.getBytes(UTF_8))
50 case z:VarBinaryVector=>z.setSafe(i,x.asInstanceOf[Array[Byte]])
51 case z:BigIntVector=>z.setSafe(i,x.asInstanceOf[Number].longValue)
52 case z:IntVector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
53 case z:SmallIntVector=>z.setSafe(i,x.asInstanceOf[Number].shortValue)
54 case z:TinyIntVector=>z.setSafe(i,x.asInstanceOf[Number].byteValue)
55 case z:Float8Vector=>z.setSafe(i,x.asInstanceOf[Number].doubleValue)
56 case z:Float4Vector=>z.setSafe(i,x.asInstanceOf[Number].floatValue)
57 case z:BitVector=>z.setSafe(i,if(x==java.lang.Boolean.TRUE || x==Integer.valueOf(1))1 else 0)
58 case z:UInt1Vector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
59 case z:UInt2Vector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
60 case z:UInt4Vector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
61 case z:UInt8Vector=>z.setSafe(i,x.asInstanceOf[Number].longValue)
62 case z:DateDayVector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
63 case z:DateMilliVector=>z.setSafe(i,x match {case n:Number=>n.longValue;case t:java.time.LocalDateTime=>t.toInstant(java.time.ZoneOffset.UTC).toEpochMilli})
64 case z:TimeSecVector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
65 case z:TimeMilliVector=>z.setSafe(i,x match {case n:Number=>n.intValue;case t:java.time.LocalDateTime=>(t.toLocalTime.toNanoOfDay/1000000).toInt})
66 case z:TimeMicroVector=>z.setSafe(i,x.asInstanceOf[Number].longValue)
67 case z:TimeNanoVector=>z.setSafe(i,x.asInstanceOf[Number].longValue)
68 case z:TimeStampVector=>
69 val ticks=x match {
70 case n:Number=>n.longValue
71 case t:java.time.LocalDateTime=>
72 val instant=t.toInstant(java.time.ZoneOffset.UTC)
73 z.getField.getType.asInstanceOf[ArrowType.Timestamp].getUnit match {
74 case org.apache.arrow.vector.types.TimeUnit.SECOND=>instant.getEpochSecond
75 case org.apache.arrow.vector.types.TimeUnit.MILLISECOND=>instant.toEpochMilli
76 case org.apache.arrow.vector.types.TimeUnit.MICROSECOND=>instant.getEpochSecond*1000000+instant.getNano/1000
77 case org.apache.arrow.vector.types.TimeUnit.NANOSECOND=>instant.getEpochSecond*1000000000+instant.getNano
78 }
79 };z.setSafe(i,ticks)
80 case z:FixedSizeBinaryVector=>z.setSafe(i,x.asInstanceOf[Array[Byte]])
81 case z:DecimalVector=>z.setSafe(i,new java.math.BigDecimal(x.toString))
82 case z:NullVector=>require(x==null,"null field cannot contain a value")
83 case _=>throw new IllegalArgumentException("sample/SQL value unsupported for "+v.getField.getType)
84 }
85 }
86 def estimated(row:Vector[AnyRef]):Long=128L+row.map{case s:String=>s.getBytes(UTF_8).length.toLong;case b:Array[Byte]=>b.length.toLong;case _=>32L}.sum
87 def rows(root:VectorSchemaRoot):Vector[Vector[AnyRef]]=(0 until root.getRowCount).map(i=>root.getFieldVectors.asScala.map(v=>value(v,i)).toVector).toVector
88 def emitRows(s:Schema,rows:Iterator[Vector[AnyRef]],send:VectorSchemaRoot=>Unit):Unit={
89 val allocator=new RootAllocator(MaxBytes);val root=VectorSchemaRoot.create(s,allocator)
90 try rows.grouped(BatchSize).foreach { batch=>check();root.clear();root.allocateNew();batch.zipWithIndex.foreach{case(row,i)=>row.zipWithIndex.foreach{case(x,c)=>set(root.getVector(c),i,x)}};root.setRowCount(batch.size);send(root) }
91 finally {root.close();allocator.close()}
92 }
93 def collect(in:Source):Vector[Vector[AnyRef]]={var size=0L;val result=Vector.newBuilder[Vector[AnyRef]];in.records(r=>rows(r).foreach{row=>size+=estimated(row);require(size<=MaxBytes,"materialized input exceeds 128 MiB");result+=row});result.result()}
94 def projected(root:VectorSchemaRoot,target:Schema,indices:Vector[Int],send:VectorSchemaRoot=>Unit):Unit={
95 val allocator=new RootAllocator(MaxBytes);val out=VectorSchemaRoot.create(target,allocator)
96 try {out.allocateNew();fields(target).zipWithIndex.foreach{case(f,c)=>val from=root.getVector(f.getName);indices.zipWithIndex.foreach{case(i,j)=>if(from==null || from.getField.getType==ArrowType.Null.INSTANCE)out.getVector(c).setNull(j)else out.getVector(c).copyFromSafe(i,j,from)}};out.setRowCount(indices.size);send(out)}finally {out.close();allocator.close()}
97 }
98 def descriptor(schema:Option[Schema]):InferenceResult=InferenceResult(Map("result"->StreamDescriptor(Set("records"),Set("arrow"),schema,if(schema.isDefined)"known"else "unknown")))
99}
100
101/** Each entrypoint shares processing with bounded previews; no input/output data is written to disk. */
102abstract class StreamOperator extends Operator {
103 import Support._
104 protected def ports:Set[String]=Set("source")
105 protected def bytesInput:Boolean=false
106 protected def validate(p:JsonNode):Unit
107 protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult
108 private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit
109 final def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={
110 require(inputs.keySet==ports,"unexpected input ports");validate(p)
111 inputs.values.foreach(d=>require(d.representations.isEmpty || d.representations(if(bytesInput)"bytes"else "records"),"input representation mismatch"))
112 infer(inputs,p)
113 }
114 final def process(ctx:OperatorContext):Unit={
115 require(ctx.inputNames.toSet==ports,"unexpected input ports");validate(ctx.params)
116 execute(ports.map(n=>n->new Source {
117 private def in=ctx.input(n)
118 def schema:Schema=in.schema
119 def records(f:VectorSchemaRoot=>Unit):Unit=in.foreachRecords(f)
120 def bytes:InputStream=in.byteStream()
121 }).toMap,new Sink {
122 def records(s:Schema):VectorSchemaRoot=>Unit={val out=ctx.recordsOutput("result",s);r=>out.write(r)}
123 def bytes():java.io.OutputStream=ctx.bytesOutput("result","csv").byteStream()
124 },ctx.params)
125 }
126 final def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
127 inferOutputs(inputs,p)
128 val allocator=new RootAllocator(MaxBytes)
129 val roots=mutable.ArrayBuffer.empty[VectorSchemaRoot]
130 var sample:OutputSample=null
131 var truncated=false
132 try {
133 val sources=ports.map { n=>
134 val d=inputs(n);val s=d.sample.getOrElse(throw new IllegalArgumentException("Upstream sample unavailable: "+n)).bounded(limits);truncated ||= s.truncated
135 n->new Source {
136 def schema:Schema=d.schema.getOrElse(throw new IllegalArgumentException("Upstream schema unavailable: "+n))
137 def bytes:InputStream=new ByteArrayInputStream(s.data.toArray)
138 def records(f:VectorSchemaRoot=>Unit):Unit={
139 val root=VectorSchemaRoot.create(schema,allocator);roots+=root;root.allocateNew()
140 fields(schema).zipWithIndex.foreach{case(field,c)=>s.rows.zipWithIndex.foreach{case(row,i)=>
141 val v=row.get(field.getName);val x:AnyRef=if(v==null||v.isNull)null else if(v.isNumber)v.numberValue else if(v.isBoolean)java.lang.Boolean.valueOf(v.asBoolean)else if(v.isBinary)v.binaryValue else v.asText
142 set(root.getVector(c),i,x)
143 }};root.setRowCount(s.rows.size);f(root)
144 }
145 }
146 }.toMap
147 val collected=mutable.ArrayBuffer.empty[JsonNode];val sampleBytes=new ByteArrayOutputStream()
148 execute(sources,new Sink {
149 def records(s:Schema):VectorSchemaRoot=>Unit={sample=OutputSample("records","arrow");r=>{
150 (0 until r.getRowCount).foreach{i=>if(collected.size>=limits.maxRows)truncated=true else {
151 val row=mapper.createObjectNode();fields(s).zipWithIndex.foreach{case(f,c)=>row.set[JsonNode](f.getName,mapper.valueToTree(value(r.getVector(c),i)))}
152 val proposed=OutputSample("records","arrow",(collected.toVector:+row)).bounded(limits)
153 if(proposed.rows.size>collected.size)collected+=row else truncated=true
154 }}
155 }}
156 def bytes():java.io.OutputStream={sample=OutputSample("bytes","csv");new java.io.OutputStream {
157 override def write(b:Int):Unit=if(sampleBytes.size<limits.maxBytes)sampleBytes.write(b)else truncated=true
158 }}
159 },p)
160 require(sample!=null,"operator produced no output")
161 Map("result"->sample.copy(rows=collected.toVector,data=sampleBytes.toByteArray.toVector,truncated=truncated).bounded(limits))
162 }finally {roots.foreach(_.close());allocator.close()}
163 }
164}