src/Support.scala
1package xstream.examples.piflow
2
3import java.io.{InputStream,ByteArrayInputStream,ByteArrayOutputStream}
4import java.nio.charset.StandardCharsets.UTF_8
5import com.fasterxml.jackson.databind.{JsonNode,ObjectMapper}
6import org.apache.arrow.memory.RootAllocator
7import org.apache.arrow.vector._
8import org.apache.arrow.vector.types.pojo.{ArrowType,Field,FieldType,Schema}
9import org.apache.arrow.vector.types.FloatingPointPrecision
10import xstream.operator.{Operator,OperatorContext}
11import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
12import scala.jdk.CollectionConverters._
13import scala.collection.mutable
14
15private[piflow] trait Source {
16 def schema:Schema
17 def records(f:VectorSchemaRoot=>Unit):Unit
18 def bytes:InputStream
19}
20private[piflow] trait Sink {
21 def records(schema:Schema):VectorSchemaRoot=>Unit
22 def bytes():java.io.OutputStream
23}
24private[piflow] object Support {
25 val mapper=new ObjectMapper()
26 val MaxBytes=128L*1024*1024
27 val BatchSize=128
28 def fields(s:Schema):Vector[Field]=s.getFields.asScala.toVector
29 def names(s:Schema):Vector[String]=fields(s).map(_.getName)
30 def unique(s:Schema):Unit=require(names(s).nonEmpty && names(s).distinct.size==names(s).size,"unique, nonempty columns required")
31 def field(name:String,t:ArrowType,nullable:Boolean=true):Field=new Field(name,new FieldType(nullable,t,null),null)
32 def nullable(f:Field,b:Boolean):Field=new Field(f.getName,new FieldType(b,f.getType,f.getDictionary,f.getMetadata),f.getChildren)
33 def rename(f:Field,n:String):Field=new Field(n,f.getFieldType,f.getChildren)
34 def schema(fs:Vector[Field],base:Schema=null):Schema=new Schema(fs.asJava,if(base==null)null else base.getCustomMetadata)
35 def columns(s:String):Vector[String]={val ns=s.split(",",-1).toVector.map(_.trim);require(ns.forall(_.nonEmpty) && ns.distinct.size==ns.size,"unique, nonempty column names required");ns}
36 def text(p:JsonNode,k:String,default:String=null):String={val n=p.get(k);if(n==null){require(default!=null,s"missing parameter: $k");default}else{require(n.isTextual,s"$k must be text");n.asText}}
37 def bool(p:JsonNode,k:String,default:Boolean):Boolean={val n=p.get(k);if(n==null)default else {require(n.isBoolean,s"$k must be boolean");n.asBoolean}}
38 def params(p:JsonNode,required:Set[String],optional:Set[String]=Set.empty):Unit={require(p.isObject,"parameters must be an object");val ks=p.fieldNames.asScala.toSet;require(required.subsetOf(ks) && ks.subsetOf(required++optional),"missing or unexpected parameters")}
39 def check():Unit=if(Thread.currentThread.isInterrupted)throw new java.util.concurrent.CancellationException()
40 def scalar(f:Field):Unit=require(f.getChildren.isEmpty && f.getDictionary==null && !f.getType.isInstanceOf[ArrowType.Union],"scalar, non-dictionary columns required")
41 def value(v:FieldVector,i:Int):AnyRef=v.getObject(i) match {
42 case null=>null
43 case t:org.apache.arrow.vector.util.Text=>t.toString
44 case x=>x
45 }
46 def set(v:FieldVector,i:Int,x:AnyRef):Unit={
47 if(x==null){v.setNull(i);return}
48 v match {
49 case z:VarCharVector=>z.setSafe(i,x.toString.getBytes(UTF_8))
50 case z:VarBinaryVector=>z.setSafe(i,x.asInstanceOf[Array[Byte]])
51 case z:BigIntVector=>z.setSafe(i,x.asInstanceOf[Number].longValue)
52 case z:IntVector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
53 case z:SmallIntVector=>z.setSafe(i,x.asInstanceOf[Number].shortValue)
54 case z:TinyIntVector=>z.setSafe(i,x.asInstanceOf[Number].byteValue)
55 case z:Float8Vector=>z.setSafe(i,x.asInstanceOf[Number].doubleValue)
56 case z:Float4Vector=>z.setSafe(i,x.asInstanceOf[Number].floatValue)
57 case z:BitVector=>z.setSafe(i,if(x==java.lang.Boolean.TRUE || x==Integer.valueOf(1))1 else 0)
58 case z:DecimalVector=>z.setSafe(i,new java.math.BigDecimal(x.toString))
59 case z:NullVector=>require(x==null,"null field cannot contain a value")
60 case _=>throw new IllegalArgumentException("sample/SQL value unsupported for "+v.getField.getType)
61 }
62 }
63 def estimated(row:Vector[AnyRef]):Long=128L+row.map{case s:String=>s.getBytes(UTF_8).length.toLong;case b:Array[Byte]=>b.length.toLong;case _=>32L}.sum
64 def rows(root:VectorSchemaRoot):Vector[Vector[AnyRef]]=(0 until root.getRowCount).map(i=>root.getFieldVectors.asScala.map(v=>value(v,i)).toVector).toVector
65 def emitRows(s:Schema,rows:Iterator[Vector[AnyRef]],send:VectorSchemaRoot=>Unit):Unit={
66 val allocator=new RootAllocator(MaxBytes);val root=VectorSchemaRoot.create(s,allocator)
67 try rows.grouped(BatchSize).foreach { batch=>check();root.clear();root.allocateNew();batch.zipWithIndex.foreach{case(row,i)=>row.zipWithIndex.foreach{case(x,c)=>set(root.getVector(c),i,x)}};root.setRowCount(batch.size);send(root) }
68 finally {root.close();allocator.close()}
69 }
70 def collect(in:Source):Vector[Vector[AnyRef]]={var size=0L;val result=Vector.newBuilder[Vector[AnyRef]];in.records(r=>rows(r).foreach{row=>size+=estimated(row);require(size<=MaxBytes,"materialized input exceeds 128 MiB");result+=row});result.result()}
71 def projected(root:VectorSchemaRoot,target:Schema,indices:Vector[Int],send:VectorSchemaRoot=>Unit):Unit={
72 val allocator=new RootAllocator(MaxBytes);val out=VectorSchemaRoot.create(target,allocator)
73 try {out.allocateNew();fields(target).zipWithIndex.foreach{case(f,c)=>val from=root.getVector(f.getName);indices.zipWithIndex.foreach{case(i,j)=>if(from==null || from.getField.getType==ArrowType.Null.INSTANCE)out.getVector(c).setNull(j)else out.getVector(c).copyFromSafe(i,j,from)}};out.setRowCount(indices.size);send(out)}finally {out.close();allocator.close()}
74 }
75 def descriptor(schema:Option[Schema]):InferenceResult=InferenceResult(Map("result"->StreamDescriptor(Set("records"),Set("arrow"),schema,if(schema.isDefined)"known"else "unknown")))
76}
77
78/** Each entrypoint shares processing with bounded previews; no input/output data is written to disk. */
79abstract class StreamOperator extends Operator {
80 import Support._
81 protected def ports:Set[String]=Set("source")
82 protected def bytesInput:Boolean=false
83 protected def validate(p:JsonNode):Unit
84 protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult
85 private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit
86 final def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={
87 require(inputs.keySet==ports,"unexpected input ports");validate(p)
88 inputs.values.foreach(d=>require(d.representations.isEmpty || d.representations(if(bytesInput)"bytes"else "records"),"input representation mismatch"))
89 infer(inputs,p)
90 }
91 final def process(ctx:OperatorContext):Unit={
92 require(ctx.inputNames.toSet==ports,"unexpected input ports");validate(ctx.params)
93 execute(ports.map(n=>n->new Source {
94 private def in=ctx.input(n)
95 def schema:Schema=in.schema
96 def records(f:VectorSchemaRoot=>Unit):Unit=in.foreachRecords(f)
97 def bytes:InputStream=in.byteStream()
98 }).toMap,new Sink {
99 def records(s:Schema):VectorSchemaRoot=>Unit={val out=ctx.recordsOutput("result",s);r=>out.write(r)}
100 def bytes():java.io.OutputStream=ctx.bytesOutput("result","csv").byteStream()
101 },ctx.params)
102 }
103 final def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
104 inferOutputs(inputs,p)
105 val allocator=new RootAllocator(MaxBytes)
106 val roots=mutable.ArrayBuffer.empty[VectorSchemaRoot]
107 var sample:OutputSample=null
108 var truncated=false
109 try {
110 val sources=ports.map { n=>
111 val d=inputs(n);val s=d.sample.getOrElse(throw new IllegalArgumentException("Upstream sample unavailable: "+n)).bounded(limits);truncated ||= s.truncated
112 n->new Source {
113 def schema:Schema=d.schema.getOrElse(throw new IllegalArgumentException("Upstream schema unavailable: "+n))
114 def bytes:InputStream=new ByteArrayInputStream(s.data.toArray)
115 def records(f:VectorSchemaRoot=>Unit):Unit={
116 val root=VectorSchemaRoot.create(schema,allocator);roots+=root;root.allocateNew()
117 fields(schema).zipWithIndex.foreach{case(field,c)=>s.rows.zipWithIndex.foreach{case(row,i)=>
118 val v=row.get(field.getName);val x:AnyRef=if(v==null||v.isNull)null else if(v.isNumber)v.numberValue else if(v.isBoolean)java.lang.Boolean.valueOf(v.asBoolean)else if(v.isBinary)v.binaryValue else v.asText
119 set(root.getVector(c),i,x)
120 }};root.setRowCount(s.rows.size);f(root)
121 }
122 }
123 }.toMap
124 val collected=mutable.ArrayBuffer.empty[JsonNode];val sampleBytes=new ByteArrayOutputStream()
125 execute(sources,new Sink {
126 def records(s:Schema):VectorSchemaRoot=>Unit={sample=OutputSample("records","arrow");r=>{
127 (0 until r.getRowCount).foreach{i=>if(collected.size>=limits.maxRows)truncated=true else {
128 val row=mapper.createObjectNode();fields(s).zipWithIndex.foreach{case(f,c)=>row.set[JsonNode](f.getName,mapper.valueToTree(value(r.getVector(c),i)))}
129 val proposed=OutputSample("records","arrow",(collected.toVector:+row)).bounded(limits)
130 if(proposed.rows.size>collected.size)collected+=row else truncated=true
131 }}
132 }}
133 def bytes():java.io.OutputStream={sample=OutputSample("bytes","csv");new java.io.OutputStream {
134 override def write(b:Int):Unit=if(sampleBytes.size<limits.maxBytes)sampleBytes.write(b)else truncated=true
135 }}
136 },p)
137 require(sample!=null,"operator produced no output")
138 Map("result"->sample.copy(rows=collected.toVector,data=sampleBytes.toByteArray.toVector,truncated=truncated).bounded(limits))
139 }finally {roots.foreach(_.close());allocator.close()}
140 }
141}