Source code

Download source archive ↓

src/Support.scala

1package xstream.examples.piflow
2
3import java.io.{InputStream,ByteArrayInputStream,ByteArrayOutputStream}
4import java.nio.charset.StandardCharsets.UTF_8
5import com.fasterxml.jackson.databind.{JsonNode,ObjectMapper}
6import org.apache.arrow.memory.RootAllocator
7import org.apache.arrow.vector._
8import org.apache.arrow.vector.types.pojo.{ArrowType,Field,FieldType,Schema}
9import org.apache.arrow.vector.types.FloatingPointPrecision
10import xstream.operator.{Operator,OperatorContext}
11import xstream.execution.{StreamDescriptor,InferenceResult,OutputSample,PreviewLimits}
12import scala.jdk.CollectionConverters._
13import scala.collection.mutable
14
15private[piflow] trait Source {
16  def schema:Schema
17  def records(f:VectorSchemaRoot=>Unit):Unit
18  def bytes:InputStream
19}
20private[piflow] trait Sink {
21  def records(schema:Schema):VectorSchemaRoot=>Unit
22  def bytes():java.io.OutputStream
23}
24private[piflow] object Support {
25  val mapper=new ObjectMapper()
26  val MaxBytes=128L*1024*1024
27  val BatchSize=128
28  def fields(s:Schema):Vector[Field]=s.getFields.asScala.toVector
29  def names(s:Schema):Vector[String]=fields(s).map(_.getName)
30  def unique(s:Schema):Unit=require(names(s).nonEmpty && names(s).distinct.size==names(s).size,"unique, nonempty columns required")
31  def field(name:String,t:ArrowType,nullable:Boolean=true):Field=new Field(name,new FieldType(nullable,t,null),null)
32  def nullable(f:Field,b:Boolean):Field=new Field(f.getName,new FieldType(b,f.getType,f.getDictionary,f.getMetadata),f.getChildren)
33  def rename(f:Field,n:String):Field=new Field(n,f.getFieldType,f.getChildren)
34  def schema(fs:Vector[Field],base:Schema=null):Schema=new Schema(fs.asJava,if(base==null)null else base.getCustomMetadata)
35  def columns(s:String):Vector[String]={val ns=s.split(",",-1).toVector.map(_.trim);require(ns.forall(_.nonEmpty) && ns.distinct.size==ns.size,"unique, nonempty column names required");ns}
36  def text(p:JsonNode,k:String,default:String=null):String={val n=p.get(k);if(n==null){require(default!=null,s"missing parameter: $k");default}else{require(n.isTextual,s"$k must be text");n.asText}}
37  def bool(p:JsonNode,k:String,default:Boolean):Boolean={val n=p.get(k);if(n==null)default else {require(n.isBoolean,s"$k must be boolean");n.asBoolean}}
38  def params(p:JsonNode,required:Set[String],optional:Set[String]=Set.empty):Unit={require(p.isObject,"parameters must be an object");val ks=p.fieldNames.asScala.toSet;require(required.subsetOf(ks) && ks.subsetOf(required++optional),"missing or unexpected parameters")}
39  def check():Unit=if(Thread.currentThread.isInterrupted)throw new java.util.concurrent.CancellationException()
40  def scalar(f:Field):Unit=require(f.getChildren.isEmpty && f.getDictionary==null && !f.getType.isInstanceOf[ArrowType.Union],"scalar, non-dictionary columns required")
41  def value(v:FieldVector,i:Int):AnyRef=v.getObject(i) match {
42    case null=>null
43    case t:org.apache.arrow.vector.util.Text=>t.toString
44    case x=>x
45  }
46  def set(v:FieldVector,i:Int,x:AnyRef):Unit={
47    if(x==null){v.setNull(i);return}
48    v match {
49      case z:VarCharVector=>z.setSafe(i,x.toString.getBytes(UTF_8))
50      case z:VarBinaryVector=>z.setSafe(i,x.asInstanceOf[Array[Byte]])
51      case z:BigIntVector=>z.setSafe(i,x.asInstanceOf[Number].longValue)
52      case z:IntVector=>z.setSafe(i,x.asInstanceOf[Number].intValue)
53      case z:SmallIntVector=>z.setSafe(i,x.asInstanceOf[Number].shortValue)
54      case z:TinyIntVector=>z.setSafe(i,x.asInstanceOf[Number].byteValue)
55      case z:Float8Vector=>z.setSafe(i,x.asInstanceOf[Number].doubleValue)
56      case z:Float4Vector=>z.setSafe(i,x.asInstanceOf[Number].floatValue)
57      case z:BitVector=>z.setSafe(i,if(x==java.lang.Boolean.TRUE || x==Integer.valueOf(1))1 else 0)
58      case z:DecimalVector=>z.setSafe(i,new java.math.BigDecimal(x.toString))
59      case z:NullVector=>require(x==null,"null field cannot contain a value")
60      case _=>throw new IllegalArgumentException("sample/SQL value unsupported for "+v.getField.getType)
61    }
62  }
63  def estimated(row:Vector[AnyRef]):Long=128L+row.map{case s:String=>s.getBytes(UTF_8).length.toLong;case b:Array[Byte]=>b.length.toLong;case _=>32L}.sum
64  def rows(root:VectorSchemaRoot):Vector[Vector[AnyRef]]=(0 until root.getRowCount).map(i=>root.getFieldVectors.asScala.map(v=>value(v,i)).toVector).toVector
65  def emitRows(s:Schema,rows:Iterator[Vector[AnyRef]],send:VectorSchemaRoot=>Unit):Unit={
66    val allocator=new RootAllocator(MaxBytes);val root=VectorSchemaRoot.create(s,allocator)
67    try rows.grouped(BatchSize).foreach { batch=>check();root.clear();root.allocateNew();batch.zipWithIndex.foreach{case(row,i)=>row.zipWithIndex.foreach{case(x,c)=>set(root.getVector(c),i,x)}};root.setRowCount(batch.size);send(root) }
68    finally {root.close();allocator.close()}
69  }
70  def collect(in:Source):Vector[Vector[AnyRef]]={var size=0L;val result=Vector.newBuilder[Vector[AnyRef]];in.records(r=>rows(r).foreach{row=>size+=estimated(row);require(size<=MaxBytes,"materialized input exceeds 128 MiB");result+=row});result.result()}
71  def projected(root:VectorSchemaRoot,target:Schema,indices:Vector[Int],send:VectorSchemaRoot=>Unit):Unit={
72    val allocator=new RootAllocator(MaxBytes);val out=VectorSchemaRoot.create(target,allocator)
73    try {out.allocateNew();fields(target).zipWithIndex.foreach{case(f,c)=>val from=root.getVector(f.getName);indices.zipWithIndex.foreach{case(i,j)=>if(from==null || from.getField.getType==ArrowType.Null.INSTANCE)out.getVector(c).setNull(j)else out.getVector(c).copyFromSafe(i,j,from)}};out.setRowCount(indices.size);send(out)}finally {out.close();allocator.close()}
74  }
75  def descriptor(schema:Option[Schema]):InferenceResult=InferenceResult(Map("result"->StreamDescriptor(Set("records"),Set("arrow"),schema,if(schema.isDefined)"known"else "unknown")))
76}
77
78/** Each entrypoint shares processing with bounded previews; no input/output data is written to disk. */
79abstract class StreamOperator extends Operator {
80  import Support._
81  protected def ports:Set[String]=Set("source")
82  protected def bytesInput:Boolean=false
83  protected def validate(p:JsonNode):Unit
84  protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult
85  private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit
86  final def inferOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult={
87    require(inputs.keySet==ports,"unexpected input ports");validate(p)
88    inputs.values.foreach(d=>require(d.representations.isEmpty || d.representations(if(bytesInput)"bytes"else "records"),"input representation mismatch"))
89    infer(inputs,p)
90  }
91  final def process(ctx:OperatorContext):Unit={
92    require(ctx.inputNames.toSet==ports,"unexpected input ports");validate(ctx.params)
93    execute(ports.map(n=>n->new Source {
94      private def in=ctx.input(n)
95      def schema:Schema=in.schema
96      def records(f:VectorSchemaRoot=>Unit):Unit=in.foreachRecords(f)
97      def bytes:InputStream=in.byteStream()
98    }).toMap,new Sink {
99      def records(s:Schema):VectorSchemaRoot=>Unit={val out=ctx.recordsOutput("result",s);r=>out.write(r)}
100      def bytes():java.io.OutputStream=ctx.bytesOutput("result","csv").byteStream()
101    },ctx.params)
102  }
103  final def previewOutputs(inputs:Map[String,StreamDescriptor],p:JsonNode,limits:PreviewLimits):Map[String,OutputSample]={
104    inferOutputs(inputs,p)
105    val allocator=new RootAllocator(MaxBytes)
106    val roots=mutable.ArrayBuffer.empty[VectorSchemaRoot]
107    var sample:OutputSample=null
108    var truncated=false
109    try {
110      val sources=ports.map { n=>
111        val d=inputs(n);val s=d.sample.getOrElse(throw new IllegalArgumentException("Upstream sample unavailable: "+n)).bounded(limits);truncated ||= s.truncated
112        n->new Source {
113          def schema:Schema=d.schema.getOrElse(throw new IllegalArgumentException("Upstream schema unavailable: "+n))
114          def bytes:InputStream=new ByteArrayInputStream(s.data.toArray)
115          def records(f:VectorSchemaRoot=>Unit):Unit={
116            val root=VectorSchemaRoot.create(schema,allocator);roots+=root;root.allocateNew()
117            fields(schema).zipWithIndex.foreach{case(field,c)=>s.rows.zipWithIndex.foreach{case(row,i)=>
118              val v=row.get(field.getName);val x:AnyRef=if(v==null||v.isNull)null else if(v.isNumber)v.numberValue else if(v.isBoolean)java.lang.Boolean.valueOf(v.asBoolean)else if(v.isBinary)v.binaryValue else v.asText
119              set(root.getVector(c),i,x)
120            }};root.setRowCount(s.rows.size);f(root)
121          }
122        }
123      }.toMap
124      val collected=mutable.ArrayBuffer.empty[JsonNode];val sampleBytes=new ByteArrayOutputStream()
125      execute(sources,new Sink {
126        def records(s:Schema):VectorSchemaRoot=>Unit={sample=OutputSample("records","arrow");r=>{
127          (0 until r.getRowCount).foreach{i=>if(collected.size>=limits.maxRows)truncated=true else {
128            val row=mapper.createObjectNode();fields(s).zipWithIndex.foreach{case(f,c)=>row.set[JsonNode](f.getName,mapper.valueToTree(value(r.getVector(c),i)))}
129            val proposed=OutputSample("records","arrow",(collected.toVector:+row)).bounded(limits)
130            if(proposed.rows.size>collected.size)collected+=row else truncated=true
131          }}
132        }}
133        def bytes():java.io.OutputStream={sample=OutputSample("bytes","csv");new java.io.OutputStream {
134          override def write(b:Int):Unit=if(sampleBytes.size<limits.maxBytes)sampleBytes.write(b)else truncated=true
135        }}
136      },p)
137      require(sample!=null,"operator produced no output")
138      Map("result"->sample.copy(rows=collected.toVector,data=sampleBytes.toByteArray.toVector,truncated=truncated).bounded(limits))
139    }finally {roots.foreach(_.close());allocator.close()}
140  }
141}