src/Relational.scala
1package xstream.examples.piflow
2
3import com.fasterxml.jackson.databind.JsonNode
4import org.apache.arrow.vector.types.pojo.{ArrowType,Schema}
5import xstream.execution.{StreamDescriptor,InferenceResult}
6import scala.collection.mutable
7import scala.jdk.CollectionConverters._
8
9private[piflow] object Relational {
10 import Support._
11 def merge(left:Schema,right:Schema):Schema={
12 unique(left);unique(right)
13 val l=fields(left).map(f=>f.getName->f).toMap;val r=fields(right).map(f=>f.getName->f).toMap
14 schema((names(left)++names(right)).distinct.map { name=>
15 (l.get(name),r.get(name)) match {
16 case (Some(a),Some(b))=>
17 require((a.getType==b.getType && a.getChildren==b.getChildren && a.getDictionary==b.getDictionary) || a.getType==ArrowType.Null.INSTANCE || b.getType==ArrowType.Null.INSTANCE,"incompatible merge field: "+name)
18 val base=if(a.getType==ArrowType.Null.INSTANCE)b else a
19 nullable(base,a.isNullable||b.isNullable||a.getType==ArrowType.Null.INSTANCE||b.getType==ArrowType.Null.INSTANCE)
20 case (a,b)=>nullable(a.orElse(b).get,true)
21 }
22 },left)
23 }
24 def keys(p:JsonNode):Vector[String]=columns(text(p,"correlationColumn"))
25 def mode(p:JsonNode):String={val m=text(p,"joinMode");require(Set("inner","left","right","full_outer")(m),"joinMode must be inner, left, right or full_outer");m}
26 def join(left:Schema,right:Schema,p:JsonNode):Schema={
27 unique(left);unique(right);val ks=keys(p);val m=mode(p)
28 ks.foreach {k=>val a=left.findField(k);val b=right.findField(k);require(a!=null&&b!=null,"unknown join key: "+k);require(a.getType==b.getType,"join key types differ: "+k)}
29 (fields(left)++fields(right)).foreach(scalar)
30 val fs=fields(left).map(f=>nullable(f,f.isNullable || m=="right" || m=="full_outer"))++fields(right).filterNot(f=>ks.contains(f.getName)).map{f=>nullable(rename(f,if(names(left).contains(f.getName))f.getName+"_right"else f.getName),f.isNullable||m=="left"||m=="full_outer")}
31 require(fs.map(_.getName).distinct.size==fs.size,"join produces ambiguous columns; rename input columns first")
32 schema(fs,left)
33 }
34}
35class MergeOperator extends StreamOperator {
36 import Support._
37 override protected def ports=Set("left","right")
38 protected def validate(p:JsonNode):Unit=params(p,Set.empty)
39 protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult=descriptor(for(l<-inputs("left").schema;r<-inputs("right").schema)yield Relational.merge(l,r))
40 private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit={
41 val s=Relational.merge(inputs("left").schema,inputs("right").schema)
42 require(fields(s).forall(_.getDictionary==null),"merge requires decoded dictionary columns")
43 val send=out.records(s)
44 Vector("left","right").foreach(n=>inputs(n).records(root=>projected(root,s,(0 until root.getRowCount).toVector,send)))
45 }
46}
47object Merge extends MergeOperator
48class JoinOperator extends StreamOperator {
49 import Support._
50 override protected def ports=Set("left","right")
51 protected def validate(p:JsonNode):Unit={params(p,Set("joinMode","correlationColumn"));Relational.keys(p);Relational.mode(p)}
52 protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult=descriptor(for(l<-inputs("left").schema;r<-inputs("right").schema)yield Relational.join(l,r,p))
53 private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit={
54 val l=inputs("left").schema;val r=inputs("right").schema;val s=Relational.join(l,r,p)
55 val ks=Relational.keys(p);val m=Relational.mode(p);val li=ks.map(names(l).indexOf);val ri=ks.map(names(r).indexOf)
56 val left=collect(inputs("left"));val right=collect(inputs("right"))
57 val extra=names(r).indices.filterNot(i=>ks.contains(names(r)(i))).toVector
58 def key(row:Vector[AnyRef],is:Vector[Int]):Vector[AnyRef]=is.map(i=>row(i) match {case b:Array[Byte]=>b.toVector;case x=>x})
59 val hash=mutable.HashMap.empty[Vector[AnyRef],mutable.ArrayBuffer[Int]]
60 right.indices.foreach {i=>check();val k=key(right(i),ri);if(!k.contains(null))hash.getOrElseUpdate(k,mutable.ArrayBuffer.empty)+=i}
61 val matched=mutable.BitSet.empty
62 val output=left.iterator.flatMap {row=>check();val k=key(row,li);val matches=if(k.contains(null))Vector.empty else hash.get(k).map(_.toVector).getOrElse(Vector.empty)
63 if(matches.nonEmpty)matches.iterator.map{i=>matched+=i;row++extra.map(right(i))}
64 else if(m=="left"||m=="full_outer")Iterator(row++extra.map(_=>null))else Iterator.empty
65 }++right.indices.iterator.filterNot(matched).filter(_=>m=="right"||m=="full_outer").map {i=>check();val row=right(i);val prefix=names(l).map(n=>if(ks.contains(n))row(names(r).indexOf(n))else null);prefix++extra.map(row)}
66 var size=0L
67 emitRows(s,output.map{row=>size+=estimated(row);require(size<=MaxBytes,"join output exceeds 128 MiB");row},out.records(s))
68 }
69}
70object Join extends JoinOperator