Source code

Download source archive ↓

src/Relational.scala

1package xstream.examples.piflow
2
3import com.fasterxml.jackson.databind.JsonNode
4import org.apache.arrow.vector.types.pojo.{ArrowType,Schema}
5import xstream.execution.{StreamDescriptor,InferenceResult}
6import scala.collection.mutable
7import scala.jdk.CollectionConverters._
8
9private[piflow] object Relational {
10  import Support._
11  def merge(left:Schema,right:Schema):Schema={
12    unique(left);unique(right)
13    val l=fields(left).map(f=>f.getName->f).toMap;val r=fields(right).map(f=>f.getName->f).toMap
14    schema((names(left)++names(right)).distinct.map { name=>
15      (l.get(name),r.get(name)) match {
16        case (Some(a),Some(b))=>
17          require((a.getType==b.getType && a.getChildren==b.getChildren && a.getDictionary==b.getDictionary) || a.getType==ArrowType.Null.INSTANCE || b.getType==ArrowType.Null.INSTANCE,"incompatible merge field: "+name)
18          val base=if(a.getType==ArrowType.Null.INSTANCE)b else a
19          nullable(base,a.isNullable||b.isNullable||a.getType==ArrowType.Null.INSTANCE||b.getType==ArrowType.Null.INSTANCE)
20        case (a,b)=>nullable(a.orElse(b).get,true)
21      }
22    },left)
23  }
24  def keys(p:JsonNode):Vector[String]=columns(text(p,"correlationColumn"))
25  def mode(p:JsonNode):String={val m=text(p,"joinMode");require(Set("inner","left","right","full_outer")(m),"joinMode must be inner, left, right or full_outer");m}
26  def join(left:Schema,right:Schema,p:JsonNode):Schema={
27    unique(left);unique(right);val ks=keys(p);val m=mode(p)
28    ks.foreach {k=>val a=left.findField(k);val b=right.findField(k);require(a!=null&&b!=null,"unknown join key: "+k);require(a.getType==b.getType,"join key types differ: "+k)}
29    (fields(left)++fields(right)).foreach(scalar)
30    val fs=fields(left).map(f=>nullable(f,f.isNullable || m=="right" || m=="full_outer"))++fields(right).filterNot(f=>ks.contains(f.getName)).map{f=>nullable(rename(f,if(names(left).contains(f.getName))f.getName+"_right"else f.getName),f.isNullable||m=="left"||m=="full_outer")}
31    require(fs.map(_.getName).distinct.size==fs.size,"join produces ambiguous columns; rename input columns first")
32    schema(fs,left)
33  }
34}
35class MergeOperator extends StreamOperator {
36  import Support._
37  override protected def ports=Set("left","right")
38  protected def validate(p:JsonNode):Unit=params(p,Set.empty)
39  protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult=descriptor(for(l<-inputs("left").schema;r<-inputs("right").schema)yield Relational.merge(l,r))
40  private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit={
41    val s=Relational.merge(inputs("left").schema,inputs("right").schema)
42    require(fields(s).forall(_.getDictionary==null),"merge requires decoded dictionary columns")
43    val send=out.records(s)
44    Vector("left","right").foreach(n=>inputs(n).records(root=>projected(root,s,(0 until root.getRowCount).toVector,send)))
45  }
46}
47object Merge extends MergeOperator
48class JoinOperator extends StreamOperator {
49  import Support._
50  override protected def ports=Set("left","right")
51  protected def validate(p:JsonNode):Unit={params(p,Set("joinMode","correlationColumn"));Relational.keys(p);Relational.mode(p)}
52  protected def infer(inputs:Map[String,StreamDescriptor],p:JsonNode):InferenceResult=descriptor(for(l<-inputs("left").schema;r<-inputs("right").schema)yield Relational.join(l,r,p))
53  private[piflow] def execute(inputs:Map[String,Source],out:Sink,p:JsonNode):Unit={
54    val l=inputs("left").schema;val r=inputs("right").schema;val s=Relational.join(l,r,p)
55    val ks=Relational.keys(p);val m=Relational.mode(p);val li=ks.map(names(l).indexOf);val ri=ks.map(names(r).indexOf)
56    val left=collect(inputs("left"));val right=collect(inputs("right"))
57    val extra=names(r).indices.filterNot(i=>ks.contains(names(r)(i))).toVector
58    def key(row:Vector[AnyRef],is:Vector[Int]):Vector[AnyRef]=is.map(i=>row(i) match {case b:Array[Byte]=>b.toVector;case x=>x})
59    val hash=mutable.HashMap.empty[Vector[AnyRef],mutable.ArrayBuffer[Int]]
60    right.indices.foreach {i=>check();val k=key(right(i),ri);if(!k.contains(null))hash.getOrElseUpdate(k,mutable.ArrayBuffer.empty)+=i}
61    val matched=mutable.BitSet.empty
62    val output=left.iterator.flatMap {row=>check();val k=key(row,li);val matches=if(k.contains(null))Vector.empty else hash.get(k).map(_.toVector).getOrElse(Vector.empty)
63      if(matches.nonEmpty)matches.iterator.map{i=>matched+=i;row++extra.map(right(i))}
64      else if(m=="left"||m=="full_outer")Iterator(row++extra.map(_=>null))else Iterator.empty
65    }++right.indices.iterator.filterNot(matched).filter(_=>m=="right"||m=="full_outer").map {i=>check();val row=right(i);val prefix=names(l).map(n=>if(ks.contains(n))row(names(r).indexOf(n))else null);prefix++extra.map(row)}
66    var size=0L
67    emitRows(s,output.map{row=>size+=estimated(row);require(size<=MaxBytes,"join output exceeds 128 MiB");row},out.records(s))
68  }
69}
70object Join extends JoinOperator