Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/common/Subtract.scala

1package cn.piflow.bundle.generic.common
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.ImageUtil
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.api.java.JavaRDD
8import org.apache.spark.sql.types.StructType
9import org.apache.spark.sql.{DataFrame, Row, SparkSession}
10
11class Subtract extends ConfigurableStop{
12  override val authorEmail: String = "yangqidong@cnic.cn"
13  override val description: String = "Delete the existing data in the right table from the left table"
14  override val inportList: List[String] =List(Port.LeftPort,Port.RightPort)
15  override val outportList: List[String] = List(Port.DefaultPort)
16
17  override def setProperties(map: Map[String, Any]): Unit = {
18  }
19
20  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
21    var descriptor : List[PropertyDescriptor] = List()
22
23    descriptor
24  }
25
26  override def getIcon(): Array[Byte] = {
27    ImageUtil.getImage("icon/common/Subtract.png")
28  }
29
30  override def getGroup(): List[String] = {
31    List(StopGroup.CommonGroup)
32  }
33
34  override def initialize(ctx: ProcessContext): Unit = {
35
36  }
37
38  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
39    val spark = pec.get[SparkSession]()
40
41    val leftDF =  in.read(Port.LeftPort)
42    val rightDF = in.read(Port.RightPort)
43
44    val outDF = leftDF.except(rightDF)
45
46    out.write(outDF)
47  }
48}