upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/common/Subtract.scala
1package cn.piflow.bundle.generic.common
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.ImageUtil
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.api.java.JavaRDD
8import org.apache.spark.sql.types.StructType
9import org.apache.spark.sql.{DataFrame, Row, SparkSession}
10
11class Subtract extends ConfigurableStop{
12 override val authorEmail: String = "yangqidong@cnic.cn"
13 override val description: String = "Delete the existing data in the right table from the left table"
14 override val inportList: List[String] =List(Port.LeftPort,Port.RightPort)
15 override val outportList: List[String] = List(Port.DefaultPort)
16
17 override def setProperties(map: Map[String, Any]): Unit = {
18 }
19
20 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
21 var descriptor : List[PropertyDescriptor] = List()
22
23 descriptor
24 }
25
26 override def getIcon(): Array[Byte] = {
27 ImageUtil.getImage("icon/common/Subtract.png")
28 }
29
30 override def getGroup(): List[String] = {
31 List(StopGroup.CommonGroup)
32 }
33
34 override def initialize(ctx: ProcessContext): Unit = {
35
36 }
37
38 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
39 val spark = pec.get[SparkSession]()
40
41 val leftDF = in.read(Port.LeftPort)
42 val rightDF = in.read(Port.RightPort)
43
44 val outDF = leftDF.except(rightDF)
45
46 out.write(outDF)
47 }
48}