upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/common/Distinct.scala
1package cn.piflow.bundle.generic.common
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.DataFrame
8
9class Distinct extends ConfigurableStop{
10 override val authorEmail: String = "yangqidong@cnic.cn"
11 override val description: String = "Duplicate based on the specified column name or all colume names"
12 override val inportList: List[String] =List(Port.DefaultPort)
13 override val outportList: List[String] = List(Port.DefaultPort)
14
15 var columnNames:String=_
16
17 override def setProperties(map: Map[String, Any]): Unit = {
18 columnNames = MapUtil.get(map,"columnNames").asInstanceOf[String]
19 }
20
21 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
22 var descriptor : List[PropertyDescriptor] = List()
23
24 val fields = new PropertyDescriptor()
25 .name("columnNames")
26 .displayName("ColumnNames")
27 .description("Fill in the column names you want to duplicate,multiple columns names separated by commas,if not,all the columns will be deduplicated")
28 .defaultValue("")
29 .required(false)
30 .example("id")
31 descriptor = fields :: descriptor
32
33 descriptor
34 }
35
36 override def getIcon(): Array[Byte] = {
37 ImageUtil.getImage("icon/common/Distinct.png")
38 }
39
40 override def getGroup(): List[String] = {
41 List(StopGroup.CommonGroup)
42 }
43
44
45 override def initialize(ctx: ProcessContext): Unit = {
46
47 }
48
49 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
50 val inDf: DataFrame = in.read()
51 var outDf: DataFrame = null
52 if(columnNames.length > 0){
53 val fileArr: Array[String] = columnNames.split(",")
54 outDf = inDf.dropDuplicates(fileArr)
55 }else{
56 outDf = inDf.distinct()
57 }
58 out.write(outDf)
59 }
60}