upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/flux/dataCheck/ConsistencyCheck.scala
1package cn.piflow.bundle.flux.dataCheck
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.SparkSession
8
9class ConsistencyCheck extends ConfigurableStop{
10 override val authorEmail: String = "ygang@cnic.cn"
11 override val description: String = "Quality control of grassland aboveground biomass data,Consistency check"
12 override val inportList: List[String] =List(Port.LeftPort,Port.RightPort)
13 override val outportList: List[String] = List(Port.DefaultPort)
14
15 var flagField:String=_
16 var correlationColumn:String=_
17
18 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
19 val spark = pec.get[SparkSession]()
20 in.read(Port.LeftPort).createOrReplaceTempView("leftTable")
21 in.read(Port.RightPort).createOrReplaceTempView("rightTable")
22
23 val strings = correlationColumn.split(",")
24 val stringBuilder = new StringBuilder
25 strings.foreach(x=>{
26 stringBuilder.append(" a."+x + "=b."+x)
27 stringBuilder.append(" and")
28 })
29
30 val outdf = spark.sql(
31 s"""
32 |select if(b.${strings(0)} is null ,1,0) as ${flagField} ,a.* from leftTable a
33 | left join rightTable b on ${stringBuilder.toString().stripSuffix("and")}
34 |""".stripMargin)
35
36 out.write(outdf)
37 }
38
39 override def setProperties(map: Map[String, Any]): Unit = {
40 flagField = MapUtil.get(map,"flagField").asInstanceOf[String]
41 correlationColumn = MapUtil.get(map,"correlationColumn").asInstanceOf[String]
42 }
43
44 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
45 var descriptor : List[PropertyDescriptor] = List()
46
47 val joinMode = new PropertyDescriptor()
48 .name("flagField")
49 .displayName("flagField")
50 .description("Consistency check and set the flag field to 0 consistently and 1 otherwise")
51 .defaultValue("flag")
52 .required(true)
53 .example("flag")
54 descriptor = joinMode :: descriptor
55
56 val correlationColumn = new PropertyDescriptor()
57 .name("correlationColumn")
58 .displayName("CorrelationColumn")
59 .description("Columns associated with tables,if multiple are separated by commas")
60 .defaultValue("")
61 .required(true)
62 .example("id,name")
63 descriptor = correlationColumn :: descriptor
64
65 descriptor
66 }
67
68 override def getIcon(): Array[Byte] = {
69 ImageUtil.getImage("png/common/Join.png",this.getClass.getName)
70 }
71
72 override def getGroup(): List[String] = {
73 List(StopGroup.ExcelGroup)
74 }
75
76
77 override def initialize(ctx: ProcessContext): Unit = {
78
79 }
80
81}