Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/flux/dataCheck/ConsistencyCheck.scala

1package cn.piflow.bundle.flux.dataCheck
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.SparkSession
8
9class ConsistencyCheck extends ConfigurableStop{
10  override val authorEmail: String = "ygang@cnic.cn"
11  override val description: String = "Quality control of grassland aboveground biomass data,Consistency check"
12  override val inportList: List[String] =List(Port.LeftPort,Port.RightPort)
13  override val outportList: List[String] = List(Port.DefaultPort)
14
15  var flagField:String=_
16  var correlationColumn:String=_
17
18  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
19    val spark = pec.get[SparkSession]()
20    in.read(Port.LeftPort).createOrReplaceTempView("leftTable")
21    in.read(Port.RightPort).createOrReplaceTempView("rightTable")
22
23    val strings = correlationColumn.split(",")
24    val stringBuilder = new StringBuilder
25    strings.foreach(x=>{
26      stringBuilder.append(" a."+x + "=b."+x)
27      stringBuilder.append(" and")
28    })
29
30    val outdf = spark.sql(
31      s"""
32         |select if(b.${strings(0)} is null ,1,0) as  ${flagField} ,a.*    from leftTable a
33         | left join rightTable b on ${stringBuilder.toString().stripSuffix("and")}
34         |""".stripMargin)
35
36    out.write(outdf)
37  }
38
39  override def setProperties(map: Map[String, Any]): Unit = {
40    flagField = MapUtil.get(map,"flagField").asInstanceOf[String]
41    correlationColumn = MapUtil.get(map,"correlationColumn").asInstanceOf[String]
42  }
43
44  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
45    var descriptor : List[PropertyDescriptor] = List()
46
47    val joinMode = new PropertyDescriptor()
48      .name("flagField")
49      .displayName("flagField")
50      .description("Consistency check and set the flag field to 0 consistently and 1 otherwise")
51      .defaultValue("flag")
52      .required(true)
53      .example("flag")
54    descriptor = joinMode :: descriptor
55
56    val correlationColumn = new PropertyDescriptor()
57      .name("correlationColumn")
58      .displayName("CorrelationColumn")
59      .description("Columns associated with tables,if multiple are separated by commas")
60      .defaultValue("")
61      .required(true)
62      .example("id,name")
63    descriptor = correlationColumn :: descriptor
64
65    descriptor
66  }
67
68  override def getIcon(): Array[Byte] = {
69    ImageUtil.getImage("png/common/Join.png",this.getClass.getName)
70  }
71
72  override def getGroup(): List[String] = {
73    List(StopGroup.ExcelGroup)
74  }
75
76
77  override def initialize(ctx: ProcessContext): Unit = {
78
79  }
80
81}