Source code

Download source archive ↓

src/BatchFieldThresholdClean.scala

1package cn.piflow.bundle.flux.pretreatment
2
3
4import cn.piflow.bundle.flux.util.FluxUtil_1
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.ImageUtil
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class BatchFieldThresholdClean extends ConfigurableStop{
12  val authorEmail: String = "ygang@cnic.cn"
13  val description: String = "Batch field threshold cleaning,Include upper and lower limits: The schema of the threshold table must be included 'field','min_value' ,'max_value','flag_value'"
14//  override val inportList: List[String] =List(Port.LeftPort,Port.RightPort)
15  override val inportList: List[String] =List("MasterTablePort","ThresholdTablePort")
16  val outportList: List[String] = List(Port.DefaultPort)
17
18
19  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20    val spark = pec.get[SparkSession]()
21
22    val MasterDF =  in.read("MasterTablePort")
23    val ThresholdDF = in.read("ThresholdTablePort")
24    val flux = new FluxUtil_1
25    val df = flux.batchFieldThresholdCleaning(spark,MasterDF,ThresholdDF)
26
27    out.write(df)
28  }
29
30  override def setProperties(map: Map[String, Any]): Unit = {
31
32  }
33
34  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
35    var descriptor : List[PropertyDescriptor] = List()
36
37    descriptor
38  }
39
40  override def getIcon(): Array[Byte] = {
41    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
42  }
43
44  override def getGroup(): List[String] = {
45    List(StopGroup.CleanGroup)
46  }
47
48  override def initialize(ctx: ProcessContext): Unit = {
49
50  }
51
52}