src/BatchFieldThresholdClean.scala
1package cn.piflow.bundle.flux.pretreatment
2
3
4import cn.piflow.bundle.flux.util.FluxUtil_1
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.ImageUtil
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class BatchFieldThresholdClean extends ConfigurableStop{
12 val authorEmail: String = "ygang@cnic.cn"
13 val description: String = "Batch field threshold cleaning,Include upper and lower limits: The schema of the threshold table must be included 'field','min_value' ,'max_value','flag_value'"
14// override val inportList: List[String] =List(Port.LeftPort,Port.RightPort)
15 override val inportList: List[String] =List("MasterTablePort","ThresholdTablePort")
16 val outportList: List[String] = List(Port.DefaultPort)
17
18
19 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20 val spark = pec.get[SparkSession]()
21
22 val MasterDF = in.read("MasterTablePort")
23 val ThresholdDF = in.read("ThresholdTablePort")
24 val flux = new FluxUtil_1
25 val df = flux.batchFieldThresholdCleaning(spark,MasterDF,ThresholdDF)
26
27 out.write(df)
28 }
29
30 override def setProperties(map: Map[String, Any]): Unit = {
31
32 }
33
34 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
35 var descriptor : List[PropertyDescriptor] = List()
36
37 descriptor
38 }
39
40 override def getIcon(): Array[Byte] = {
41 ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
42 }
43
44 override def getGroup(): List[String] = {
45 List(StopGroup.CleanGroup)
46 }
47
48 override def initialize(ctx: ProcessContext): Unit = {
49
50 }
51
52}