Source code

Download source archive ↓

src/SingleFieldThresholdClean.scala

1package cn.piflow.bundle.flux.pretreatment
2
3
4import cn.piflow.bundle.flux.util.FluxUtil_1
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class SingleFieldThresholdClean extends ConfigurableStop{
12  val authorEmail: String = "ygang@cnic.cn"
13  val description: String = "Threshold cleaning of the fields,Include upper and lower limits"
14  val inportList: List[String] = List(Port.DefaultPort)
15  val outportList: List[String] = List(Port.DefaultPort)
16
17  var fields: String = _
18  var maxValue: String = _
19  var minValue: String = _
20  var flagValue: String = _
21  var additionalCondition: String = _
22
23  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24
25    val spark = pec.get[SparkSession]()
26    val flux = new FluxUtil_1
27    val df = flux.singleFieldThresholdCleaning(spark, in.read(),fields, maxValue, minValue,flagValue,additionalCondition)
28
29    out.write(df)
30  }
31
32  override def setProperties(map: Map[String, Any]): Unit = {
33    fields = MapUtil.get(map,"fields").asInstanceOf[String]
34    maxValue = MapUtil.get(map,"maxValue").asInstanceOf[String]
35    minValue = MapUtil.get(map,"minValue").asInstanceOf[String]
36    flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
37    additionalCondition = MapUtil.get(map,"additionalCondition").asInstanceOf[String]
38
39  }
40
41  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
42    var descriptor : List[PropertyDescriptor] = List()
43
44    val fields = new PropertyDescriptor()
45      .name("fields")
46      .displayName("fields")
47      .description("Threshold processing fields, multiple separated by commas")
48      .defaultValue("")
49      .required(true)
50      .example("a,b")
51    descriptor = fields :: descriptor
52
53    val maxValue = new PropertyDescriptor()
54      .name("maxValue")
55      .displayName("maxValue")
56      .description("Upper threshold")
57      .defaultValue("")
58      .required(true)
59      .example("")
60    descriptor = maxValue :: descriptor
61
62    val minValue = new PropertyDescriptor()
63      .name("minValue")
64      .displayName("minValue")
65      .description("Lower threshold")
66      .defaultValue("")
67      .required(true)
68      .example("")
69    descriptor = minValue :: descriptor
70
71    val flagValue = new PropertyDescriptor()
72      .name("flagValue")
73      .displayName("flagValue")
74      .description("Identification value")
75      .defaultValue("-9999")
76      .required(true)
77      .example("")
78    descriptor = flagValue :: descriptor
79
80    val additionalCondition = new PropertyDescriptor()
81      .name("additionalCondition")
82      .displayName("additionalCondition")
83      .description("Additional condition")
84      .defaultValue("1=1")
85      .required(true)
86      .example("")
87    descriptor = additionalCondition :: descriptor
88
89
90    descriptor
91  }
92
93  override def getIcon(): Array[Byte] = {
94    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
95  }
96
97  override def getGroup(): List[String] = {
98    List(StopGroup.CleanGroup)
99  }
100
101  override def initialize(ctx: ProcessContext): Unit = {
102
103  }
104
105}