src/SingleFieldThresholdClean.scala
1package cn.piflow.bundle.flux.pretreatment
2
3
4import cn.piflow.bundle.flux.util.FluxUtil_1
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class SingleFieldThresholdClean extends ConfigurableStop{
12 val authorEmail: String = "ygang@cnic.cn"
13 val description: String = "Threshold cleaning of the fields,Include upper and lower limits"
14 val inportList: List[String] = List(Port.DefaultPort)
15 val outportList: List[String] = List(Port.DefaultPort)
16
17 var fields: String = _
18 var maxValue: String = _
19 var minValue: String = _
20 var flagValue: String = _
21 var additionalCondition: String = _
22
23 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24
25 val spark = pec.get[SparkSession]()
26 val flux = new FluxUtil_1
27 val df = flux.singleFieldThresholdCleaning(spark, in.read(),fields, maxValue, minValue,flagValue,additionalCondition)
28
29 out.write(df)
30 }
31
32 override def setProperties(map: Map[String, Any]): Unit = {
33 fields = MapUtil.get(map,"fields").asInstanceOf[String]
34 maxValue = MapUtil.get(map,"maxValue").asInstanceOf[String]
35 minValue = MapUtil.get(map,"minValue").asInstanceOf[String]
36 flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
37 additionalCondition = MapUtil.get(map,"additionalCondition").asInstanceOf[String]
38
39 }
40
41 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
42 var descriptor : List[PropertyDescriptor] = List()
43
44 val fields = new PropertyDescriptor()
45 .name("fields")
46 .displayName("fields")
47 .description("Threshold processing fields, multiple separated by commas")
48 .defaultValue("")
49 .required(true)
50 .example("a,b")
51 descriptor = fields :: descriptor
52
53 val maxValue = new PropertyDescriptor()
54 .name("maxValue")
55 .displayName("maxValue")
56 .description("Upper threshold")
57 .defaultValue("")
58 .required(true)
59 .example("")
60 descriptor = maxValue :: descriptor
61
62 val minValue = new PropertyDescriptor()
63 .name("minValue")
64 .displayName("minValue")
65 .description("Lower threshold")
66 .defaultValue("")
67 .required(true)
68 .example("")
69 descriptor = minValue :: descriptor
70
71 val flagValue = new PropertyDescriptor()
72 .name("flagValue")
73 .displayName("flagValue")
74 .description("Identification value")
75 .defaultValue("-9999")
76 .required(true)
77 .example("")
78 descriptor = flagValue :: descriptor
79
80 val additionalCondition = new PropertyDescriptor()
81 .name("additionalCondition")
82 .displayName("additionalCondition")
83 .description("Additional condition")
84 .defaultValue("1=1")
85 .required(true)
86 .example("")
87 descriptor = additionalCondition :: descriptor
88
89
90 descriptor
91 }
92
93 override def getIcon(): Array[Byte] = {
94 ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
95 }
96
97 override def getGroup(): List[String] = {
98 List(StopGroup.CleanGroup)
99 }
100
101 override def initialize(ctx: ProcessContext): Unit = {
102
103 }
104
105}