src/WildSpotCullingMDV_I.scala
1package cn.piflow.bundle.flux.exceptionValueHandling
2
3
4import cn.piflow.bundle.flux.util.FluxOutlierHandlingUtil_3
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class WildSpotCullingMDV_I extends ConfigurableStop{
12 val authorEmail: String = "ygang@cnic.cn"
13 val description: String = "Using a fixed window, the standard deviation or variance is calculated " +
14 " within a specified size window. * for data exceeding x times " +
15 " variance or standard deviation, the quality control is marked as Ao and eliminated."
16 val inportList: List[String] = List(Port.DefaultPort)
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var fields: String = _
20 var mdv_num: String = _
21 var multiple: String = _
22 var statisticalType: String = _
23 var flagValue: String = _
24
25
26 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27 val spark = pec.get[SparkSession]()
28 val fluxCBS = in.read()
29
30 val flux = new FluxOutlierHandlingUtil_3
31 val df = flux.FluxOutfieldEMDVFixed(spark,fluxCBS,fields,mdv_num.toInt,multiple.toDouble,statisticalType,flagValue)
32
33 out.write(df)
34 }
35
36 override def setProperties(map: Map[String, Any]): Unit = {
37 fields = MapUtil.get(map,"fields").asInstanceOf[String]
38 mdv_num = MapUtil.get(map,"mdv_num").asInstanceOf[String]
39 multiple = MapUtil.get(map,"multiple").asInstanceOf[String]
40 statisticalType = MapUtil.get(map,"statisticalType").asInstanceOf[String]
41 flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
42
43 }
44
45 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
46 var descriptor : List[PropertyDescriptor] = List()
47
48 val fields = new PropertyDescriptor()
49 .name("fields")
50 .displayName("fields")
51 .description("Threshold processing fields, multiple separated by commas")
52 .defaultValue("")
53 .required(true)
54 .example("a,b")
55 descriptor = fields :: descriptor
56
57 val mdv_num = new PropertyDescriptor()
58 .name("mdv_num")
59 .displayName("mdv_num")
60 .description("Window time")
61 .defaultValue("7")
62 .required(true)
63 .example("")
64 descriptor = mdv_num :: descriptor
65
66 val multiple = new PropertyDescriptor()
67 .name("multiple")
68 .displayName("multiple")
69 .description("Multiple of statistical type")
70 .defaultValue("1")
71 .required(true)
72 .example("")
73 descriptor = multiple :: descriptor
74
75 val statisticalType = new PropertyDescriptor()
76 .name("statisticalType")
77 .displayName("statisticalType")
78 .description("Calculation type,Variance or standard deviation (stddev,variance)")
79 .defaultValue("stddev")
80 .required(true)
81 .example("")
82 descriptor = statisticalType :: descriptor
83
84
85 val flagValue = new PropertyDescriptor()
86 .name("flagValue")
87 .displayName("flagValue")
88 .description("Identification value")
89 .defaultValue("")
90 .required(true)
91 .example("")
92 descriptor = flagValue :: descriptor
93
94 descriptor
95 }
96
97 override def getIcon(): Array[Byte] = {
98 ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
99 }
100
101 override def getGroup(): List[String] = {
102 List("Flux_exceptionValueHandling")
103 }
104
105 override def initialize(ctx: ProcessContext): Unit = {
106
107 }
108
109}