src/WildSpotCullingMDV_G.scala
1package cn.piflow.bundle.flux.exceptionValueHandling
2
3
4import cn.piflow.bundle.flux.util.FluxOutlierHandlingUtil_3
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class WildSpotCullingMDV_G extends ConfigurableStop{
12 val authorEmail: String = "ygang@cnic.cn"
13 val description: String = "Use sliding window mode, the standard deviation or variance is calculated " +
14 " within a specified size window. * for data exceeding x times " +
15 " variance or standard deviation, the quality control is marked as Ao and eliminated."
16 val inportList: List[String] = List(Port.DefaultPort)
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var fields: String = _
20 var mdv_num: String = _
21 var multiple: String = _
22 var statisticalType: String = _
23 var flagValue: String = _
24
25
26 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27 val spark = pec.get[SparkSession]()
28 var df = in.read()
29
30 val flux = new FluxOutlierHandlingUtil_3
31 fields.split(",").foreach(x=>{
32 df = flux.FluxOutfieldEliminationMDVSliding(spark,df,x,mdv_num.toInt,multiple.toDouble,statisticalType,flagValue)
33 })
34
35 out.write(df)
36 }
37
38 override def setProperties(map: Map[String, Any]): Unit = {
39 fields = MapUtil.get(map,"fields").asInstanceOf[String]
40 mdv_num = MapUtil.get(map,"mdv_num").asInstanceOf[String]
41 multiple = MapUtil.get(map,"multiple").asInstanceOf[String]
42 statisticalType = MapUtil.get(map,"statisticalType").asInstanceOf[String]
43 flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
44
45 }
46
47 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
48 var descriptor : List[PropertyDescriptor] = List()
49
50 val fields = new PropertyDescriptor()
51 .name("fields")
52 .displayName("fields")
53 .description("Threshold processing fields, multiple separated by commas")
54 .defaultValue("")
55 .required(true)
56 .example("a,b")
57 descriptor = fields :: descriptor
58
59 val mdv_num = new PropertyDescriptor()
60 .name("mdv_num")
61 .displayName("mdv_num")
62 .description("Window time")
63 .defaultValue("7")
64 .required(true)
65 .example("")
66 descriptor = mdv_num :: descriptor
67
68 val multiple = new PropertyDescriptor()
69 .name("multiple")
70 .displayName("multiple")
71 .description("Multiple of statistical type")
72 .defaultValue("1")
73 .required(true)
74 .example("")
75 descriptor = multiple :: descriptor
76
77 val statisticalType = new PropertyDescriptor()
78 .name("statisticalType")
79 .displayName("statisticalType")
80 .description("Calculation type,Variance or standard deviation (stddev,variance)")
81 .defaultValue("stddev")
82 .required(true)
83 .example("")
84 descriptor = statisticalType :: descriptor
85
86 val flagValue = new PropertyDescriptor()
87 .name("flagValue")
88 .displayName("flagValue")
89 .description("Identification value")
90 .defaultValue("")
91 .required(true)
92 .example("")
93 descriptor = flagValue :: descriptor
94
95 descriptor
96 }
97
98 override def getIcon(): Array[Byte] = {
99 ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
100 }
101
102 override def getGroup(): List[String] = {
103 List("Flux_exceptionValueHandling")
104 }
105
106 override def initialize(ctx: ProcessContext): Unit = {
107
108 }
109
110}