Source code

Download source archive ↓

src/WildSpotCullingMDV_I.scala

1package cn.piflow.bundle.flux.exceptionValueHandling
2
3
4import cn.piflow.bundle.flux.util.FluxOutlierHandlingUtil_3
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class WildSpotCullingMDV_I extends ConfigurableStop{
12  val authorEmail: String = "ygang@cnic.cn"
13  val description: String = "Using a fixed window, the standard deviation or variance is calculated " +
14    " within a specified size window. * for data exceeding x times " +
15    " variance or standard deviation, the quality control is marked as Ao and eliminated."
16  val inportList: List[String] = List(Port.DefaultPort)
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var fields: String = _
20  var mdv_num: String = _
21  var multiple: String = _
22  var statisticalType: String = _
23  var flagValue: String = _
24
25
26  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27    val spark = pec.get[SparkSession]()
28    val fluxCBS = in.read()
29
30    val flux = new FluxOutlierHandlingUtil_3
31    val df = flux.FluxOutfieldEMDVFixed(spark,fluxCBS,fields,mdv_num.toInt,multiple.toDouble,statisticalType,flagValue)
32
33    out.write(df)
34  }
35
36  override def setProperties(map: Map[String, Any]): Unit = {
37    fields = MapUtil.get(map,"fields").asInstanceOf[String]
38    mdv_num = MapUtil.get(map,"mdv_num").asInstanceOf[String]
39    multiple = MapUtil.get(map,"multiple").asInstanceOf[String]
40    statisticalType = MapUtil.get(map,"statisticalType").asInstanceOf[String]
41    flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
42
43  }
44
45  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
46    var descriptor : List[PropertyDescriptor] = List()
47
48    val fields = new PropertyDescriptor()
49      .name("fields")
50      .displayName("fields")
51      .description("Threshold processing fields, multiple separated by commas")
52      .defaultValue("")
53      .required(true)
54      .example("a,b")
55    descriptor = fields :: descriptor
56
57    val mdv_num = new PropertyDescriptor()
58      .name("mdv_num")
59      .displayName("mdv_num")
60      .description("Window time")
61      .defaultValue("7")
62      .required(true)
63      .example("")
64    descriptor = mdv_num :: descriptor
65
66    val multiple = new PropertyDescriptor()
67      .name("multiple")
68      .displayName("multiple")
69      .description("Multiple of statistical type")
70      .defaultValue("1")
71      .required(true)
72      .example("")
73    descriptor = multiple :: descriptor
74
75    val statisticalType = new PropertyDescriptor()
76      .name("statisticalType")
77      .displayName("statisticalType")
78      .description("Calculation type,Variance or standard deviation (stddev,variance)")
79      .defaultValue("stddev")
80      .required(true)
81      .example("")
82    descriptor = statisticalType :: descriptor
83
84
85    val flagValue = new PropertyDescriptor()
86      .name("flagValue")
87      .displayName("flagValue")
88      .description("Identification value")
89      .defaultValue("")
90      .required(true)
91      .example("")
92    descriptor = flagValue :: descriptor
93
94    descriptor
95  }
96
97  override def getIcon(): Array[Byte] = {
98    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
99  }
100
101  override def getGroup(): List[String] = {
102    List("Flux_exceptionValueHandling")
103  }
104
105  override def initialize(ctx: ProcessContext): Unit = {
106
107  }
108
109}