Source code

Download source archive ↓

src/WildSpotCullingMDV_G.scala

1package cn.piflow.bundle.flux.exceptionValueHandling
2
3
4import cn.piflow.bundle.flux.util.FluxOutlierHandlingUtil_3
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class WildSpotCullingMDV_G extends ConfigurableStop{
12  val authorEmail: String = "ygang@cnic.cn"
13  val description: String = "Use sliding window mode, the standard deviation or variance is calculated " +
14    " within a specified size window. * for data exceeding x times " +
15    " variance or standard deviation, the quality control is marked as Ao and eliminated."
16  val inportList: List[String] = List(Port.DefaultPort)
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var fields: String = _
20  var mdv_num: String = _
21  var multiple: String = _
22  var statisticalType: String = _
23  var flagValue: String = _
24
25
26  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27    val spark = pec.get[SparkSession]()
28    var df = in.read()
29
30    val flux = new FluxOutlierHandlingUtil_3
31    fields.split(",").foreach(x=>{
32      df = flux.FluxOutfieldEliminationMDVSliding(spark,df,x,mdv_num.toInt,multiple.toDouble,statisticalType,flagValue)
33    })
34
35    out.write(df)
36  }
37
38  override def setProperties(map: Map[String, Any]): Unit = {
39    fields = MapUtil.get(map,"fields").asInstanceOf[String]
40    mdv_num = MapUtil.get(map,"mdv_num").asInstanceOf[String]
41    multiple = MapUtil.get(map,"multiple").asInstanceOf[String]
42    statisticalType = MapUtil.get(map,"statisticalType").asInstanceOf[String]
43    flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
44
45  }
46
47  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
48    var descriptor : List[PropertyDescriptor] = List()
49
50    val fields = new PropertyDescriptor()
51      .name("fields")
52      .displayName("fields")
53      .description("Threshold processing fields, multiple separated by commas")
54      .defaultValue("")
55      .required(true)
56      .example("a,b")
57    descriptor = fields :: descriptor
58
59    val mdv_num = new PropertyDescriptor()
60      .name("mdv_num")
61      .displayName("mdv_num")
62      .description("Window time")
63      .defaultValue("7")
64      .required(true)
65      .example("")
66    descriptor = mdv_num :: descriptor
67
68    val multiple = new PropertyDescriptor()
69      .name("multiple")
70      .displayName("multiple")
71      .description("Multiple of statistical type")
72      .defaultValue("1")
73      .required(true)
74      .example("")
75    descriptor = multiple :: descriptor
76
77    val statisticalType = new PropertyDescriptor()
78      .name("statisticalType")
79      .displayName("statisticalType")
80      .description("Calculation type,Variance or standard deviation (stddev,variance)")
81      .defaultValue("stddev")
82      .required(true)
83      .example("")
84    descriptor = statisticalType :: descriptor
85
86    val flagValue = new PropertyDescriptor()
87      .name("flagValue")
88      .displayName("flagValue")
89      .description("Identification value")
90      .defaultValue("")
91      .required(true)
92      .example("")
93    descriptor = flagValue :: descriptor
94
95    descriptor
96  }
97
98  override def getIcon(): Array[Byte] = {
99    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
100  }
101
102  override def getGroup(): List[String] = {
103    List("Flux_exceptionValueHandling")
104  }
105
106  override def initialize(ctx: ProcessContext): Unit = {
107
108  }
109
110}