Source code

Download source archive ↓

src/RainfallDataCulling.scala

1package cn.piflow.bundle.flux.exceptionValueHandling
2
3
4import cn.piflow.bundle.flux.util.FluxOutlierHandlingUtil_3
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class RainfallDataCulling extends ConfigurableStop{
12  val authorEmail: String = "ygang@cnic.cn"
13  val description: String = "Elimination of rainfall data in the same period(The occurrence of precipitation disturbs " +
14    "the normal atmospheric flow in the ecosystem and leads to instrument failure, " +
15    "so it is necessary to eliminate the precipitation observation data at the same time to ensure the data quality.)"
16  val inportList: List[String] = List("fluxCbsPort","metroCbsPort")
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var fields: String = _
20  var rainFallField: String = _
21  var flagValue: String = _
22
23
24  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
25    val spark = pec.get[SparkSession]()
26    val metroCBS = in.read("metroCbsPort")
27    val fluxCBS = in.read("fluxCbsPort")
28
29    val flux = new FluxOutlierHandlingUtil_3
30    val df = flux.RainfallDataCulling(spark,metroCBS,fluxCBS,fields,rainFallField,flagValue.toInt)
31    out.write(df)
32  }
33
34  override def setProperties(map: Map[String, Any]): Unit = {
35    fields = MapUtil.get(map,"fields").asInstanceOf[String]
36    rainFallField = MapUtil.get(map,"rainFallField").asInstanceOf[String]
37    flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
38
39  }
40
41  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
42    var descriptor : List[PropertyDescriptor] = List()
43
44    val fields = new PropertyDescriptor()
45      .name("fields")
46      .displayName("fields")
47      .description("Threshold processing fields, multiple separated by commas")
48      .defaultValue("Fc,LE,Hs")
49      .required(true)
50      .example("a,b")
51    descriptor = fields :: descriptor
52
53    val rainFallField = new PropertyDescriptor()
54      .name("rainFallField")
55      .displayName("rainFallField")
56      .description("Meteorological data rainfall")
57      .defaultValue("Rain_0_TOT")
58      .required(true)
59      .example("")
60    descriptor = rainFallField :: descriptor
61
62
63    val flagValue = new PropertyDescriptor()
64      .name("flagValue")
65      .displayName("flagValue")
66      .description("Identification value")
67      .defaultValue("-9999")
68      .required(true)
69      .example("")
70    descriptor = flagValue :: descriptor
71
72    descriptor
73  }
74
75  override def getIcon(): Array[Byte] = {
76    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
77  }
78
79  override def getGroup(): List[String] = {
80    List("Flux_exceptionValueHandling")
81  }
82
83  override def initialize(ctx: ProcessContext): Unit = {
84
85  }
86
87}