src/RainfallDataCulling.scala
1package cn.piflow.bundle.flux.exceptionValueHandling
2
3
4import cn.piflow.bundle.flux.util.FluxOutlierHandlingUtil_3
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class RainfallDataCulling extends ConfigurableStop{
12 val authorEmail: String = "ygang@cnic.cn"
13 val description: String = "Elimination of rainfall data in the same period(The occurrence of precipitation disturbs " +
14 "the normal atmospheric flow in the ecosystem and leads to instrument failure, " +
15 "so it is necessary to eliminate the precipitation observation data at the same time to ensure the data quality.)"
16 val inportList: List[String] = List("fluxCbsPort","metroCbsPort")
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var fields: String = _
20 var rainFallField: String = _
21 var flagValue: String = _
22
23
24 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
25 val spark = pec.get[SparkSession]()
26 val metroCBS = in.read("metroCbsPort")
27 val fluxCBS = in.read("fluxCbsPort")
28
29 val flux = new FluxOutlierHandlingUtil_3
30 val df = flux.RainfallDataCulling(spark,metroCBS,fluxCBS,fields,rainFallField,flagValue.toInt)
31 out.write(df)
32 }
33
34 override def setProperties(map: Map[String, Any]): Unit = {
35 fields = MapUtil.get(map,"fields").asInstanceOf[String]
36 rainFallField = MapUtil.get(map,"rainFallField").asInstanceOf[String]
37 flagValue = MapUtil.get(map,"flagValue").asInstanceOf[String]
38
39 }
40
41 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
42 var descriptor : List[PropertyDescriptor] = List()
43
44 val fields = new PropertyDescriptor()
45 .name("fields")
46 .displayName("fields")
47 .description("Threshold processing fields, multiple separated by commas")
48 .defaultValue("Fc,LE,Hs")
49 .required(true)
50 .example("a,b")
51 descriptor = fields :: descriptor
52
53 val rainFallField = new PropertyDescriptor()
54 .name("rainFallField")
55 .displayName("rainFallField")
56 .description("Meteorological data rainfall")
57 .defaultValue("Rain_0_TOT")
58 .required(true)
59 .example("")
60 descriptor = rainFallField :: descriptor
61
62
63 val flagValue = new PropertyDescriptor()
64 .name("flagValue")
65 .displayName("flagValue")
66 .description("Identification value")
67 .defaultValue("-9999")
68 .required(true)
69 .example("")
70 descriptor = flagValue :: descriptor
71
72 descriptor
73 }
74
75 override def getIcon(): Array[Byte] = {
76 ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
77 }
78
79 override def getGroup(): List[String] = {
80 List("Flux_exceptionValueHandling")
81 }
82
83 override def initialize(ctx: ProcessContext): Unit = {
84
85 }
86
87}