Source code

Download source archive ↓

src/Reichstein.scala

1package cn.piflow.bundle.flux.util
2
3
4import org.apache.spark.mllib.stat.Statistics
5import org.apache.spark.rdd.RDD
6import org.apache.spark.sql.functions.{col, max, min}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8
9import scala.collection.mutable.ArrayBuffer
10
11/**
12 * 异常值处理:u*校正,reichstein法
13 *
14 * @Author renhao
15 * @Description:
16 * @Data 2023/2/23 16:15
17 * @Modified By:
18 */
19class Reichstein extends Serializable{
20
21  //  val spark = SparkSession.builder()
22  //    .master("local[*]")
23  //    .appName("dataCompare")
24  //    .config("spark.sql.caseSensitive", "true")
25  //    .enableHiveSupport()
26  //    .getOrCreate()
27  //
28  //  val metroCBS = MetroExcel(spark)
29  //  val fluxCBS = CBSExcel(spark)
30  //  val metroJoinFlux = metroCBS.join(fluxCBS, Seq("year", "month", "day", "hour"), "left")
31
32//  def main(args: Array[String]): Unit = {
33//    //    val metroJoinFlux = metroCBS.join(fluxCBS, Seq("year", "month", "day", "hour"), "left")
34//    //    //5cm土壤温度m5
35//    //    val soilTemperatureField = "Ts_107_1_AVG"
36//    //    //摩擦风速字段,即U*
37//    //    val FrictionWindSpeedField = "p5"
38//    //    //FC值 CO2通量 NEE
39//    //    val FC = "p1"
40//    //    val flagValue = -99999
41//    //    //设置通量百分比
42//    //    val percentFlux = 0.99
43//    //    reichsteinCorrectU(metroJoinFlux,soilTemperatureField,FrictionWindSpeedField,FC,flagValue,percentFlux).show()
44//  }
45
46
47  //综合6个温度组的U*Cg值计算这个数据集的u*c值并校正U*
48  def reichsteinCorrectU(spark: SparkSession, metroJoinFlux: DataFrame, soilTemperatureField: String, FrictionWindSpeedField: String
49                         , FC: String, flagValue: Integer, percentFlux: Double): DataFrame = {
50    metroJoinFlux.createOrReplaceTempView("temp")
51    spark.sql(
52      s"""
53         |select year,month,hour,day_status
54         |,row_number() over(order by ${soilTemperatureField}) as orderNumber
55         |,${soilTemperatureField},${FrictionWindSpeedField},${FC} from temp
56         |""".stripMargin).createOrReplaceTempView("metroJoinFluxOrderByTem")
57    val total = metroJoinFlux.count()
58    val groupNumber = metroJoinFlux.count() / 6
59    val arrUCG = new ArrayBuffer[Double]()
60    for (index <- 1 to 6) {
61      var metroJoinFluxGroup: DataFrame = null
62      if (index < 6) {
63        metroJoinFluxGroup = spark.sql(
64          s"""
65             |select * from metroJoinFluxOrderByTem where orderNumber > ${index - 1}*${groupNumber}
66             |and orderNumber <= ${index}*${groupNumber}
67             |""".stripMargin)
68      } else {
69        metroJoinFluxGroup = spark.sql(
70          s"""
71             |select * from metroJoinFluxOrderByTem where orderNumber > 5*${groupNumber}
72             |and orderNumber <= ${total}
73             |""".stripMargin)
74      }
75      if (pearson(metroJoinFluxGroup, soilTemperatureField, FrictionWindSpeedField) < 0.4) {
76        arrUCG.append(calculateCriticalValue(spark,metroJoinFluxGroup, FrictionWindSpeedField, FC, flagValue, percentFlux))
77      }
78    }
79    if (arrUCG.filterNot(_ == flagValue).length < 1) {
80      throw new Exception("not found U*C")
81    }
82    //    综合6个温度组的U*Cg值计算这个数据集的u*c值
83    val Uc = median(arrUCG)
84    val cols = metroJoinFlux.schema.map(_.name).map(x => {
85      if (x == FC) {
86        s"case when ${FrictionWindSpeedField}<$Uc and day_status='night' then $flagValue else ${FC} end as $FC"
87      } else {
88        x
89      }
90    }).mkString(",")
91    spark.sql(
92      s"""
93         |select $cols from temp
94         |""".stripMargin)
95  }
96
97  //求集合中位数
98  def median(arr: ArrayBuffer[Double]): Double = {
99    val arrOrder = arr.sorted
100    if (arr.length % 2 == 1) {
101      arrOrder(arrOrder.length / 2)
102    } else {
103      (arrOrder(arrOrder.length / 2) + arrOrder(arrOrder.length / 2 - 1)) / 2
104    }
105  }
106
107  //计算每个温度组的临界值
108  def calculateCriticalValue(spark: SparkSession,metroJoinFluxGroup: DataFrame, FrictionWindSpeedField: String
109                             , FC: String, flagValue: Integer, percentFlux: Double): Double = {
110    metroJoinFluxGroup.createOrReplaceTempView("metroJoinFluxGroup")
111    val u_max = metroJoinFluxGroup.select(col(FrictionWindSpeedField).cast("Double").as(FrictionWindSpeedField))
112      .select(max(FrictionWindSpeedField))
113      .rdd.map(_.get(0).asInstanceOf[Double]).collect()(0)
114    val u_min = metroJoinFluxGroup.select(col(FrictionWindSpeedField).cast("Double").as((FrictionWindSpeedField)))
115      .select(min(FrictionWindSpeedField))
116      .rdd.map(_.get(0).asInstanceOf[Double]).collect()(0)
117    val u_D_value = (u_max - u_min) / 20
118    for (index <- 1 to 20) {
119      val metroJoinFluxGroupSub = spark.sql(
120        s"""
121           |select $FC,$FrictionWindSpeedField from metroJoinFluxGroup where $FrictionWindSpeedField >= ($index-1)*$u_D_value+$u_min
122
123          |and $FrictionWindSpeedField < $index * $u_D_value+$u_max
124
125          |""".stripMargin)
126      //从第index+1个u*小组开始计算该温度组的FC均值
127      val FC_AVG_index: Double = spark.sql(
128        s"""
129           |select avg(cast($FC as double))
130           |from metroJoinFluxGroup
131           |where $FrictionWindSpeedField >= $index*$u_D_value+$u_min
132           |""".stripMargin).rdd.map(_.get(0).asInstanceOf[Double]).collect()(0)
133      metroJoinFluxGroupSub.select(col(FC).cast("Double"),col(FrictionWindSpeedField) ).rdd
134        .map(x=>(x.get(0) .asInstanceOf[Double],x.get(1).toString.toDouble)) .collect().foreach(x=>{
135        if(x._1 > FC_AVG_index *percentFlux){
136          return x._2
137        }
138      })
139    }
140    flagValue.toDouble
141  }
142
143
144  //计算dataframe中某两列的皮尔逊相关系数
145  def pearson( df:DataFrame ,col1 :String,col2:String): Double ={
146    val seriesX: RDD[Double] = df.select(col1).rdd.map(_.get(0).toString.toDouble)
147    val seriesY: RDD[Double] = df.select(col2).rdd.map(_.get(0).toString.toDouble)
148    val correlation: Double = Statistics.corr(seriesX, seriesY, "pearson")
149    correlation
150  }
151
152}