src/Reichstein.scala
1package cn.piflow.bundle.flux.util
2
3
4import org.apache.spark.mllib.stat.Statistics
5import org.apache.spark.rdd.RDD
6import org.apache.spark.sql.functions.{col, max, min}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8
9import scala.collection.mutable.ArrayBuffer
10
11/**
12 * 异常值处理:u*校正,reichstein法
13 *
14 * @Author renhao
15 * @Description:
16 * @Data 2023/2/23 16:15
17 * @Modified By:
18 */
19class Reichstein extends Serializable{
20
21 // val spark = SparkSession.builder()
22 // .master("local[*]")
23 // .appName("dataCompare")
24 // .config("spark.sql.caseSensitive", "true")
25 // .enableHiveSupport()
26 // .getOrCreate()
27 //
28 // val metroCBS = MetroExcel(spark)
29 // val fluxCBS = CBSExcel(spark)
30 // val metroJoinFlux = metroCBS.join(fluxCBS, Seq("year", "month", "day", "hour"), "left")
31
32// def main(args: Array[String]): Unit = {
33// // val metroJoinFlux = metroCBS.join(fluxCBS, Seq("year", "month", "day", "hour"), "left")
34// // //5cm土壤温度m5
35// // val soilTemperatureField = "Ts_107_1_AVG"
36// // //摩擦风速字段,即U*
37// // val FrictionWindSpeedField = "p5"
38// // //FC值 CO2通量 NEE
39// // val FC = "p1"
40// // val flagValue = -99999
41// // //设置通量百分比
42// // val percentFlux = 0.99
43// // reichsteinCorrectU(metroJoinFlux,soilTemperatureField,FrictionWindSpeedField,FC,flagValue,percentFlux).show()
44// }
45
46
47 //综合6个温度组的U*Cg值计算这个数据集的u*c值并校正U*
48 def reichsteinCorrectU(spark: SparkSession, metroJoinFlux: DataFrame, soilTemperatureField: String, FrictionWindSpeedField: String
49 , FC: String, flagValue: Integer, percentFlux: Double): DataFrame = {
50 metroJoinFlux.createOrReplaceTempView("temp")
51 spark.sql(
52 s"""
53 |select year,month,hour,day_status
54 |,row_number() over(order by ${soilTemperatureField}) as orderNumber
55 |,${soilTemperatureField},${FrictionWindSpeedField},${FC} from temp
56 |""".stripMargin).createOrReplaceTempView("metroJoinFluxOrderByTem")
57 val total = metroJoinFlux.count()
58 val groupNumber = metroJoinFlux.count() / 6
59 val arrUCG = new ArrayBuffer[Double]()
60 for (index <- 1 to 6) {
61 var metroJoinFluxGroup: DataFrame = null
62 if (index < 6) {
63 metroJoinFluxGroup = spark.sql(
64 s"""
65 |select * from metroJoinFluxOrderByTem where orderNumber > ${index - 1}*${groupNumber}
66 |and orderNumber <= ${index}*${groupNumber}
67 |""".stripMargin)
68 } else {
69 metroJoinFluxGroup = spark.sql(
70 s"""
71 |select * from metroJoinFluxOrderByTem where orderNumber > 5*${groupNumber}
72 |and orderNumber <= ${total}
73 |""".stripMargin)
74 }
75 if (pearson(metroJoinFluxGroup, soilTemperatureField, FrictionWindSpeedField) < 0.4) {
76 arrUCG.append(calculateCriticalValue(spark,metroJoinFluxGroup, FrictionWindSpeedField, FC, flagValue, percentFlux))
77 }
78 }
79 if (arrUCG.filterNot(_ == flagValue).length < 1) {
80 throw new Exception("not found U*C")
81 }
82 // 综合6个温度组的U*Cg值计算这个数据集的u*c值
83 val Uc = median(arrUCG)
84 val cols = metroJoinFlux.schema.map(_.name).map(x => {
85 if (x == FC) {
86 s"case when ${FrictionWindSpeedField}<$Uc and day_status='night' then $flagValue else ${FC} end as $FC"
87 } else {
88 x
89 }
90 }).mkString(",")
91 spark.sql(
92 s"""
93 |select $cols from temp
94 |""".stripMargin)
95 }
96
97 //求集合中位数
98 def median(arr: ArrayBuffer[Double]): Double = {
99 val arrOrder = arr.sorted
100 if (arr.length % 2 == 1) {
101 arrOrder(arrOrder.length / 2)
102 } else {
103 (arrOrder(arrOrder.length / 2) + arrOrder(arrOrder.length / 2 - 1)) / 2
104 }
105 }
106
107 //计算每个温度组的临界值
108 def calculateCriticalValue(spark: SparkSession,metroJoinFluxGroup: DataFrame, FrictionWindSpeedField: String
109 , FC: String, flagValue: Integer, percentFlux: Double): Double = {
110 metroJoinFluxGroup.createOrReplaceTempView("metroJoinFluxGroup")
111 val u_max = metroJoinFluxGroup.select(col(FrictionWindSpeedField).cast("Double").as(FrictionWindSpeedField))
112 .select(max(FrictionWindSpeedField))
113 .rdd.map(_.get(0).asInstanceOf[Double]).collect()(0)
114 val u_min = metroJoinFluxGroup.select(col(FrictionWindSpeedField).cast("Double").as((FrictionWindSpeedField)))
115 .select(min(FrictionWindSpeedField))
116 .rdd.map(_.get(0).asInstanceOf[Double]).collect()(0)
117 val u_D_value = (u_max - u_min) / 20
118 for (index <- 1 to 20) {
119 val metroJoinFluxGroupSub = spark.sql(
120 s"""
121 |select $FC,$FrictionWindSpeedField from metroJoinFluxGroup where $FrictionWindSpeedField >= ($index-1)*$u_D_value+$u_min
122
123 |and $FrictionWindSpeedField < $index * $u_D_value+$u_max
124
125 |""".stripMargin)
126 //从第index+1个u*小组开始计算该温度组的FC均值
127 val FC_AVG_index: Double = spark.sql(
128 s"""
129 |select avg(cast($FC as double))
130 |from metroJoinFluxGroup
131 |where $FrictionWindSpeedField >= $index*$u_D_value+$u_min
132 |""".stripMargin).rdd.map(_.get(0).asInstanceOf[Double]).collect()(0)
133 metroJoinFluxGroupSub.select(col(FC).cast("Double"),col(FrictionWindSpeedField) ).rdd
134 .map(x=>(x.get(0) .asInstanceOf[Double],x.get(1).toString.toDouble)) .collect().foreach(x=>{
135 if(x._1 > FC_AVG_index *percentFlux){
136 return x._2
137 }
138 })
139 }
140 flagValue.toDouble
141 }
142
143
144 //计算dataframe中某两列的皮尔逊相关系数
145 def pearson( df:DataFrame ,col1 :String,col2:String): Double ={
146 val seriesX: RDD[Double] = df.select(col1).rdd.map(_.get(0).toString.toDouble)
147 val seriesY: RDD[Double] = df.select(col2).rdd.map(_.get(0).toString.toDouble)
148 val correlation: Double = Statistics.corr(seriesX, seriesY, "pearson")
149 correlation
150 }
151
152}