src/PM_LE.scala
1package cn.piflow.bundle.flux.util
2
3
4import cn.piflow.bundle.flux.util.CommonUtil.toTimeStamp
5import org.apache.spark.sql.functions.{col, when}
6import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
7
8import scala.collection.mutable
9import scala.collection.mutable.ArrayBuffer
10
11/**
12 * @Author renhao
13 * @Description:
14 * @Data 2023/3/24 18:19
15 * @Modified By:
16 */
17class PM_LE extends Serializable{
18
19// def main(args: Array[String]): Unit = {
20//
21// val fluxCBS = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/通量数据-wpl.xls")
22// val meteorological = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/插补处理-气象数据.xls")
23//
24// //通量设备安装高度6层
25// val arrayH = List(8.0, 8.0, 4.0, 4.0, 6.0, 1.0, 0.0).reverse
26//
27// //待校正存储项LE字段名
28// val LE = "LE"
29// //H2O浓度对应字段名
30// val arrayLE = List("Pvapor_1_AVG", "Pvapor_2_AVG", "Pvapor_3_AVG", "Pvapor_4_AVG", "Pvapor_5_AVG", "Pvapor_6_AVG", "Pvapor_7_AVG")
31// //温度对应字段名
32// val arrayHS = List("Ta_1_AVG", "Ta_2_AVG", "Ta_3_AVG", "Ta_4_AVG", "Ta_5_AVG", "Ta_6_AVG", "Ta_7_AVG")
33// //时间间隔,单位为秒
34// val time_nterval = 1800
35//
36// val LV: Double = 2440.0
37//
38// calculateStoredItem(fluxCBS,meteorological, arrayH, LE, arrayLE, arrayHS, time_nterval, LV)
39// .orderBy(col("year").cast("double"), $"month".cast("double"), $"day".cast("double"), $"hour".cast("double"))
40// .show()
41// // .take(15).foreach(println)
42// }
43
44 //廓线法(PM)校正存储项FC、LE、HS
45 def calculateStoredItem(spark:SparkSession,fluxCBS: DataFrame, meteorological:DataFrame,arrayH: List[Double], LE: String, arrayLE: List[String], arrayHS: List[String]
46 , time_nterval: Integer, LV: Double): DataFrame = {
47
48 import spark.implicits._
49
50 val arrayCols = List(LE).++(arrayLE).++(arrayHS).++(List("timeStamp")).distinct
51
52 val toTimeS = spark.udf.register("toTimeStamp", toTimeStamp _)
53 fluxCBS.createOrReplaceTempView("flux")
54
55 val fluxTimeStamp = fluxCBS.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
56 val meteorological_Stamp = meteorological.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
57
58 val inputJoinDF = fluxTimeStamp.join(meteorological_Stamp,Seq("timeStamp"),"left").select(arrayCols.map(col(_)): _*)
59 val inputJoinDFM = inputJoinDF.withColumn("timeStamp",$"timeStamp".plus(time_nterval * 1000))
60
61 val temp = inputJoinDF.as("a").join(inputJoinDFM.as("b"), Seq("timeStamp"), "left")
62
63 val fluxJoinSemih_temp = temp.where(col(s"b.${arrayLE(0)}").isNotNull)
64 val fluxJoinSemih = fluxJoinSemih_temp.select(arrayCols.map(x => {
65 if (x == LE) {
66 Array(col(s"a.$x").as(s"a_$x"))
67 } else if (x == "timeStamp") {
68 Array(col("timeStamp"))
69 } else {
70 Array(col(s"a.$x").as(s"a_$x")
71 , col(s"b.$x").as(s"b_$x"))
72 }
73 }).flatMap(x => x): _*)
74
75 val fluxJoinSemihArrayKeys: Seq[String] = fluxJoinSemih.schema.map(_.name)
76 val flux_ca = fluxJoinSemih.rdd
77 .map(rowToMap(fluxJoinSemihArrayKeys, _))
78 .map(x => {
79 val le_s_t = calculateDif(x, arrayH, arrayLE, arrayHS)
80 val le_s = x.get(s"a_$LE").get.toString.toDouble + (le_s_t.last / time_nterval * LV)
81 (x.get("timeStamp").get.toString, le_s_t(0), le_s_t(1)
82 , le_s_t(2)
83 , le_s_t(3)
84 , le_s_t(4)
85 , le_s_t(5)
86 , le_s_t(6)
87 , le_s_t(7)
88 , le_s.formatted("%.5f"))
89 }).toDF("timeStamp", "d1"
90 , "d2"
91 , "d3"
92 , "d4"
93 , "d5"
94 , "d6"
95 , "d7"
96 , "d8"
97 , s"flux_$LE")
98
99 val colsResult: Seq[Column] = fluxCBS.schema.map(_.name).map(x => {
100 if (x == LE) {
101 when(col(s"flux_$x").isNull, col(x))
102 .otherwise(col(s"flux_$x")).as(x)
103 } else {
104 col(x)
105 }
106 })
107 val resultDF = fluxTimeStamp.join(flux_ca, Seq("timeStamp"), "left")
108 .select(colsResult: _*)
109 resultDF
110 }
111
112 //计算水汽密度
113 def calulateV(double_LE: Double, double_HS: Double): Double = {
114 216.5 * double_LE / (273.15 + double_HS)
115 }
116
117 //两个时刻的数据,计算CO2浓度、潜热通量、显热通量差值进行计算
118 def calculateDif(map: Map[String, Any], arrayH: List[Double], arrayV: List[String], arrayHS: List[String]): ArrayBuffer[Double] = {
119 val arr: ArrayBuffer[Double] = new ArrayBuffer[Double]()
120 var result = 0.0
121 for (index <- 0 to arrayH.length - 1) {
122 val fc_interval = arrayH(index) * (
123 calulateV(map.get(s"a_${arrayV(index)}").get.toString.toDouble, map.get(s"a_${arrayHS(index)}").get.toString.toDouble)
124 - calulateV(map.get(s"b_${arrayV(index)}").get.toString.toDouble, map.get(s"b_${arrayHS(index)}").get.toString.toDouble)
125 )
126 arr.append(fc_interval)
127 result += fc_interval
128 }
129 arr.append(result)
130 arr
131 }
132
133 //将dataFrame中每一行数据转化成map[String,Any]数据
134 def rowToMap(array: Seq[String], row: Row): Map[String, Any] = {
135 val map = new mutable.HashMap[String, Any]()
136 for (index <- 0 to array.length - 1) {
137 map.+=((array(index), row.get(index)))
138 }
139 map.toMap
140 }
141
142}