src/PM_HS.scala
1package cn.piflow.bundle.flux.util
2
3
4import cn.piflow.bundle.flux.util.CommonUtil.toTimeStamp
5import org.apache.spark.sql.functions.{col, when}
6import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
7
8import scala.collection.mutable
9import scala.collection.mutable.ArrayBuffer
10
11/**
12 * @Author renhao
13 * @Description:
14 * @Data 2023/3/24 18:19
15 * @Modified By:
16 */
17class PM_HS extends Serializable {
18
19// val spark = SparkSession.builder()
20// .master("local[*]")
21// .appName("dataCompare")
22// .config("spark.sql.caseSensitive", "true")
23// .enableHiveSupport()
24// .getOrCreate()
25
26
27
28// def main(args: Array[String]): Unit = {
29//
30// val fluxCBS = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/通量数据-wpl.xls")
31// val meteorological = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/插补处理-气象数据.xls")
32//
33// //通量设备安装高度6层
34// val arrayH = List(8.0, 8.0, 4.0, 4.0, 6.0, 1.0, 0.0).reverse
35//
36// //待校正存储项LE字段名
37// val HS = "Hs"
38// //温度对应字段名
39// val arrayHS = List("Ta_1_AVG", "Ta_2_AVG", "Ta_3_AVG", "Ta_4_AVG", "Ta_5_AVG", "Ta_6_AVG", "Ta_7_AVG")
40// //时间间隔,单位为秒
41// val time_nterval = 1800
42//
43// val rho_a_Avg = "rho_a_Avg"
44//
45// val CP: Double = 1004.67
46//
47// calculateStoredItem(fluxCBS,meteorological, arrayH, HS, arrayHS, time_nterval, CP,rho_a_Avg)
48// .orderBy(col("year").cast("double"), $"month".cast("double"), $"day".cast("double"), $"hour".cast("double"))
49// .show()
50// // .take(15).foreach(println)
51// }
52
53 //廓线法(PM)校正存储项FC、LE、HS
54 def calculateStoredItem(spark:SparkSession,fluxCBS: DataFrame, meteorological:DataFrame,arrayH: List[Double], LE: String, arrayHS: List[String]
55 , time_nterval: Integer, LV: Double,rho_a_Avg:String): DataFrame = {
56
57 import spark.implicits._
58 val arrayCols = List(rho_a_Avg).++(List(LE)).++(arrayHS).++(List("timeStamp")).distinct
59
60 val toTimeS = spark.udf.register("toTimeStamp", toTimeStamp _)
61 fluxCBS.createOrReplaceTempView("flux")
62
63 val fluxTimeStamp = fluxCBS.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
64 val meteorological_Stamp = meteorological.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
65
66 val inputJoinDF = fluxTimeStamp.join(meteorological_Stamp,Seq("timeStamp"),"left").select(arrayCols.map(col(_)): _*)
67 val inputJoinDFM = inputJoinDF.withColumn("timeStamp",$"timeStamp".plus(time_nterval * 1000))
68
69 val temp = inputJoinDF.as("a").join(inputJoinDFM.as("b"), Seq("timeStamp"), "left")
70
71 val fluxJoinSemih_temp = temp.where(col(s"b.${arrayHS(0)}").isNotNull)
72 val fluxJoinSemih = fluxJoinSemih_temp.select(arrayCols.map(x => {
73 if (x == LE) {
74 Array(col(s"a.$x").as(s"a_$x"))
75 } else if (x == "timeStamp") {
76 Array(col("timeStamp"))
77 } else if(x==rho_a_Avg){
78 Array(col(s"b.${rho_a_Avg}"))
79 }else{
80 Array(col(s"a.$x").as(s"a_$x")
81 , col(s"b.$x").as(s"b_$x"))
82 }
83 }).flatMap(x => x): _*)
84
85 val fluxJoinSemihArrayKeys: Seq[String] = fluxJoinSemih.schema.map(_.name)
86 val flux_ca = fluxJoinSemih.rdd
87 .map(rowToMap(fluxJoinSemihArrayKeys, _))
88 .map(x => {
89 val le_s_t = calculateDif(x, arrayH, arrayHS)
90 val le_s = x.get(s"a_$LE").get.toString.toDouble + (le_s_t.last / time_nterval * LV*x.get(rho_a_Avg).get.toString.toDouble)
91 (x.get("timeStamp").get.toString, le_s_t(0), le_s_t(1)
92 , le_s_t(2)
93 , le_s_t(3)
94 , le_s_t(4)
95 , le_s_t(5)
96 , le_s_t(6)
97 , le_s_t(7)
98 ,(le_s_t.last / time_nterval * LV*x.get(rho_a_Avg).get.toString.toDouble)
99 , le_s.formatted("%.5f"))
100 }).toDF("timeStamp", "d1"
101 , "d2"
102 , "d3"
103 , "d4"
104 , "d5"
105 , "d6"
106 , "d7"
107 , "d8"
108 , "d9"
109 , s"flux_$LE")
110// flux_ca.orderBy(col("timeStamp")).show()
111 val colsResult: Seq[Column] = fluxCBS.schema.map(_.name).map(x => {
112 if (x == LE) {
113 when(col(s"flux_$x").isNull, col(x))
114 .otherwise(col(s"flux_$x")).as(x)
115 } else {
116 col(x)
117 }
118 })
119 val resultDF = fluxTimeStamp.join(flux_ca, Seq("timeStamp"), "left")
120 .select(colsResult: _*)
121 resultDF
122 }
123
124
125 //两个时刻的数据,计算CO2浓度、潜热通量、显热通量差值进行计算
126 def calculateDif(map: Map[String, Any], arrayH: List[Double], arrayV: List[String]): ArrayBuffer[Double] = {
127 val arr: ArrayBuffer[Double] = new ArrayBuffer[Double]()
128 var result = 0.0
129 for (index <- 0 to arrayH.length - 1) {
130 val fc_interval = arrayH(index) * (
131 map.get(s"a_${arrayV(index)}").get.toString.toDouble
132 - map.get(s"b_${arrayV(index)}").get.toString.toDouble
133 )
134 arr.append(fc_interval)
135 result += fc_interval
136 }
137 arr.append(result)
138 arr
139 }
140
141 //将dataFrame中每一行数据转化成map[String,Any]数据
142 def rowToMap(array: Seq[String], row: Row): Map[String, Any] = {
143 val map = new mutable.HashMap[String, Any]()
144 for (index <- 0 to array.length - 1) {
145 map.+=((array(index), row.get(index)))
146 }
147 map.toMap
148 }
149
150}