Source code

Download source archive ↓

src/PM_LE.scala

1package cn.piflow.bundle.flux.util
2
3
4import cn.piflow.bundle.flux.util.CommonUtil.toTimeStamp
5import org.apache.spark.sql.functions.{col, when}
6import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
7
8import scala.collection.mutable
9import scala.collection.mutable.ArrayBuffer
10
11/**
12 * @Author renhao
13 * @Description:
14 * @Data 2023/3/24 18:19
15 * @Modified By:
16 */
17class PM_LE extends Serializable{
18
19//  def main(args: Array[String]): Unit = {
20//
21//    val fluxCBS = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/通量数据-wpl.xls")
22//    val meteorological = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/插补处理-气象数据.xls")
23//
24//    //通量设备安装高度6层
25//    val arrayH = List(8.0, 8.0, 4.0, 4.0, 6.0, 1.0, 0.0).reverse
26//
27//    //待校正存储项LE字段名
28//    val LE = "LE"
29//    //H2O浓度对应字段名
30//    val arrayLE = List("Pvapor_1_AVG", "Pvapor_2_AVG", "Pvapor_3_AVG", "Pvapor_4_AVG", "Pvapor_5_AVG", "Pvapor_6_AVG", "Pvapor_7_AVG")
31//    //温度对应字段名
32//    val arrayHS = List("Ta_1_AVG", "Ta_2_AVG", "Ta_3_AVG", "Ta_4_AVG", "Ta_5_AVG", "Ta_6_AVG", "Ta_7_AVG")
33//    //时间间隔,单位为秒
34//    val time_nterval = 1800
35//
36//    val LV: Double = 2440.0
37//
38//    calculateStoredItem(fluxCBS,meteorological, arrayH, LE, arrayLE, arrayHS, time_nterval, LV)
39//      .orderBy(col("year").cast("double"), $"month".cast("double"), $"day".cast("double"), $"hour".cast("double"))
40//      .show()
41//    //      .take(15).foreach(println)
42//  }
43
44  //廓线法(PM)校正存储项FC、LE、HS
45  def calculateStoredItem(spark:SparkSession,fluxCBS: DataFrame, meteorological:DataFrame,arrayH: List[Double], LE: String, arrayLE: List[String], arrayHS: List[String]
46                          , time_nterval: Integer, LV: Double): DataFrame = {
47
48    import spark.implicits._
49
50    val arrayCols = List(LE).++(arrayLE).++(arrayHS).++(List("timeStamp")).distinct
51
52    val toTimeS = spark.udf.register("toTimeStamp", toTimeStamp _)
53    fluxCBS.createOrReplaceTempView("flux")
54
55    val fluxTimeStamp = fluxCBS.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
56    val meteorological_Stamp = meteorological.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
57
58    val inputJoinDF = fluxTimeStamp.join(meteorological_Stamp,Seq("timeStamp"),"left").select(arrayCols.map(col(_)): _*)
59    val inputJoinDFM = inputJoinDF.withColumn("timeStamp",$"timeStamp".plus(time_nterval * 1000))
60
61    val temp = inputJoinDF.as("a").join(inputJoinDFM.as("b"), Seq("timeStamp"), "left")
62
63    val fluxJoinSemih_temp = temp.where(col(s"b.${arrayLE(0)}").isNotNull)
64    val fluxJoinSemih = fluxJoinSemih_temp.select(arrayCols.map(x => {
65      if (x == LE) {
66        Array(col(s"a.$x").as(s"a_$x"))
67      } else if (x == "timeStamp") {
68        Array(col("timeStamp"))
69      } else {
70        Array(col(s"a.$x").as(s"a_$x")
71          , col(s"b.$x").as(s"b_$x"))
72      }
73    }).flatMap(x => x): _*)
74
75    val fluxJoinSemihArrayKeys: Seq[String] = fluxJoinSemih.schema.map(_.name)
76    val flux_ca = fluxJoinSemih.rdd
77      .map(rowToMap(fluxJoinSemihArrayKeys, _))
78      .map(x => {
79        val le_s_t = calculateDif(x, arrayH, arrayLE, arrayHS)
80        val le_s = x.get(s"a_$LE").get.toString.toDouble + (le_s_t.last / time_nterval * LV)
81        (x.get("timeStamp").get.toString, le_s_t(0), le_s_t(1)
82          , le_s_t(2)
83          , le_s_t(3)
84          , le_s_t(4)
85          , le_s_t(5)
86          , le_s_t(6)
87          , le_s_t(7)
88          , le_s.formatted("%.5f"))
89      }).toDF("timeStamp", "d1"
90      , "d2"
91      , "d3"
92      , "d4"
93      , "d5"
94      , "d6"
95      , "d7"
96      , "d8"
97      , s"flux_$LE")
98
99    val colsResult: Seq[Column] = fluxCBS.schema.map(_.name).map(x => {
100      if (x == LE) {
101        when(col(s"flux_$x").isNull, col(x))
102          .otherwise(col(s"flux_$x")).as(x)
103      } else {
104        col(x)
105      }
106    })
107    val resultDF = fluxTimeStamp.join(flux_ca, Seq("timeStamp"), "left")
108          .select(colsResult: _*)
109    resultDF
110  }
111
112  //计算水汽密度
113  def calulateV(double_LE: Double, double_HS: Double): Double = {
114    216.5 * double_LE / (273.15 + double_HS)
115  }
116
117  //两个时刻的数据,计算CO2浓度、潜热通量、显热通量差值进行计算
118  def calculateDif(map: Map[String, Any], arrayH: List[Double], arrayV: List[String], arrayHS: List[String]): ArrayBuffer[Double] = {
119    val arr: ArrayBuffer[Double] = new ArrayBuffer[Double]()
120    var result = 0.0
121    for (index <- 0 to arrayH.length - 1) {
122      val fc_interval = arrayH(index) * (
123        calulateV(map.get(s"a_${arrayV(index)}").get.toString.toDouble, map.get(s"a_${arrayHS(index)}").get.toString.toDouble)
124          - calulateV(map.get(s"b_${arrayV(index)}").get.toString.toDouble, map.get(s"b_${arrayHS(index)}").get.toString.toDouble)
125        )
126      arr.append(fc_interval)
127      result += fc_interval
128    }
129    arr.append(result)
130    arr
131  }
132
133  //将dataFrame中每一行数据转化成map[String,Any]数据
134  def rowToMap(array: Seq[String], row: Row): Map[String, Any] = {
135    val map = new mutable.HashMap[String, Any]()
136    for (index <- 0 to array.length - 1) {
137      map.+=((array(index), row.get(index)))
138    }
139    map.toMap
140  }
141
142}