Source code

Download source archive ↓

src/PM_HS.scala

1package cn.piflow.bundle.flux.util
2
3
4import cn.piflow.bundle.flux.util.CommonUtil.toTimeStamp
5import org.apache.spark.sql.functions.{col, when}
6import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
7
8import scala.collection.mutable
9import scala.collection.mutable.ArrayBuffer
10
11/**
12 * @Author renhao
13 * @Description:
14 * @Data 2023/3/24 18:19
15 * @Modified By:
16 */
17class PM_HS extends Serializable {
18
19//  val spark = SparkSession.builder()
20//    .master("local[*]")
21//    .appName("dataCompare")
22//    .config("spark.sql.caseSensitive", "true")
23//    .enableHiveSupport()
24//    .getOrCreate()
25
26
27
28//  def main(args: Array[String]): Unit = {
29//
30//    val fluxCBS = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/通量数据-wpl.xls")
31//    val meteorological = readExcel(spark, "/Users/renhao/Desktop/关键技术小组/piflow算法处理组件/PM/插补处理-气象数据.xls")
32//
33//    //通量设备安装高度6层
34//    val arrayH = List(8.0, 8.0, 4.0, 4.0, 6.0, 1.0, 0.0).reverse
35//
36//    //待校正存储项LE字段名
37//    val HS = "Hs"
38//    //温度对应字段名
39//    val arrayHS = List("Ta_1_AVG", "Ta_2_AVG", "Ta_3_AVG", "Ta_4_AVG", "Ta_5_AVG", "Ta_6_AVG", "Ta_7_AVG")
40//    //时间间隔,单位为秒
41//    val time_nterval = 1800
42//
43//    val rho_a_Avg = "rho_a_Avg"
44//
45//    val CP: Double = 1004.67
46//
47//    calculateStoredItem(fluxCBS,meteorological, arrayH, HS, arrayHS, time_nterval, CP,rho_a_Avg)
48//      .orderBy(col("year").cast("double"), $"month".cast("double"), $"day".cast("double"), $"hour".cast("double"))
49//      .show()
50//    //      .take(15).foreach(println)
51//  }
52
53  //廓线法(PM)校正存储项FC、LE、HS
54  def calculateStoredItem(spark:SparkSession,fluxCBS: DataFrame, meteorological:DataFrame,arrayH: List[Double], LE: String, arrayHS: List[String]
55                          , time_nterval: Integer, LV: Double,rho_a_Avg:String): DataFrame = {
56
57    import spark.implicits._
58    val arrayCols = List(rho_a_Avg).++(List(LE)).++(arrayHS).++(List("timeStamp")).distinct
59
60    val toTimeS = spark.udf.register("toTimeStamp", toTimeStamp _)
61    fluxCBS.createOrReplaceTempView("flux")
62
63    val fluxTimeStamp = fluxCBS.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
64    val meteorological_Stamp = meteorological.withColumn("timeStamp", toTimeS(col("year"),col("month"),col("day"),col("hour")))
65
66    val inputJoinDF = fluxTimeStamp.join(meteorological_Stamp,Seq("timeStamp"),"left").select(arrayCols.map(col(_)): _*)
67    val inputJoinDFM = inputJoinDF.withColumn("timeStamp",$"timeStamp".plus(time_nterval * 1000))
68
69    val temp = inputJoinDF.as("a").join(inputJoinDFM.as("b"), Seq("timeStamp"), "left")
70
71    val fluxJoinSemih_temp = temp.where(col(s"b.${arrayHS(0)}").isNotNull)
72    val fluxJoinSemih = fluxJoinSemih_temp.select(arrayCols.map(x => {
73      if (x == LE) {
74        Array(col(s"a.$x").as(s"a_$x"))
75      } else if (x == "timeStamp") {
76        Array(col("timeStamp"))
77      } else if(x==rho_a_Avg){
78        Array(col(s"b.${rho_a_Avg}"))
79      }else{
80        Array(col(s"a.$x").as(s"a_$x")
81          , col(s"b.$x").as(s"b_$x"))
82      }
83    }).flatMap(x => x): _*)
84
85    val fluxJoinSemihArrayKeys: Seq[String] = fluxJoinSemih.schema.map(_.name)
86    val flux_ca = fluxJoinSemih.rdd
87      .map(rowToMap(fluxJoinSemihArrayKeys, _))
88      .map(x => {
89        val le_s_t = calculateDif(x, arrayH, arrayHS)
90        val le_s = x.get(s"a_$LE").get.toString.toDouble + (le_s_t.last / time_nterval * LV*x.get(rho_a_Avg).get.toString.toDouble)
91        (x.get("timeStamp").get.toString, le_s_t(0), le_s_t(1)
92          , le_s_t(2)
93          , le_s_t(3)
94          , le_s_t(4)
95          , le_s_t(5)
96          , le_s_t(6)
97          , le_s_t(7)
98          ,(le_s_t.last / time_nterval * LV*x.get(rho_a_Avg).get.toString.toDouble)
99          , le_s.formatted("%.5f"))
100      }).toDF("timeStamp", "d1"
101      , "d2"
102      , "d3"
103      , "d4"
104      , "d5"
105      , "d6"
106      , "d7"
107      , "d8"
108      , "d9"
109      , s"flux_$LE")
110//    flux_ca.orderBy(col("timeStamp")).show()
111    val colsResult: Seq[Column] = fluxCBS.schema.map(_.name).map(x => {
112      if (x == LE) {
113        when(col(s"flux_$x").isNull, col(x))
114          .otherwise(col(s"flux_$x")).as(x)
115      } else {
116        col(x)
117      }
118    })
119    val resultDF = fluxTimeStamp.join(flux_ca, Seq("timeStamp"), "left")
120      .select(colsResult: _*)
121    resultDF
122  }
123
124
125  //两个时刻的数据,计算CO2浓度、潜热通量、显热通量差值进行计算
126  def calculateDif(map: Map[String, Any], arrayH: List[Double], arrayV: List[String]): ArrayBuffer[Double] = {
127    val arr: ArrayBuffer[Double] = new ArrayBuffer[Double]()
128    var result = 0.0
129    for (index <- 0 to arrayH.length - 1) {
130      val fc_interval = arrayH(index) * (
131        map.get(s"a_${arrayV(index)}").get.toString.toDouble
132          -  map.get(s"b_${arrayV(index)}").get.toString.toDouble
133        )
134      arr.append(fc_interval)
135      result += fc_interval
136    }
137    arr.append(result)
138    arr
139  }
140
141  //将dataFrame中每一行数据转化成map[String,Any]数据
142  def rowToMap(array: Seq[String], row: Row): Map[String, Any] = {
143    val map = new mutable.HashMap[String, Any]()
144    for (index <- 0 to array.length - 1) {
145      map.+=((array(index), row.get(index)))
146    }
147    map.toMap
148  }
149
150}