Source code

Download source archive ↓

src/InterpolationMetro.scala

1package cn.piflow.bundle.flux.util
2
3
4import cn.piflow.bundle.flux.util.CommonUtil.toTimeStamp
5import org.apache.spark.sql.functions.{col, lit, udf}
6import org.apache.spark.sql.{DataFrame, SparkSession}
7
8/**
9 * @Author renhao
10 * @Description:
11 * @Data 2023/3/28 13:35
12 * @Modified By:
13 */
14class InterpolationMetro extends Serializable {
15
16
17  def runin(d1:Double,d2:Double,d3:Double,d4:Double,d5:Double,flagValue:Integer): Double ={
18
19    if(d1==null || d5==null){
20      println("-----------------------------------------------------------------------------------------------------")
21      println(d1,d2,d3,d4,d5)
22    }
23    if(d2==flagValue&&d3==flagValue&&d4!=flagValue&&d1!=null&&d1.toString.toDouble!=flagValue){
24//      (d1.toString.toDouble+d4)/2
25      d1.toString.toDouble+(d4-d1.toString.toDouble)*2/3
26    }else if(d3==flagValue&&d4==flagValue&&d2!=flagValue&&d5!=null&&d5.toString.toDouble!=flagValue){
27//      (d2+d5.toString.toDouble)/2
28      d2+(d5.toString.toDouble-d2)/3
29    }else{
30      d3
31    }
32  }
33  def filterFile(d:String,flagValue:Integer): Double ={
34    if(d==null){
35      flagValue.toDouble
36    }else{
37      d.toDouble
38    }
39  }
40
41  def imputationReplacement(spark:SparkSession,df:DataFrame,arrayFiled:String,time_nterval:Integer,flagVaue:Integer): DataFrame ={
42    import spark.implicits._
43    val toTime = udf(toTimeStamp _)
44    val df1 = df.withColumn("timeStamp",toTime(col("year"),col("month"),col("day"),col("hour")))
45
46    val df2 = df1.as("c").join(df1.as("a"),$"c.timeStamp"===$"a.timeStamp".plus(time_nterval*1000*2),"left")
47      .join(df1.as("b"),$"c.timeStamp"===$"b.timeStamp".plus(time_nterval*1000),"left")
48      .join(df1.as("d"),$"c.timeStamp"===$"d.timeStamp".minus(time_nterval*1000),"left")
49      .join(df1.as("e"),$"c.timeStamp"===$"e.timeStamp".minus(time_nterval*1000*2),"left")
50      .where(s"b.${arrayFiled.split(",")(0)} is not null and d.${arrayFiled.split(",")(0)} is not null")
51
52//    df2.orderBy($"c.timeStamp").select($"c.timeStamp",$"c.year",$"c.month",$"c.day",$"c.hour",$"c.Ta_1_AVG"
53//      ,$"a.Ta_1_AVG",$"b.Ta_1_AVG",$"d.Ta_1_AVG",$"e.Ta_1_AVG").show(50)
54    //a b c d e
55    val caculate = udf(runin _)
56    val f = udf(filterFile _)
57    val df3 = df2.select(arrayFiled.split(",")
58      .map(x=>caculate(f($"a.$x",lit(flagVaue)),f($"b.$x",lit(flagVaue)),f($"c.$x",lit(flagVaue)),f($"d.$x",lit(flagVaue)),f($"e.$x",lit(flagVaue)),lit(flagVaue)).as(x)).++(Array(col("c.timeStamp"))): _*)
59    val resultCol = df.schema.map(_.name)
60    val resultDF = df1.as("a").join(df3.as("b"),Seq("timeStamp"),"left")
61    resultDF.where(s"b.${arrayFiled.split(",")(0)} is not null")
62      .select(resultCol.map(x=>{
63        if(arrayFiled.split(",").contains(x)){
64          col(s"b.$x")
65        }else{
66          col(x)
67        }
68      }): _*)
69      .union(resultDF.where(s"b.${arrayFiled.split(",")(0)} is null").select(resultCol.map(x=>col(s"a.$x")): _*))
70  }
71}