src/InterpolationMetro.scala
1package cn.piflow.bundle.flux.util
2
3
4import cn.piflow.bundle.flux.util.CommonUtil.toTimeStamp
5import org.apache.spark.sql.functions.{col, lit, udf}
6import org.apache.spark.sql.{DataFrame, SparkSession}
7
8/**
9 * @Author renhao
10 * @Description:
11 * @Data 2023/3/28 13:35
12 * @Modified By:
13 */
14class InterpolationMetro extends Serializable {
15
16
17 def runin(d1:Double,d2:Double,d3:Double,d4:Double,d5:Double,flagValue:Integer): Double ={
18
19 if(d1==null || d5==null){
20 println("-----------------------------------------------------------------------------------------------------")
21 println(d1,d2,d3,d4,d5)
22 }
23 if(d2==flagValue&&d3==flagValue&&d4!=flagValue&&d1!=null&&d1.toString.toDouble!=flagValue){
24// (d1.toString.toDouble+d4)/2
25 d1.toString.toDouble+(d4-d1.toString.toDouble)*2/3
26 }else if(d3==flagValue&&d4==flagValue&&d2!=flagValue&&d5!=null&&d5.toString.toDouble!=flagValue){
27// (d2+d5.toString.toDouble)/2
28 d2+(d5.toString.toDouble-d2)/3
29 }else{
30 d3
31 }
32 }
33 def filterFile(d:String,flagValue:Integer): Double ={
34 if(d==null){
35 flagValue.toDouble
36 }else{
37 d.toDouble
38 }
39 }
40
41 def imputationReplacement(spark:SparkSession,df:DataFrame,arrayFiled:String,time_nterval:Integer,flagVaue:Integer): DataFrame ={
42 import spark.implicits._
43 val toTime = udf(toTimeStamp _)
44 val df1 = df.withColumn("timeStamp",toTime(col("year"),col("month"),col("day"),col("hour")))
45
46 val df2 = df1.as("c").join(df1.as("a"),$"c.timeStamp"===$"a.timeStamp".plus(time_nterval*1000*2),"left")
47 .join(df1.as("b"),$"c.timeStamp"===$"b.timeStamp".plus(time_nterval*1000),"left")
48 .join(df1.as("d"),$"c.timeStamp"===$"d.timeStamp".minus(time_nterval*1000),"left")
49 .join(df1.as("e"),$"c.timeStamp"===$"e.timeStamp".minus(time_nterval*1000*2),"left")
50 .where(s"b.${arrayFiled.split(",")(0)} is not null and d.${arrayFiled.split(",")(0)} is not null")
51
52// df2.orderBy($"c.timeStamp").select($"c.timeStamp",$"c.year",$"c.month",$"c.day",$"c.hour",$"c.Ta_1_AVG"
53// ,$"a.Ta_1_AVG",$"b.Ta_1_AVG",$"d.Ta_1_AVG",$"e.Ta_1_AVG").show(50)
54 //a b c d e
55 val caculate = udf(runin _)
56 val f = udf(filterFile _)
57 val df3 = df2.select(arrayFiled.split(",")
58 .map(x=>caculate(f($"a.$x",lit(flagVaue)),f($"b.$x",lit(flagVaue)),f($"c.$x",lit(flagVaue)),f($"d.$x",lit(flagVaue)),f($"e.$x",lit(flagVaue)),lit(flagVaue)).as(x)).++(Array(col("c.timeStamp"))): _*)
59 val resultCol = df.schema.map(_.name)
60 val resultDF = df1.as("a").join(df3.as("b"),Seq("timeStamp"),"left")
61 resultDF.where(s"b.${arrayFiled.split(",")(0)} is not null")
62 .select(resultCol.map(x=>{
63 if(arrayFiled.split(",").contains(x)){
64 col(s"b.$x")
65 }else{
66 col(x)
67 }
68 }): _*)
69 .union(resultDF.where(s"b.${arrayFiled.split(",")(0)} is null").select(resultCol.map(x=>col(s"a.$x")): _*))
70 }
71}