Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/ScopeNormalization.scala

1package cn.piflow.bundle.normalization
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.functions._
8import org.apache.spark.sql.{DataFrame, SparkSession}
9
10class ScopeNormalization extends ConfigurableStop {
11
12  // 组件的作者信息
13  val authorEmail: String = "zljxnu@163.com"
14  // 组件的描述信息
15  val description: String = "Scope standardization"
16  // 定义输入端口
17  val inportList: List[String] = List(Port.DefaultPort)
18  // 定义输出端口
19  val outportList: List[String] = List(Port.DefaultPort)
20
21  // 定义输入列名称
22  var inputCol: String = _
23  // 定义输出列名称
24  var outputCol: String = _
25  // 定义目标范围 [a, b]
26  var range: (Double, Double) = (0.0, 1.0)
27
28  // 实际的数据处理逻辑
29  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
30    // 获取SparkSession
31    val spark = pec.get[SparkSession]()
32
33    // 读取输入数据
34    val dfOld = in.read()
35
36    // 使用范围映射公式进行数据处理
37    val dfNew = mapToRange(dfOld, inputCol, outputCol, range)
38
39    // 将处理后的数据写出
40    out.write(dfNew)
41  }
42
43  // 初始化方法
44  def initialize(ctx: ProcessContext): Unit = {}
45
46  // 设置组件属性
47  def setProperties(map: Map[String, Any]): Unit = {
48    inputCol = MapUtil.get(map, key = "inputCol").asInstanceOf[String]
49    outputCol = MapUtil.get(map, key = "outputCol").asInstanceOf[String]
50    val values = MapUtil.get(map, key = "range").asInstanceOf[String].stripPrefix("(").stripSuffix(")").split(",").map(_.toDouble)
51    range = (values(0), values(1))
52
53////    range = MapUtil.get(map, key = "range").asInstanceOf[(Double, Double)]
54//    //把string解析成元组映射给range
55//    val jsonString: String = MapUtil.get(map, key = "range").asInstanceOf[String]
56//    // 移除括号并分割字符串
57//    val values = jsonString.stripPrefix("(").stripSuffix(")").split(",").map(_.toDouble)
58//    // 创建 Scala 元组
59//    val range: (Double, Double) = (values(0), values(1))
60
61
62
63  }
64
65  // 定义组件的属性描述
66  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
67    var descriptor: List[PropertyDescriptor] = List()
68    val inputCol = new PropertyDescriptor()
69      .name("inputCol")
70      .displayName("Input Column")
71      .description("要映射的输入列的名称")
72      .defaultValue("")
73      .required(true)
74      .example("input_data")
75
76    val outputCol = new PropertyDescriptor()
77      .name("outputCol")
78      .displayName("Output Column")
79      .description("映射后的输出列的名称")
80      .defaultValue("")
81      .required(true)
82      .example("normalized_data")
83
84    val range = new PropertyDescriptor()
85      .name("range")
86      .displayName("Range")
87      .description("目标范围 [a, b],以元组的形式表示")
88      .defaultValue("")
89      .required(true)
90      .example("(0.0, 1.0)")
91
92    descriptor = inputCol :: outputCol :: range :: descriptor
93    descriptor
94  }
95
96  // 定义组件的图标(可选)
97  override def getIcon(): Array[Byte] = {
98    ImageUtil.getImage("icon/normalization/ScopeNormalization.png")
99  }
100
101  // 定义组件所属的分组(可选)
102  override def getGroup(): List[String] = {
103    List(StopGroup.NormalizationGroup)
104  }
105
106  // 实现范围映射的方法
107  private def mapToRange(df: DataFrame, inputCol: String, outputCol: String, range: (Double, Double)): DataFrame = {
108    // 使用Spark SQL的functions库来进行数据处理
109    val min = df.agg(Map(inputCol -> "min")).collect()(0)(0).asInstanceOf[Double]
110    val max = df.agg(Map(inputCol -> "max")).collect()(0)(0).asInstanceOf[Double]
111    val dfNew = df.withColumn(outputCol, (col(inputCol) - min) / (max - min) * (range._2 - range._1) + range._1)
112    dfNew
113  }
114}