upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/ScopeNormalization.scala
1package cn.piflow.bundle.normalization
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.functions._
8import org.apache.spark.sql.{DataFrame, SparkSession}
9
10class ScopeNormalization extends ConfigurableStop {
11
12 // 组件的作者信息
13 val authorEmail: String = "zljxnu@163.com"
14 // 组件的描述信息
15 val description: String = "Scope standardization"
16 // 定义输入端口
17 val inportList: List[String] = List(Port.DefaultPort)
18 // 定义输出端口
19 val outportList: List[String] = List(Port.DefaultPort)
20
21 // 定义输入列名称
22 var inputCol: String = _
23 // 定义输出列名称
24 var outputCol: String = _
25 // 定义目标范围 [a, b]
26 var range: (Double, Double) = (0.0, 1.0)
27
28 // 实际的数据处理逻辑
29 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
30 // 获取SparkSession
31 val spark = pec.get[SparkSession]()
32
33 // 读取输入数据
34 val dfOld = in.read()
35
36 // 使用范围映射公式进行数据处理
37 val dfNew = mapToRange(dfOld, inputCol, outputCol, range)
38
39 // 将处理后的数据写出
40 out.write(dfNew)
41 }
42
43 // 初始化方法
44 def initialize(ctx: ProcessContext): Unit = {}
45
46 // 设置组件属性
47 def setProperties(map: Map[String, Any]): Unit = {
48 inputCol = MapUtil.get(map, key = "inputCol").asInstanceOf[String]
49 outputCol = MapUtil.get(map, key = "outputCol").asInstanceOf[String]
50 val values = MapUtil.get(map, key = "range").asInstanceOf[String].stripPrefix("(").stripSuffix(")").split(",").map(_.toDouble)
51 range = (values(0), values(1))
52
53//// range = MapUtil.get(map, key = "range").asInstanceOf[(Double, Double)]
54// //把string解析成元组映射给range
55// val jsonString: String = MapUtil.get(map, key = "range").asInstanceOf[String]
56// // 移除括号并分割字符串
57// val values = jsonString.stripPrefix("(").stripSuffix(")").split(",").map(_.toDouble)
58// // 创建 Scala 元组
59// val range: (Double, Double) = (values(0), values(1))
60
61
62
63 }
64
65 // 定义组件的属性描述
66 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
67 var descriptor: List[PropertyDescriptor] = List()
68 val inputCol = new PropertyDescriptor()
69 .name("inputCol")
70 .displayName("Input Column")
71 .description("要映射的输入列的名称")
72 .defaultValue("")
73 .required(true)
74 .example("input_data")
75
76 val outputCol = new PropertyDescriptor()
77 .name("outputCol")
78 .displayName("Output Column")
79 .description("映射后的输出列的名称")
80 .defaultValue("")
81 .required(true)
82 .example("normalized_data")
83
84 val range = new PropertyDescriptor()
85 .name("range")
86 .displayName("Range")
87 .description("目标范围 [a, b],以元组的形式表示")
88 .defaultValue("")
89 .required(true)
90 .example("(0.0, 1.0)")
91
92 descriptor = inputCol :: outputCol :: range :: descriptor
93 descriptor
94 }
95
96 // 定义组件的图标(可选)
97 override def getIcon(): Array[Byte] = {
98 ImageUtil.getImage("icon/normalization/ScopeNormalization.png")
99 }
100
101 // 定义组件所属的分组(可选)
102 override def getGroup(): List[String] = {
103 List(StopGroup.NormalizationGroup)
104 }
105
106 // 实现范围映射的方法
107 private def mapToRange(df: DataFrame, inputCol: String, outputCol: String, range: (Double, Double)): DataFrame = {
108 // 使用Spark SQL的functions库来进行数据处理
109 val min = df.agg(Map(inputCol -> "min")).collect()(0)(0).asInstanceOf[Double]
110 val max = df.agg(Map(inputCol -> "max")).collect()(0)(0).asInstanceOf[Double]
111 val dfNew = df.withColumn(outputCol, (col(inputCol) - min) / (max - min) * (range._2 - range._1) + range._1)
112 dfNew
113 }
114}