upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/MaxMinNormalization.scala
1package cn.piflow.bundle.normalization
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8
9class MaxMinNormalization extends ConfigurableStop {
10 // 作者信息
11 val authorEmail: String = "zljxnu@163.com"
12 // 组件描述
13 val description: String = "maximum and minimum value standardization"
14 // 输入端口列表
15 val inportList: List[String] = List(Port.DefaultPort)
16 // 输出端口列表
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 // 定义属性:要标准化的列名
20 var inputCol: String = _
21
22 // 定义属性:输出列名
23 var outputCol: String = _
24
25 // 初始化方法
26 def initialize(ctx: ProcessContext): Unit = {}
27
28 // 执行方法
29 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
30 // 获取 SparkSession
31 val spark = pec.get[SparkSession]()
32
33 // 从输入端口读取数据
34 val df = in.read()
35
36 // 计算列的最大值和最小值
37 val max = df.agg(Map(inputCol -> "max")).collect()(0)(0).asInstanceOf[Double]
38 val min = df.agg(Map(inputCol -> "min")).collect()(0)(0).asInstanceOf[Double]
39
40 // 使用公式进行最小-最大值标准化
41 val scaledDf: DataFrame = df.withColumn(outputCol, (df(inputCol) - min) / (max - min))
42
43 // 将标准化后的数据写入输出端口
44 out.write(scaledDf)
45 }
46
47 // 设置属性
48 def setProperties(map: Map[String, Any]): Unit = {
49 inputCol = MapUtil.get(map, "inputCol").asInstanceOf[String]
50 outputCol = MapUtil.get(map, "outputCol").asInstanceOf[String]
51 }
52
53 // 获取属性描述
54 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
55 var descriptor: List[PropertyDescriptor] = List()
56 val inputCol = new PropertyDescriptor()
57 .name("inputCol")
58 .displayName("输入列名")
59 .description("要进行最小-最大值标准化的列名")
60 .defaultValue("")
61 .required(true)
62
63 val outputCol = new PropertyDescriptor()
64 .name("outputCol")
65 .displayName("Column_Name输出列名")
66 .description("Column names with numerical data to be scaled 标准化后的列名")
67 .defaultValue("")
68 .required(true)
69
70 descriptor = inputCol :: outputCol :: descriptor
71 descriptor
72 }
73
74 // 获取组件图标
75 override def getIcon(): Array[Byte] = {
76 ImageUtil.getImage("icon/normalization/MaxMinNormalization.png")
77 }
78
79 // 获取组件所属的组
80 override def getGroup(): List[String] = {
81 List(StopGroup.NormalizationGroup)
82 }
83}
84
85
86//package cn.piflow.bundle.normalization
87//
88//import cn.piflow.bundle.util.CleanUtil
89//import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
90//import cn.piflow.conf._
91//import cn.piflow.conf.bean.PropertyDescriptor
92//import cn.piflow.conf.util.{ImageUtil, MapUtil}
93//import org.apache.spark.sql.SparkSession
94//
95//class MaxMinNormalization extends ConfigurableStop {
96//
97// // 作者邮箱
98// val authorEmail: String = "zljxnu@163.com"
99// // 描述
100// val description: String = "MinMax scaling for numerical data"
101// // 输入端口列表
102// val inportList: List[String] = List(Port.DefaultPort)
103// // 输出端口列表
104// val outportList: List[String] = List(Port.DefaultPort)
105//
106// // 需要标准化的列名,从属性中设置
107// var columnName: String = _
108//
109// // 执行标准化操作
110// def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
111// val spark = pec.get[SparkSession]()
112// val sqlContext = spark.sqlContext
113// // 读取输入数据
114// val dfOld = in.read()
115// // 将输入数据创建为临时表
116// dfOld.createOrReplaceTempView("data")
117// // 解析需要标准化的列名
118// val columnNames = columnName.split(",").toSet
119//
120// val sqlNewFieldStr = new StringBuilder
121// // 针对每个指定的列名,生成标准化的 SQL 代码
122// columnNames.foreach(c => {
123// sqlNewFieldStr ++= ",((("
124// sqlNewFieldStr ++= c
125// sqlNewFieldStr ++= " - min("
126// sqlNewFieldStr ++= c
127// sqlNewFieldStr ++= ")) / (max("
128// sqlNewFieldStr ++= c
129// sqlNewFieldStr ++= ") - min("
130// sqlNewFieldStr ++= c
131// sqlNewFieldStr ++= "))) as "
132// sqlNewFieldStr ++= c
133// sqlNewFieldStr ++= "_scaled "
134// })
135//
136// // 构建最终的 SQL 查询文本
137// val sqlText: String = "select * " + sqlNewFieldStr + " from data"
138//
139// // 执行 SQL 查询,得到标准化后的 DataFrame
140// val dfNew = sqlContext.sql(sqlText)
141// dfNew.createOrReplaceTempView("scaled_data")
142//
143// // 将标准化后的数据写入输出
144// out.write(dfNew)
145// }
146//
147// // 初始化方法
148// def initialize(ctx: ProcessContext): Unit = {}
149//
150// // 设置属性
151// def setProperties(map: Map[String, Any]): Unit = {
152// // 从属性映射中获取需要标准化的列名
153// columnName = MapUtil.get(map, key = "columnName").asInstanceOf[String]
154// }
155//
156// // 定义属性描述符
157// override def getPropertyDescriptor(): List[PropertyDescriptor] = {
158// var descriptor: List[PropertyDescriptor] = List()
159// val columnNameDesc = new PropertyDescriptor()
160// .name("columnName")
161// .displayName("Column_Name")
162// .description("Column names with numerical data to be scaled (comma-separated)")
163// .defaultValue("")
164// .required(true)
165// .example("feature1,feature2")
166//
167// descriptor = columnNameDesc :: descriptor
168// descriptor
169// }
170//
171// // 获取图标
172// override def getIcon(): Array[Byte] = {
173// ImageUtil.getImage("icon/normalization/MaxMinNormalization.png")
174// }
175//
176// // 获取所属组
177// override def getGroup(): List[String] = {
178// List(StopGroup.NormalizationGroup)
179// }
180//}