Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/MaxMinNormalization.scala

1package cn.piflow.bundle.normalization
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8
9class MaxMinNormalization extends ConfigurableStop {
10  // 作者信息
11  val authorEmail: String = "zljxnu@163.com"
12  // 组件描述
13  val description: String = "maximum and minimum value standardization"
14  // 输入端口列表
15  val inportList: List[String] = List(Port.DefaultPort)
16  // 输出端口列表
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  // 定义属性:要标准化的列名
20  var inputCol: String = _
21
22  // 定义属性:输出列名
23  var outputCol: String = _
24
25  // 初始化方法
26  def initialize(ctx: ProcessContext): Unit = {}
27
28  // 执行方法
29  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
30    // 获取 SparkSession
31    val spark = pec.get[SparkSession]()
32
33    // 从输入端口读取数据
34    val df = in.read()
35
36    // 计算列的最大值和最小值
37    val max = df.agg(Map(inputCol -> "max")).collect()(0)(0).asInstanceOf[Double]
38    val min = df.agg(Map(inputCol -> "min")).collect()(0)(0).asInstanceOf[Double]
39
40    // 使用公式进行最小-最大值标准化
41    val scaledDf: DataFrame = df.withColumn(outputCol, (df(inputCol) - min) / (max - min))
42
43    // 将标准化后的数据写入输出端口
44    out.write(scaledDf)
45  }
46
47  // 设置属性
48  def setProperties(map: Map[String, Any]): Unit = {
49    inputCol = MapUtil.get(map, "inputCol").asInstanceOf[String]
50    outputCol = MapUtil.get(map, "outputCol").asInstanceOf[String]
51  }
52
53  // 获取属性描述
54  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
55    var descriptor: List[PropertyDescriptor] = List()
56    val inputCol = new PropertyDescriptor()
57      .name("inputCol")
58      .displayName("输入列名")
59      .description("要进行最小-最大值标准化的列名")
60      .defaultValue("")
61      .required(true)
62
63    val outputCol = new PropertyDescriptor()
64      .name("outputCol")
65      .displayName("Column_Name输出列名")
66      .description("Column names with numerical data to be scaled 标准化后的列名")
67      .defaultValue("")
68      .required(true)
69
70    descriptor = inputCol :: outputCol :: descriptor
71    descriptor
72  }
73
74  // 获取组件图标
75  override def getIcon(): Array[Byte] = {
76    ImageUtil.getImage("icon/normalization/MaxMinNormalization.png")
77  }
78
79  // 获取组件所属的组
80  override def getGroup(): List[String] = {
81    List(StopGroup.NormalizationGroup)
82  }
83}
84
85
86//package cn.piflow.bundle.normalization
87//
88//import cn.piflow.bundle.util.CleanUtil
89//import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
90//import cn.piflow.conf._
91//import cn.piflow.conf.bean.PropertyDescriptor
92//import cn.piflow.conf.util.{ImageUtil, MapUtil}
93//import org.apache.spark.sql.SparkSession
94//
95//class MaxMinNormalization extends ConfigurableStop {
96//
97//  // 作者邮箱
98//  val authorEmail: String = "zljxnu@163.com"
99//  // 描述
100//  val description: String = "MinMax scaling for numerical data"
101//  // 输入端口列表
102//  val inportList: List[String] = List(Port.DefaultPort)
103//  // 输出端口列表
104//  val outportList: List[String] = List(Port.DefaultPort)
105//
106//  // 需要标准化的列名,从属性中设置
107//  var columnName: String = _
108//
109//  // 执行标准化操作
110//  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
111//    val spark = pec.get[SparkSession]()
112//    val sqlContext = spark.sqlContext
113//    // 读取输入数据
114//    val dfOld = in.read()
115//    // 将输入数据创建为临时表
116//    dfOld.createOrReplaceTempView("data")
117//    // 解析需要标准化的列名
118//    val columnNames = columnName.split(",").toSet
119//
120//    val sqlNewFieldStr = new StringBuilder
121//    // 针对每个指定的列名,生成标准化的 SQL 代码
122//    columnNames.foreach(c => {
123//      sqlNewFieldStr ++= ",((("
124//      sqlNewFieldStr ++= c
125//      sqlNewFieldStr ++= " - min("
126//      sqlNewFieldStr ++= c
127//      sqlNewFieldStr ++= ")) / (max("
128//      sqlNewFieldStr ++= c
129//      sqlNewFieldStr ++= ") - min("
130//      sqlNewFieldStr ++= c
131//      sqlNewFieldStr ++= "))) as "
132//      sqlNewFieldStr ++= c
133//      sqlNewFieldStr ++= "_scaled "
134//    })
135//
136//    // 构建最终的 SQL 查询文本
137//    val sqlText: String = "select * " + sqlNewFieldStr + " from data"
138//
139//    // 执行 SQL 查询,得到标准化后的 DataFrame
140//    val dfNew = sqlContext.sql(sqlText)
141//    dfNew.createOrReplaceTempView("scaled_data")
142//
143//    // 将标准化后的数据写入输出
144//    out.write(dfNew)
145//  }
146//
147//  // 初始化方法
148//  def initialize(ctx: ProcessContext): Unit = {}
149//
150//  // 设置属性
151//  def setProperties(map: Map[String, Any]): Unit = {
152//    // 从属性映射中获取需要标准化的列名
153//    columnName = MapUtil.get(map, key = "columnName").asInstanceOf[String]
154//  }
155//
156//  // 定义属性描述符
157//  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
158//    var descriptor: List[PropertyDescriptor] = List()
159//    val columnNameDesc = new PropertyDescriptor()
160//      .name("columnName")
161//      .displayName("Column_Name")
162//      .description("Column names with numerical data to be scaled (comma-separated)")
163//      .defaultValue("")
164//      .required(true)
165//      .example("feature1,feature2")
166//
167//    descriptor = columnNameDesc :: descriptor
168//    descriptor
169//  }
170//
171//  // 获取图标
172//  override def getIcon(): Array[Byte] = {
173//    ImageUtil.getImage("icon/normalization/MaxMinNormalization.png")
174//  }
175//
176//  // 获取所属组
177//  override def getGroup(): List[String] = {
178//    List(StopGroup.NormalizationGroup)
179//  }
180//}