upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/ZScore.scala
1package cn.piflow.bundle.normalization
2
3import cn.piflow.conf._
4import cn.piflow.conf.bean.PropertyDescriptor
5import cn.piflow.conf.util.{ImageUtil, MapUtil}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8import org.apache.spark.sql.functions._
9
10class ZScore extends ConfigurableStop {
11
12 // 作者邮箱
13 val authorEmail: String = "zljxnu@163.cn"
14 // 描述
15 val description: String = "ZScore standardization"
16 // 输入端口
17 val inportList: List[String] = List(Port.DefaultPort)
18 // 输出端口
19 val outportList: List[String] = List(Port.DefaultPort)
20
21 // 输入列名称
22 var inputCols: String = _
23 // 输出列名称
24 var outputCols: String = _
25
26 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27 val spark = pec.get[SparkSession]()
28 val df = in.read()
29
30 // 将逗号分隔的输入和输出列名称拆分为列表
31 val inputColList = inputCols.split(",").map(_.trim)
32 val outputColList = outputCols.split(",").map(_.trim)
33
34 // 计算均值和标准差
35 val stats = inputColList.foldLeft(df) {
36 case (currentDf, inputCol) =>
37 val mean = currentDf.select(avg(col(inputCol))).first().getDouble(0)
38 val stdDev = currentDf.select(stddev(col(inputCol))).first().getDouble(0)
39 // 创建一个新列名:{inputCol}_zscore
40 val zScoreCol = s"${inputCol}_zscore"
41
42 // 使用公式进行 z-score 标准化
43 currentDf.withColumn(zScoreCol, (col(inputCol) - mean) / stdDev)
44 }
45
46 // 重命名输出列以匹配原始列名称
47 val finalDf = inputColList.zip(outputColList).foldLeft(stats) {
48 case (currentDf, (inputCol, outputCol)) =>
49 currentDf.withColumnRenamed(s"${inputCol}_zscore", outputCol)
50 }
51
52 out.write(finalDf)
53 }
54
55 def initialize(ctx: ProcessContext): Unit = {}
56
57 def setProperties(map: Map[String, Any]): Unit = {
58 inputCols = MapUtil.get(map, key = "inputCols").asInstanceOf[String]
59 outputCols = MapUtil.get(map, key = "outputCols").asInstanceOf[String]
60 }
61
62 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
63 var descriptor: List[PropertyDescriptor] = List()
64 val inputCols = new PropertyDescriptor()
65 .name("inputCols")
66 .displayName("输入列")
67 .description("要标准化的列,用逗号分隔。")
68 .defaultValue("")
69 .required(true)
70 .example("特征1, 特征2")
71
72 val outputCols = new PropertyDescriptor()
73 .name("outputCols")
74 .displayName("输出列")
75 .description("用于存储标准化值的相应输出列,用逗号分隔。")
76 .defaultValue("")
77 .required(true)
78 .example("标准化特征1, 标准化特征2")
79
80 descriptor = inputCols :: outputCols :: descriptor
81 descriptor
82 }
83
84 override def getIcon(): Array[Byte] = {
85 ImageUtil.getImage("icon/normalization/ZScoreNormalization.png")
86 }
87
88 override def getGroup(): List[String] = {
89 List(StopGroup.NormalizationGroup)
90 }
91}