Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/ZScore.scala

1package cn.piflow.bundle.normalization
2
3import cn.piflow.conf._
4import cn.piflow.conf.bean.PropertyDescriptor
5import cn.piflow.conf.util.{ImageUtil, MapUtil}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8import org.apache.spark.sql.functions._
9
10class ZScore extends ConfigurableStop {
11
12  // 作者邮箱
13  val authorEmail: String = "zljxnu@163.cn"
14  // 描述
15  val description: String = "ZScore standardization"
16  // 输入端口
17  val inportList: List[String] = List(Port.DefaultPort)
18  // 输出端口
19  val outportList: List[String] = List(Port.DefaultPort)
20
21  // 输入列名称
22  var inputCols: String = _
23  // 输出列名称
24  var outputCols: String = _
25
26  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27    val spark = pec.get[SparkSession]()
28    val df = in.read()
29
30    // 将逗号分隔的输入和输出列名称拆分为列表
31    val inputColList = inputCols.split(",").map(_.trim)
32    val outputColList = outputCols.split(",").map(_.trim)
33
34    // 计算均值和标准差
35    val stats = inputColList.foldLeft(df) {
36      case (currentDf, inputCol) =>
37        val mean = currentDf.select(avg(col(inputCol))).first().getDouble(0)
38        val stdDev = currentDf.select(stddev(col(inputCol))).first().getDouble(0)
39        // 创建一个新列名:{inputCol}_zscore
40        val zScoreCol = s"${inputCol}_zscore"
41
42        // 使用公式进行 z-score 标准化
43        currentDf.withColumn(zScoreCol, (col(inputCol) - mean) / stdDev)
44    }
45
46    // 重命名输出列以匹配原始列名称
47    val finalDf = inputColList.zip(outputColList).foldLeft(stats) {
48      case (currentDf, (inputCol, outputCol)) =>
49        currentDf.withColumnRenamed(s"${inputCol}_zscore", outputCol)
50    }
51
52    out.write(finalDf)
53  }
54
55  def initialize(ctx: ProcessContext): Unit = {}
56
57  def setProperties(map: Map[String, Any]): Unit = {
58    inputCols = MapUtil.get(map, key = "inputCols").asInstanceOf[String]
59    outputCols = MapUtil.get(map, key = "outputCols").asInstanceOf[String]
60  }
61
62  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
63    var descriptor: List[PropertyDescriptor] = List()
64    val inputCols = new PropertyDescriptor()
65      .name("inputCols")
66      .displayName("输入列")
67      .description("要标准化的列,用逗号分隔。")
68      .defaultValue("")
69      .required(true)
70      .example("特征1, 特征2")
71
72    val outputCols = new PropertyDescriptor()
73      .name("outputCols")
74      .displayName("输出列")
75      .description("用于存储标准化值的相应输出列,用逗号分隔。")
76      .defaultValue("")
77      .required(true)
78      .example("标准化特征1, 标准化特征2")
79
80    descriptor = inputCols :: outputCols :: descriptor
81    descriptor
82  }
83
84  override def getIcon(): Array[Byte] = {
85    ImageUtil.getImage("icon/normalization/ZScoreNormalization.png")
86  }
87
88  override def getGroup(): List[String] = {
89    List(StopGroup.NormalizationGroup)
90  }
91}