Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/Discretization.scala

1package cn.piflow.bundle.normalization
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.{KMeans, KMeansModel}
8import org.apache.spark.ml.feature.VectorAssembler
9import org.apache.spark.ml.feature.Bucketizer
10import org.apache.spark.sql.{DataFrame, SparkSession}
11import org.apache.spark.ml.feature.QuantileDiscretizer
12
13
14class Discretization extends ConfigurableStop {
15
16  val authorEmail: String = "zljxnu@163.com"
17  val description: String = "continuous numerical discretization"
18  val inportList: List[String] = List(Port.DefaultPort)
19  val outportList: List[String] = List(Port.DefaultPort)
20
21  var inputCol: String = _
22  var outputCol: String = _
23  var method: String = _
24  var numBins: Int = _
25  var k: Int = _
26
27  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
28    val spark = pec.get[SparkSession]()
29    val df = in.read()
30
31    // 根据用户选择的方法进行相应的离散化
32    val discretizedDF = method match {
33      case "EqualWidth" => equalWidthDiscretization(df, inputCol, outputCol, numBins)
34      case "EqualFrequency" => equalFrequencyDiscretization(df, inputCol, outputCol, numBins)
35      case "KMeans" => kMeansDiscretization(df, inputCol, outputCol, k)
36      case _ => df // 默认情况下不进行任何处理
37    }
38
39    out.write(discretizedDF)
40  }
41
42  // 等宽法离散化
43  def equalWidthDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int): DataFrame = {
44    val bucketizer = new Bucketizer()
45      .setInputCol(inputCol)
46      .setOutputCol(outputCol)
47//      .setSplits((0 to numBins).map(_.toDouble))
48      .setSplits((0 to numBins).map(_.toDouble).toArray)
49    bucketizer.transform(df)
50  }
51
52//  // 等频离散化
53//  def equalFrequencyDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int): DataFrame = {
54//    val discretizer = new QuantileDiscretizer()
55//      .setInputCol(inputCol)
56//      .setOutputCol(outputCol)
57//      .setNumBins(numBins)
58//    discretizer.fit(df).transform(df)
59//  }
60//
61//  // 定义一个方法来执行等频离散化
62//  def equalFrequencyDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int ): DataFrame = {
63//    // 使用QuantileDiscretizer进行等频离散化
64//    val discretizer = new QuantileDiscretizer()
65//      .setInputCol(inputCol)
66//      .setOutputCol(outputCol)
67//      .setNumBins(numBins)
68//
69//    val dfNew = discretizer.fit(df).transform(df)
70//    dfNew
71//  }
72
73  // 等频离散化
74  def equalFrequencyDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int): DataFrame = {
75    // 创建一个QuantileDiscretizer实例,用于等频离散化
76    val discretizer = new QuantileDiscretizer()
77      .setInputCol(inputCol) // 设置输入列
78      .setOutputCol(outputCol) // 设置输出列
79      .setNumBuckets(numBins) // 设置桶的数量
80
81    // 使用数据来拟合(discretizer.fit)并进行离散化转换(discretizer.transform)
82    val dfNew = discretizer.fit(df).transform(df)
83    dfNew // 返回离散化后的DataFrame
84  }
85
86  // 聚类离散化
87  def kMeansDiscretization(df: DataFrame, inputCol: String, outputCol: String, k: Int): DataFrame = {
88    // 使用KMeans算法将数值列映射到[0, k-1]的整数
89    val assembler = new VectorAssembler()
90      .setInputCols(Array(inputCol))
91      .setOutputCol("features")
92    val vectorizedDF = assembler.transform(df)
93
94    val kmeans = new KMeans()
95      .setK(k)
96      .setSeed(1L)
97      .setFeaturesCol("features")
98      .setPredictionCol(outputCol)
99    val model = kmeans.fit(vectorizedDF)
100
101    val clusteredDF = model.transform(vectorizedDF)
102    clusteredDF.drop("features")
103  }
104
105  def initialize(ctx: ProcessContext): Unit = {}
106
107  def setProperties(map: Map[String, Any]): Unit = {
108    inputCol = MapUtil.get(map, "inputCol").asInstanceOf[String]
109    outputCol = MapUtil.get(map, "outputCol").asInstanceOf[String]
110    method = MapUtil.get(map, "method").asInstanceOf[String]
111    numBins = MapUtil.get(map, "numBins").asInstanceOf[String].toInt
112    k = MapUtil.get(map, "k").asInstanceOf[String].toInt
113  }
114
115  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
116    var descriptor: List[PropertyDescriptor] = List()
117
118    val inputColDescriptor = new PropertyDescriptor()
119      .name("inputCol")
120      .displayName("Input Column")
121      .description("The name of the input column to be discretized.")
122      .defaultValue("")
123      .required(true)
124
125    val outputColDescriptor = new PropertyDescriptor()
126      .name("outputCol")
127      .displayName("Output Column")
128      .description("The name of the output column to store discretized values.")
129      .defaultValue("")
130      .required(true)
131
132    val methodDescriptor = new PropertyDescriptor()
133      .name("method")
134      .displayName("Discretization Method")
135      .description("Choose the discretization method: EqualWidth, EqualFrequency, or KMeans.")
136      .allowableValues(Set("EqualWidth", "EqualFrequency", "KMeans"))
137      .defaultValue("EqualWidth")
138      .required(true)
139
140    val numBinsDescriptor = new PropertyDescriptor()
141      .name("numBins")
142      .displayName("Number of Bins")
143      .description("The number of bins to use for EqualWidth and EqualFrequency methods.")
144      .defaultValue("10")
145      .required(false)
146
147    val kDescriptor = new PropertyDescriptor()
148      .name("k")
149      .displayName("Number of Clusters (KMeans only)")
150      .description("The number of clusters to use for the KMeans method.")
151      .defaultValue("3")
152      .required(false)
153
154    descriptor = inputColDescriptor :: descriptor
155    descriptor = outputColDescriptor :: descriptor
156    descriptor = methodDescriptor :: descriptor
157    descriptor = numBinsDescriptor :: descriptor
158    descriptor = kDescriptor :: descriptor
159
160    descriptor
161  }
162
163  override def getIcon(): Array[Byte] = {
164    // 返回组件图标
165    ImageUtil.getImage("icon/normalization/DiscretizationNormalization.png")
166  }
167
168  override def getGroup(): List[String] = {
169    List(StopGroup.NormalizationGroup)
170  }
171}