upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/normalization/Discretization.scala
1package cn.piflow.bundle.normalization
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.{KMeans, KMeansModel}
8import org.apache.spark.ml.feature.VectorAssembler
9import org.apache.spark.ml.feature.Bucketizer
10import org.apache.spark.sql.{DataFrame, SparkSession}
11import org.apache.spark.ml.feature.QuantileDiscretizer
12
13
14class Discretization extends ConfigurableStop {
15
16 val authorEmail: String = "zljxnu@163.com"
17 val description: String = "continuous numerical discretization"
18 val inportList: List[String] = List(Port.DefaultPort)
19 val outportList: List[String] = List(Port.DefaultPort)
20
21 var inputCol: String = _
22 var outputCol: String = _
23 var method: String = _
24 var numBins: Int = _
25 var k: Int = _
26
27 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
28 val spark = pec.get[SparkSession]()
29 val df = in.read()
30
31 // 根据用户选择的方法进行相应的离散化
32 val discretizedDF = method match {
33 case "EqualWidth" => equalWidthDiscretization(df, inputCol, outputCol, numBins)
34 case "EqualFrequency" => equalFrequencyDiscretization(df, inputCol, outputCol, numBins)
35 case "KMeans" => kMeansDiscretization(df, inputCol, outputCol, k)
36 case _ => df // 默认情况下不进行任何处理
37 }
38
39 out.write(discretizedDF)
40 }
41
42 // 等宽法离散化
43 def equalWidthDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int): DataFrame = {
44 val bucketizer = new Bucketizer()
45 .setInputCol(inputCol)
46 .setOutputCol(outputCol)
47// .setSplits((0 to numBins).map(_.toDouble))
48 .setSplits((0 to numBins).map(_.toDouble).toArray)
49 bucketizer.transform(df)
50 }
51
52// // 等频离散化
53// def equalFrequencyDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int): DataFrame = {
54// val discretizer = new QuantileDiscretizer()
55// .setInputCol(inputCol)
56// .setOutputCol(outputCol)
57// .setNumBins(numBins)
58// discretizer.fit(df).transform(df)
59// }
60//
61// // 定义一个方法来执行等频离散化
62// def equalFrequencyDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int ): DataFrame = {
63// // 使用QuantileDiscretizer进行等频离散化
64// val discretizer = new QuantileDiscretizer()
65// .setInputCol(inputCol)
66// .setOutputCol(outputCol)
67// .setNumBins(numBins)
68//
69// val dfNew = discretizer.fit(df).transform(df)
70// dfNew
71// }
72
73 // 等频离散化
74 def equalFrequencyDiscretization(df: DataFrame, inputCol: String, outputCol: String, numBins: Int): DataFrame = {
75 // 创建一个QuantileDiscretizer实例,用于等频离散化
76 val discretizer = new QuantileDiscretizer()
77 .setInputCol(inputCol) // 设置输入列
78 .setOutputCol(outputCol) // 设置输出列
79 .setNumBuckets(numBins) // 设置桶的数量
80
81 // 使用数据来拟合(discretizer.fit)并进行离散化转换(discretizer.transform)
82 val dfNew = discretizer.fit(df).transform(df)
83 dfNew // 返回离散化后的DataFrame
84 }
85
86 // 聚类离散化
87 def kMeansDiscretization(df: DataFrame, inputCol: String, outputCol: String, k: Int): DataFrame = {
88 // 使用KMeans算法将数值列映射到[0, k-1]的整数
89 val assembler = new VectorAssembler()
90 .setInputCols(Array(inputCol))
91 .setOutputCol("features")
92 val vectorizedDF = assembler.transform(df)
93
94 val kmeans = new KMeans()
95 .setK(k)
96 .setSeed(1L)
97 .setFeaturesCol("features")
98 .setPredictionCol(outputCol)
99 val model = kmeans.fit(vectorizedDF)
100
101 val clusteredDF = model.transform(vectorizedDF)
102 clusteredDF.drop("features")
103 }
104
105 def initialize(ctx: ProcessContext): Unit = {}
106
107 def setProperties(map: Map[String, Any]): Unit = {
108 inputCol = MapUtil.get(map, "inputCol").asInstanceOf[String]
109 outputCol = MapUtil.get(map, "outputCol").asInstanceOf[String]
110 method = MapUtil.get(map, "method").asInstanceOf[String]
111 numBins = MapUtil.get(map, "numBins").asInstanceOf[String].toInt
112 k = MapUtil.get(map, "k").asInstanceOf[String].toInt
113 }
114
115 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
116 var descriptor: List[PropertyDescriptor] = List()
117
118 val inputColDescriptor = new PropertyDescriptor()
119 .name("inputCol")
120 .displayName("Input Column")
121 .description("The name of the input column to be discretized.")
122 .defaultValue("")
123 .required(true)
124
125 val outputColDescriptor = new PropertyDescriptor()
126 .name("outputCol")
127 .displayName("Output Column")
128 .description("The name of the output column to store discretized values.")
129 .defaultValue("")
130 .required(true)
131
132 val methodDescriptor = new PropertyDescriptor()
133 .name("method")
134 .displayName("Discretization Method")
135 .description("Choose the discretization method: EqualWidth, EqualFrequency, or KMeans.")
136 .allowableValues(Set("EqualWidth", "EqualFrequency", "KMeans"))
137 .defaultValue("EqualWidth")
138 .required(true)
139
140 val numBinsDescriptor = new PropertyDescriptor()
141 .name("numBins")
142 .displayName("Number of Bins")
143 .description("The number of bins to use for EqualWidth and EqualFrequency methods.")
144 .defaultValue("10")
145 .required(false)
146
147 val kDescriptor = new PropertyDescriptor()
148 .name("k")
149 .displayName("Number of Clusters (KMeans only)")
150 .description("The number of clusters to use for the KMeans method.")
151 .defaultValue("3")
152 .required(false)
153
154 descriptor = inputColDescriptor :: descriptor
155 descriptor = outputColDescriptor :: descriptor
156 descriptor = methodDescriptor :: descriptor
157 descriptor = numBinsDescriptor :: descriptor
158 descriptor = kDescriptor :: descriptor
159
160 descriptor
161 }
162
163 override def getIcon(): Array[Byte] = {
164 // 返回组件图标
165 ImageUtil.getImage("icon/normalization/DiscretizationNormalization.png")
166 }
167
168 override def getGroup(): List[String] = {
169 List(StopGroup.NormalizationGroup)
170 }
171}