upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_clustering/BisectingKMeansTraining.scala
1package cn.piflow.bundle.ml_clustering
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.BisectingKMeans
8import org.apache.spark.sql.SparkSession
9
10class BisectingKMeansTraining extends ConfigurableStop{
11 val authorEmail: String = "06whuxx@163.com"
12 val description: String = "BisectingKMeans clustering"
13 val inportList: List[String] = List(Port.DefaultPort)
14 val outportList: List[String] = List(Port.DefaultPort)
15 var training_data_path:String =_
16 var model_save_path:String=_
17 var maxIter:String=_
18 var k:Int=_
19
20
21 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
22 val spark = pec.get[SparkSession]()
23
24 //load data stored in libsvm format as a dataframe
25 val data=spark.read.format("libsvm").load(training_data_path)
26
27
28 //Param for maximum number of iterations (>= 0)
29 var maxIterValue:Int=50
30 if(maxIter!=""){
31 maxIterValue=maxIter.toInt
32 }
33
34
35 //clustering with BisectingKMeans algorithm
36 val model=new BisectingKMeans()
37 .setMaxIter(maxIterValue)
38 .setK(k)
39 .fit(data)
40
41 //model persistence
42 model.save(model_save_path)
43
44 import spark.implicits._
45 val dfOut=Seq(model_save_path).toDF
46 dfOut.show()
47 out.write(dfOut)
48
49 }
50
51 def initialize(ctx: ProcessContext): Unit = {
52
53 }
54
55
56 def setProperties(map: Map[String, Any]): Unit = {
57 training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
58 model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
59 maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
60 k=Integer.parseInt(MapUtil.get(map,key="k").toString)
61 }
62
63 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
64 var descriptor : List[PropertyDescriptor] = List()
65 val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
66 val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("").defaultValue("").required(true)
67 val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0).").defaultValue("").required(false)
68 val k=new PropertyDescriptor().name("k").displayName("K").description("The number of clusters. ").defaultValue("").required(true)
69 descriptor = training_data_path :: descriptor
70 descriptor = model_save_path :: descriptor
71 descriptor = maxIter :: descriptor
72 descriptor = k :: descriptor
73 descriptor
74 }
75
76 override def getIcon(): Array[Byte] = {
77 ImageUtil.getImage("icon/ml_clustering/BisetingKmeansTraining.png")
78 }
79
80 override def getGroup(): List[String] = {
81 List(StopGroup.MLGroup.toString)
82 }
83
84}