Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_clustering/KmeansTraining.scala

1package cn.piflow.bundle.ml_clustering
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.KMeans
8import org.apache.spark.sql.SparkSession
9
10class KmeansTraining extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "Kmeans clustering"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var training_data_path:String =_
16  var model_save_path:String=_
17  var maxIter:String=_
18  var k:Int=_
19  var minTol:String=_
20
21
22  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23    val spark = pec.get[SparkSession]()
24
25    //load data stored in libsvm format as a dataframe
26    val data=spark.read.format("libsvm").load(training_data_path)
27
28
29    //Param for maximum number of iterations (>= 0)
30    var maxIterValue:Int=50
31    if(maxIter!=""){
32      maxIterValue=maxIter.toInt
33    }
34
35    //Param for the convergence tolerance for iterative algorithms (>= 0).
36    var minTolValue:Double=1E-6
37    if(minTol!=""){
38      minTolValue=minTol.toDouble
39    }
40
41
42    //clustering with kmeans algorithm
43    val model=new KMeans()
44      .setMaxIter(maxIterValue)
45      .setTol(minTolValue)
46      .setK(k)
47      .fit(data)
48
49    //model persistence
50    model.save(model_save_path)
51
52    import spark.implicits._
53    val dfOut=Seq(model_save_path).toDF
54    dfOut.show()
55    out.write(dfOut)
56
57  }
58
59  def initialize(ctx: ProcessContext): Unit = {
60
61  }
62
63
64  def setProperties(map: Map[String, Any]): Unit = {
65    training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
66    model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
67    maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
68    minTol=MapUtil.get(map,key="minTol").asInstanceOf[String]
69    k=Integer.parseInt(MapUtil.get(map,key="k").toString)
70  }
71
72  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
73    var descriptor : List[PropertyDescriptor] = List()
74    val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
75    val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("").defaultValue("").required(true)
76    val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0).").defaultValue("").required(false)
77    val minTol=new PropertyDescriptor().name("minTol").displayName("MIN_TOL").description("Param for the convergence tolerance for iterative algorithms (>= 0).").defaultValue("").required(false)
78    val k=new PropertyDescriptor().name("k").displayName("K").description("The number of clusters. ").defaultValue("").required(true)
79    descriptor = training_data_path :: descriptor
80    descriptor = model_save_path :: descriptor
81    descriptor = maxIter :: descriptor
82    descriptor = minTol :: descriptor
83    descriptor = k :: descriptor
84    descriptor
85  }
86
87  override def getIcon(): Array[Byte] = {
88    ImageUtil.getImage("icon/ml_clustering/KmeansTraining.png")
89  }
90
91  override def getGroup(): List[String] = {
92    List(StopGroup.MLGroup.toString)
93  }
94
95}