Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_clustering/GaussianMixtureTraining.scala

1package cn.piflow.bundle.ml_clustering
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.GaussianMixture
8import org.apache.spark.sql.SparkSession
9
10class GaussianMixtureTraining extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "GaussianMixture clustering"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var training_data_path:String =_
16  var model_save_path:String=_
17  var k:String=_
18  var maxIter:String=_
19  var tol:String=_
20
21  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
22    val spark = pec.get[SparkSession]()
23
24    //load data stored in libsvm format as a dataframe
25    val data=spark.read.format("libsvm").load(training_data_path)
26
27    var kValue:Int=2
28    if(k!=""){
29      kValue=k.toInt
30    }
31
32    //Param for maximum number of iterations (>= 0)
33    var maxIterValue:Int=50
34    if(maxIter!=""){
35      maxIterValue=maxIter.toInt
36    }
37
38
39    var tolValue:Double=0.0001
40    if(tol=="online"){
41      tolValue=tol.toDouble
42    }
43
44
45    //clustering with GaussianMixture algorithm
46    val model=new GaussianMixture()
47      .setMaxIter(maxIterValue)
48      .setTol(tolValue)
49      .setK(kValue)
50      .fit(data)
51
52    //model persistence
53    model.save(model_save_path)
54
55    import spark.implicits._
56    val dfOut=Seq(model_save_path).toDF
57    dfOut.show()
58    out.write(dfOut)
59
60  }
61
62  def initialize(ctx: ProcessContext): Unit = {
63
64  }
65
66
67  def setProperties(map: Map[String, Any]): Unit = {
68    training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
69    model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
70    maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
71    k=MapUtil.get(map,key="k").asInstanceOf[String]
72    tol=MapUtil.get(map,key="tol").asInstanceOf[String]
73
74  }
75
76  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
77    var descriptor : List[PropertyDescriptor] = List()
78    val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
79    val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("").defaultValue("").required(true)
80    val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0).").defaultValue("").required(false)
81    val k=new PropertyDescriptor().name("k").displayName("K").description("Number of independent Gaussians in the mixture model. Must be greater than 1. Default: 2. ").defaultValue("").required(false)
82    val tol=new PropertyDescriptor().name("tol").displayName("TOL").description("Param for the convergence tolerance for iterative algorithms (>= 0).").defaultValue("").required(false)
83    descriptor = training_data_path :: descriptor
84    descriptor = model_save_path :: descriptor
85    descriptor = maxIter :: descriptor
86    descriptor = tol :: descriptor
87    descriptor = k :: descriptor
88    descriptor
89  }
90
91  override def getIcon(): Array[Byte] = {
92    ImageUtil.getImage("icon/ml_clustering/GaussianMixtureTraining.png")
93  }
94
95  override def getGroup(): List[String] = {
96    List(StopGroup.MLGroup.toString)
97  }
98
99}