Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_clustering/LDATraining.scala

1package cn.piflow.bundle.ml_clustering
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.LDA
8import org.apache.spark.sql.SparkSession
9
10class LDATraining extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "LDA clustering"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var training_data_path:String =_
16  var model_save_path:String=_
17  var k:Int=_
18  var checkpointInterval:Int=_
19  var maxIter:String=_
20  var docConcentration:String=_
21  var optimizer:String=_
22  var topicConcentration:String=_
23
24  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
25    val spark = pec.get[SparkSession]()
26
27    //load data stored in libsvm format as a dataframe
28    val data=spark.read.format("libsvm").load(training_data_path)
29
30
31    //Param for maximum number of iterations (>= 0)
32    var maxIterValue:Int=50
33    if(maxIter!=""){
34      maxIterValue=maxIter.toInt
35    }
36
37
38    var optimizerValue:String="online"
39    if(optimizer!=""){
40      optimizerValue=optimizer
41    }
42
43    var docConcentrationValue:Double=1/k
44    if(optimizer=="online"){
45      docConcentrationValue=1/k
46    }else{
47      docConcentrationValue=50/k+1
48    }
49    if(docConcentration!=""){
50      docConcentrationValue=docConcentration.toDouble
51    }
52
53    var topicConcentrationValue:Double=1/k
54    if(optimizer=="online"){
55      topicConcentrationValue=1/k
56    }else{
57      topicConcentrationValue=1.1
58    }
59    if(topicConcentration!=""){
60      topicConcentrationValue=topicConcentration.toDouble
61    }
62
63
64
65    //clustering with LDA algorithm
66    val model=new LDA()
67      .setMaxIter(maxIterValue)
68      .setCheckpointInterval(checkpointInterval)
69      .setDocConcentration(docConcentrationValue)
70      .setOptimizer(optimizerValue)
71      .setTopicConcentration(topicConcentrationValue)
72      .setK(k)
73      .fit(data)
74
75    //model persistence
76    model.save(model_save_path)
77
78    import spark.implicits._
79    val dfOut=Seq(model_save_path).toDF
80    dfOut.show()
81    out.write(dfOut)
82
83  }
84
85  def initialize(ctx: ProcessContext): Unit = {
86
87  }
88
89
90  def setProperties(map: Map[String, Any]): Unit = {
91    training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
92    model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
93    maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
94    docConcentration=MapUtil.get(map,key="docConcentration").asInstanceOf[String]
95    k=Integer.parseInt(MapUtil.get(map,key="k").toString)
96    topicConcentration=MapUtil.get(map,key="topicConcentration").asInstanceOf[String]
97    optimizer=MapUtil.get(map,key="optimizer").asInstanceOf[String]
98    checkpointInterval=Integer.parseInt(MapUtil.get(map,key="checkpointInterval").toString)
99
100  }
101
102  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
103    var descriptor : List[PropertyDescriptor] = List()
104    val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
105    val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("").defaultValue("").required(true)
106    val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0).").defaultValue("").required(false)
107    val k=new PropertyDescriptor().name("k").displayName("K").description("The number of topics. ").defaultValue("").required(true)
108    val docConcentration=new PropertyDescriptor().name("docConcentration").displayName("DOC_CONCENTRATION").description("Concentration parameter (commonly named \"alpha\") for the prior placed on documents' distributions over topics (\"theta\").").defaultValue("").required(false)
109    val topicConcentration=new PropertyDescriptor().name("topicConcentration").displayName("TOPIC_CONCENTRATION").description("Concentration parameter (commonly named \"beta\" or \"eta\") for the prior placed on topics' distributions over terms.").defaultValue("").required(false)
110    val checkpointInterval=new PropertyDescriptor().name("checkpointInterval").displayName("CHECK_POINT_INTERVAL").description("Param for set checkpoint interval (>= 1) or disable checkpoint (-1). E.g. 10 means that the cache will get checkpointed every 10 iterations.").defaultValue("").required(true)
111    val optimizer=new PropertyDescriptor().name("optimizer").displayName("OPTIMIZER").description("Optimizer or inference algorithm used to estimate the LDA model.").defaultValue("").required(false)
112    descriptor = training_data_path :: descriptor
113    descriptor = model_save_path :: descriptor
114    descriptor = maxIter :: descriptor
115    descriptor = docConcentration :: descriptor
116    descriptor = topicConcentration :: descriptor
117    descriptor = checkpointInterval :: descriptor
118    descriptor = optimizer :: descriptor
119    descriptor = k :: descriptor
120    descriptor
121  }
122
123  override def getIcon(): Array[Byte] = {
124    ImageUtil.getImage("icon/ml_clustering/LDATraining.png")
125  }
126
127  override def getGroup(): List[String] = {
128    List(StopGroup.MLGroup.toString)
129  }
130
131}