upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_clustering/LDATraining.scala
1package cn.piflow.bundle.ml_clustering
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.LDA
8import org.apache.spark.sql.SparkSession
9
10class LDATraining extends ConfigurableStop{
11 val authorEmail: String = "06whuxx@163.com"
12 val description: String = "LDA clustering"
13 val inportList: List[String] = List(Port.DefaultPort)
14 val outportList: List[String] = List(Port.DefaultPort)
15 var training_data_path:String =_
16 var model_save_path:String=_
17 var k:Int=_
18 var checkpointInterval:Int=_
19 var maxIter:String=_
20 var docConcentration:String=_
21 var optimizer:String=_
22 var topicConcentration:String=_
23
24 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
25 val spark = pec.get[SparkSession]()
26
27 //load data stored in libsvm format as a dataframe
28 val data=spark.read.format("libsvm").load(training_data_path)
29
30
31 //Param for maximum number of iterations (>= 0)
32 var maxIterValue:Int=50
33 if(maxIter!=""){
34 maxIterValue=maxIter.toInt
35 }
36
37
38 var optimizerValue:String="online"
39 if(optimizer!=""){
40 optimizerValue=optimizer
41 }
42
43 var docConcentrationValue:Double=1/k
44 if(optimizer=="online"){
45 docConcentrationValue=1/k
46 }else{
47 docConcentrationValue=50/k+1
48 }
49 if(docConcentration!=""){
50 docConcentrationValue=docConcentration.toDouble
51 }
52
53 var topicConcentrationValue:Double=1/k
54 if(optimizer=="online"){
55 topicConcentrationValue=1/k
56 }else{
57 topicConcentrationValue=1.1
58 }
59 if(topicConcentration!=""){
60 topicConcentrationValue=topicConcentration.toDouble
61 }
62
63
64
65 //clustering with LDA algorithm
66 val model=new LDA()
67 .setMaxIter(maxIterValue)
68 .setCheckpointInterval(checkpointInterval)
69 .setDocConcentration(docConcentrationValue)
70 .setOptimizer(optimizerValue)
71 .setTopicConcentration(topicConcentrationValue)
72 .setK(k)
73 .fit(data)
74
75 //model persistence
76 model.save(model_save_path)
77
78 import spark.implicits._
79 val dfOut=Seq(model_save_path).toDF
80 dfOut.show()
81 out.write(dfOut)
82
83 }
84
85 def initialize(ctx: ProcessContext): Unit = {
86
87 }
88
89
90 def setProperties(map: Map[String, Any]): Unit = {
91 training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
92 model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
93 maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
94 docConcentration=MapUtil.get(map,key="docConcentration").asInstanceOf[String]
95 k=Integer.parseInt(MapUtil.get(map,key="k").toString)
96 topicConcentration=MapUtil.get(map,key="topicConcentration").asInstanceOf[String]
97 optimizer=MapUtil.get(map,key="optimizer").asInstanceOf[String]
98 checkpointInterval=Integer.parseInt(MapUtil.get(map,key="checkpointInterval").toString)
99
100 }
101
102 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
103 var descriptor : List[PropertyDescriptor] = List()
104 val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
105 val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("").defaultValue("").required(true)
106 val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0).").defaultValue("").required(false)
107 val k=new PropertyDescriptor().name("k").displayName("K").description("The number of topics. ").defaultValue("").required(true)
108 val docConcentration=new PropertyDescriptor().name("docConcentration").displayName("DOC_CONCENTRATION").description("Concentration parameter (commonly named \"alpha\") for the prior placed on documents' distributions over topics (\"theta\").").defaultValue("").required(false)
109 val topicConcentration=new PropertyDescriptor().name("topicConcentration").displayName("TOPIC_CONCENTRATION").description("Concentration parameter (commonly named \"beta\" or \"eta\") for the prior placed on topics' distributions over terms.").defaultValue("").required(false)
110 val checkpointInterval=new PropertyDescriptor().name("checkpointInterval").displayName("CHECK_POINT_INTERVAL").description("Param for set checkpoint interval (>= 1) or disable checkpoint (-1). E.g. 10 means that the cache will get checkpointed every 10 iterations.").defaultValue("").required(true)
111 val optimizer=new PropertyDescriptor().name("optimizer").displayName("OPTIMIZER").description("Optimizer or inference algorithm used to estimate the LDA model.").defaultValue("").required(false)
112 descriptor = training_data_path :: descriptor
113 descriptor = model_save_path :: descriptor
114 descriptor = maxIter :: descriptor
115 descriptor = docConcentration :: descriptor
116 descriptor = topicConcentration :: descriptor
117 descriptor = checkpointInterval :: descriptor
118 descriptor = optimizer :: descriptor
119 descriptor = k :: descriptor
120 descriptor
121 }
122
123 override def getIcon(): Array[Byte] = {
124 ImageUtil.getImage("icon/ml_clustering/LDATraining.png")
125 }
126
127 override def getGroup(): List[String] = {
128 List(StopGroup.MLGroup.toString)
129 }
130
131}