Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_clustering/LDAPrediction.scala

1package cn.piflow.bundle.ml_clustering
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.clustering.{DistributedLDAModel, LDAModel, LocalLDAModel}
8import org.apache.spark.sql.SparkSession
9
10class LDAPrediction extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "Use an existing LDAModel to predict"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var test_data_path:String =_
16  var model_path:String=_
17
18
19  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20    val spark = pec.get[SparkSession]()
21    //load data stored in libsvm format as a dataframe
22    val data=spark.read.format("libsvm").load(test_data_path)
23    //data.show()
24
25    //load model
26    val model=LocalLDAModel.load(model_path)
27
28    val predictions=model.transform(data)
29    predictions.show()
30    out.write(predictions)
31
32  }
33
34  def initialize(ctx: ProcessContext): Unit = {
35
36  }
37
38
39  def setProperties(map: Map[String, Any]): Unit = {
40    test_data_path=MapUtil.get(map,key="test_data_path").asInstanceOf[String]
41    model_path=MapUtil.get(map,key="model_path").asInstanceOf[String]
42  }
43
44  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
45    var descriptor : List[PropertyDescriptor] = List()
46    val test_data_path = new PropertyDescriptor().name("test_data_path").displayName("TEST_DATA_PATH").defaultValue("").required(true)
47    val model_path = new PropertyDescriptor().name("model_path").displayName("MODEL_PATH").defaultValue("").required(true)
48    descriptor = test_data_path :: descriptor
49    descriptor = model_path :: descriptor
50    descriptor
51  }
52
53  override def getIcon(): Array[Byte] = {
54    ImageUtil.getImage("icon/ml_clustering/LDAPrediction.png")
55  }
56
57  override def getGroup(): List[String] = {
58    List(StopGroup.MLGroup.toString)
59  }
60
61}