Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_classification/NaiveBayesTraining.scala

1package cn.piflow.bundle.ml_classification
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.ml.classification.NaiveBayes
8import org.apache.spark.sql.SparkSession
9
10class NaiveBayesTraining extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "Train a NaiveBayes model"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var training_data_path:String =_
16  var smoothing_value:String=_
17  var model_save_path:String=_
18
19
20  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21    val spark = pec.get[SparkSession]()
22
23    //load data stored in libsvm format as a dataframe
24    val data=spark.read.format("libsvm").load(training_data_path)
25
26    //get smoothing factor
27    var smoothing_factor:Double=0
28    if(smoothing_value!=""){
29      smoothing_factor=smoothing_value.toDouble
30    }
31
32    //training a NaiveBayes model
33    val model=new NaiveBayes().setSmoothing(smoothing_factor).fit(data)
34
35    //model persistence
36    model.save(model_save_path)
37
38    import spark.implicits._
39    val dfOut=Seq(model_save_path).toDF
40    dfOut.show()
41    out.write(dfOut)
42
43  }
44
45  def initialize(ctx: ProcessContext): Unit = {
46
47  }
48
49
50  def setProperties(map: Map[String, Any]): Unit = {
51    training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
52    smoothing_value=MapUtil.get(map,key="smoothing_value").asInstanceOf[String]
53    model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
54  }
55
56  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
57    var descriptor : List[PropertyDescriptor] = List()
58    val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
59    val smoothing_value = new PropertyDescriptor().name("smoothing_value").displayName("SMOOTHING_FACTOR").defaultValue("0").required(false)
60    val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").defaultValue("").required(true)
61    descriptor = training_data_path :: descriptor
62    descriptor = smoothing_value :: descriptor
63    descriptor = model_save_path :: descriptor
64    descriptor
65  }
66
67  override def getIcon(): Array[Byte] = {
68    ImageUtil.getImage("icon/ml_classification/NavieBayesTraining.png")
69  }
70
71  override def getGroup(): List[String] = {
72    List(StopGroup.MLGroup.toString)
73  }
74
75}