upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_classification/NaiveBayesTraining.scala
1package cn.piflow.bundle.ml_classification
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.ml.classification.NaiveBayes
8import org.apache.spark.sql.SparkSession
9
10class NaiveBayesTraining extends ConfigurableStop{
11 val authorEmail: String = "06whuxx@163.com"
12 val description: String = "Train a NaiveBayes model"
13 val inportList: List[String] = List(Port.DefaultPort)
14 val outportList: List[String] = List(Port.DefaultPort)
15 var training_data_path:String =_
16 var smoothing_value:String=_
17 var model_save_path:String=_
18
19
20 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21 val spark = pec.get[SparkSession]()
22
23 //load data stored in libsvm format as a dataframe
24 val data=spark.read.format("libsvm").load(training_data_path)
25
26 //get smoothing factor
27 var smoothing_factor:Double=0
28 if(smoothing_value!=""){
29 smoothing_factor=smoothing_value.toDouble
30 }
31
32 //training a NaiveBayes model
33 val model=new NaiveBayes().setSmoothing(smoothing_factor).fit(data)
34
35 //model persistence
36 model.save(model_save_path)
37
38 import spark.implicits._
39 val dfOut=Seq(model_save_path).toDF
40 dfOut.show()
41 out.write(dfOut)
42
43 }
44
45 def initialize(ctx: ProcessContext): Unit = {
46
47 }
48
49
50 def setProperties(map: Map[String, Any]): Unit = {
51 training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
52 smoothing_value=MapUtil.get(map,key="smoothing_value").asInstanceOf[String]
53 model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
54 }
55
56 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
57 var descriptor : List[PropertyDescriptor] = List()
58 val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
59 val smoothing_value = new PropertyDescriptor().name("smoothing_value").displayName("SMOOTHING_FACTOR").defaultValue("0").required(false)
60 val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").defaultValue("").required(true)
61 descriptor = training_data_path :: descriptor
62 descriptor = smoothing_value :: descriptor
63 descriptor = model_save_path :: descriptor
64 descriptor
65 }
66
67 override def getIcon(): Array[Byte] = {
68 ImageUtil.getImage("icon/ml_classification/NavieBayesTraining.png")
69 }
70
71 override def getGroup(): List[String] = {
72 List(StopGroup.MLGroup.toString)
73 }
74
75}