Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_classification/DecisionTreeTraining.scala

1package cn.piflow.bundle.ml_classification
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.ml.classification.DecisionTreeClassifier
8import org.apache.spark.sql.SparkSession
9
10class DecisionTreeTraining extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "Train a decision tree model"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var training_data_path:String =_
16  var model_save_path:String=_
17  var maxBins:String=_
18  var maxDepth:String=_
19  var minInfoGain:String=_
20  var minInstancesPerNode:String=_
21  var impurity:String=_
22
23  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24    val spark = pec.get[SparkSession]()
25
26    //load data stored in libsvm format as a dataframe
27    val data=spark.read.format("libsvm").load(training_data_path)
28
29    //Maximum number of bins used for discretizing continuous features and for choosing how to split on features at each node. More bins give higher granularity.Must be >= 2 and >= number of categories in any categorical feature.
30    var maxBinsValue:Int=40
31    if(maxBins!=""){
32      maxBinsValue=maxBins.toInt
33    }
34
35    //Maximum depth of the tree (>= 0).The maximum is 30.
36    var maxDepthValue:Int=30
37    if(maxDepth!=""){
38      maxDepthValue=maxDepth.toInt
39    }
40
41    //Minimum information gain for a split to be considered at a tree node.
42    var minInfoGainValue:Double=0.2
43    if(minInfoGain!=""){
44      minInfoGainValue=minInfoGain.toDouble
45    }
46
47    //Minimum number of instances each child must have after split.
48    var minInstancesPerNodeValue:Int=3
49    if(minInstancesPerNode!=""){
50      minInstancesPerNodeValue=minInstancesPerNode.toInt
51    }
52
53    //Param for the name of family which is a description of the label distribution to be used in the model
54    var impurityValue="gini"
55    if(impurity!=""){
56      impurityValue=impurity
57    }
58
59    //training a Decision Tree model
60    val model=new DecisionTreeClassifier()
61      .setMaxBins(maxBinsValue)
62      .setMaxDepth(maxDepthValue)
63      .setMinInfoGain(minInfoGainValue)
64      .setMinInstancesPerNode(minInstancesPerNodeValue)
65      .setImpurity(impurityValue)
66      .fit(data)
67
68    //model persistence
69    model.save(model_save_path)
70
71    import spark.implicits._
72    val dfOut=Seq(model_save_path).toDF
73    dfOut.show()
74    out.write(dfOut)
75
76  }
77
78  def initialize(ctx: ProcessContext): Unit = {
79
80  }
81
82
83  def setProperties(map: Map[String, Any]): Unit = {
84    training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
85    model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
86    maxBins=MapUtil.get(map,key="maxBins").asInstanceOf[String]
87    maxDepth=MapUtil.get(map,key="maxDepth").asInstanceOf[String]
88    minInfoGain=MapUtil.get(map,key="minInfoGain").asInstanceOf[String]
89    minInstancesPerNode=MapUtil.get(map,key="minInstancesPerNode").asInstanceOf[String]
90    impurity=MapUtil.get(map,key="impurity").asInstanceOf[String]
91
92  }
93
94  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
95    var descriptor : List[PropertyDescriptor] = List()
96    val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
97    val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("ddd").defaultValue("").required(true)
98    val maxBins=new PropertyDescriptor().name("maxBins").displayName("MAX_BINS").description("Maximum number of bins used for discretizing continuous features and for choosing how to split on features at each node.").defaultValue("").required(true)
99    val maxDepth=new PropertyDescriptor().name("maxDepth").displayName("MAX_DEPTH").description("Maximum depth of the tree").defaultValue("").required(true)
100    val minInfoGain=new PropertyDescriptor().name("minInfoGain").displayName("MIN_INFO_GAIN").description("Minimum information gain for a split to be considered at a tree node").defaultValue("").required(true)
101    val minInstancesPerNode=new PropertyDescriptor().name("minInstancesPerNode").displayName("MIN_INSTANCES_PER_NODE").description("Minimum number of instances each child must have after split.").defaultValue("").required(true)
102    val impurity=new PropertyDescriptor().name("impurity").displayName("IMPURITY").description("Criterion used for information gain calculation (case-insensitive). Supported: \"entropy\" and \"gini\". (default = gini)").defaultValue("").required(true)
103    descriptor = training_data_path :: descriptor
104    descriptor = model_save_path :: descriptor
105    descriptor = maxBins :: descriptor
106    descriptor = maxDepth :: descriptor
107    descriptor = minInfoGain :: descriptor
108    descriptor = minInstancesPerNode :: descriptor
109    descriptor = impurity :: descriptor
110    descriptor
111  }
112
113  override def getIcon(): Array[Byte] = {
114    ImageUtil.getImage("icon/ml_classification/DecisionTreeTraining.png")
115  }
116
117  override def getGroup(): List[String] = {
118    List(StopGroup.MLGroup.toString)
119  }
120
121}