Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_classification/GBTTraining.scala

1package cn.piflow.bundle.ml_classification
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.classification.GBTClassifier
8import org.apache.spark.sql.SparkSession
9
10class GBTTraining extends ConfigurableStop{
11  val authorEmail: String = "06whuxx@163.com"
12  val description: String = "Train a GBT Model"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15  var training_data_path:String =_
16  var model_save_path:String=_
17  var maxBins:String=_
18  var maxDepth:String=_
19  var minInfoGain:String=_
20  var minInstancesPerNode:String=_
21  var impurity:String=_
22  var subSamplingRate:String=_
23  var lossType:String=_
24  var stepSize:String=_
25
26  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
27    val spark = pec.get[SparkSession]()
28
29    //load data stored in libsvm format as a dataframe
30    val data=spark.read.format("libsvm").load(training_data_path)
31
32    //Maximum number of bins used for discretizing continuous features and for choosing how to split on features at each node. More bins give higher granularity.Must be >= 2 and >= number of categories in any categorical feature.
33    var maxBinsValue:Int=40
34    if(maxBins!=""){
35      maxBinsValue=maxBins.toInt
36    }
37
38    //Maximum depth of the tree (>= 0).The maximum is 30.
39    var maxDepthValue:Int=30
40    if(maxDepth!=""){
41      maxDepthValue=maxDepth.toInt
42    }
43
44    //Minimum information gain for a split to be considered at a tree node.
45    var minInfoGainValue:Double=0.2
46    if(minInfoGain!=""){
47      minInfoGainValue=minInfoGain.toDouble
48    }
49
50    //Minimum number of instances each child must have after split.
51    var minInstancesPerNodeValue:Int=3
52    if(minInstancesPerNode!=""){
53      minInstancesPerNodeValue=minInstancesPerNode.toInt
54    }
55
56    //Param for the name of family which is a description of the label distribution to be used in the model
57    var impurityValue="gini"
58    if(impurity!=""){
59      impurityValue=impurity
60    }
61
62    var subSamplingRateValue:Double=0.6
63    if(subSamplingRate!=""){
64      subSamplingRateValue=subSamplingRate.toDouble
65    }
66
67    var lossTypeValue="logistic"
68    if(lossType!=""){
69      lossTypeValue=lossType
70    }
71
72    var stepSizeValue:Double=0.1
73    if(stepSize!=""){
74      stepSizeValue=stepSize.toDouble
75    }
76
77    //training a GBT model
78    val model=new GBTClassifier()
79      .setMaxBins(maxBinsValue)
80      .setMaxDepth(maxDepthValue)
81      .setMinInfoGain(minInfoGainValue)
82      .setMinInstancesPerNode(minInstancesPerNodeValue)
83      .setImpurity(impurityValue)
84      .setLossType(lossTypeValue)
85      .setSubsamplingRate(subSamplingRateValue)
86      .setStepSize(stepSizeValue)
87      .fit(data)
88
89    //model persistence
90    model.save(model_save_path)
91
92    import spark.implicits._
93    val dfOut=Seq(model_save_path).toDF
94    dfOut.show()
95    out.write(dfOut)
96
97  }
98
99  def initialize(ctx: ProcessContext): Unit = {
100
101  }
102
103
104  def setProperties(map: Map[String, Any]): Unit = {
105    training_data_path=MapUtil.get(map,key="training_data_path").asInstanceOf[String]
106    model_save_path=MapUtil.get(map,key="model_save_path").asInstanceOf[String]
107    maxBins=MapUtil.get(map,key="maxBins").asInstanceOf[String]
108    maxDepth=MapUtil.get(map,key="maxDepth").asInstanceOf[String]
109    minInfoGain=MapUtil.get(map,key="minInfoGain").asInstanceOf[String]
110    minInstancesPerNode=MapUtil.get(map,key="minInstancesPerNode").asInstanceOf[String]
111    impurity=MapUtil.get(map,key="impurity").asInstanceOf[String]
112    subSamplingRate=MapUtil.get(map,key="subSamplingRate").asInstanceOf[String]
113    lossType=MapUtil.get(map,key="lossType").asInstanceOf[String]
114    stepSize=MapUtil.get(map,key="stepSize").asInstanceOf[String]
115  }
116
117  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
118    var descriptor : List[PropertyDescriptor] = List()
119    val training_data_path = new PropertyDescriptor().name("training_data_path").displayName("TRAINING_DATA_PATH").defaultValue("").required(true)
120    val model_save_path = new PropertyDescriptor().name("model_save_path").displayName("MODEL_SAVE_PATH").description("").defaultValue("").required(true)
121    val maxBins=new PropertyDescriptor().name("maxBins").displayName("MAX_BINS").description("Maximum number of bins used for discretizing continuous features and for choosing how to split on features at each node.").defaultValue("").required(false)
122    val maxDepth=new PropertyDescriptor().name("maxDepth").displayName("MAX_DEPTH").description("Maximum depth of the tree").defaultValue("").required(false)
123    val minInfoGain=new PropertyDescriptor().name("minInfoGain").displayName("MIN_INFO_GAIN").description("Minimum information gain for a split to be considered at a tree node").defaultValue("").required(false)
124    val minInstancesPerNode=new PropertyDescriptor().name("minInstancesPerNode").displayName("MIN_INSTANCES_PER_NODE").description("Minimum number of instances each child must have after split.").defaultValue("").required(false)
125    val impurity=new PropertyDescriptor().name("impurity").displayName("IMPURITY").description("Criterion used for information gain calculation (case-insensitive). Supported: \"entropy\" and \"gini\". (default = gini)").defaultValue("").required(false)
126    val subSamplingRate=new PropertyDescriptor().name("subSamplingRate").displayName("SUB_SAMPLING_RATE").description("Fraction of the training data used for learning each decision tree, in range (0, 1].").defaultValue("").required(false)
127    val lossType=new PropertyDescriptor().name("lossType").displayName("LOSS_TYPE").description("Loss function which GBT tries to minimize. (case-insensitive) Supported: \"logistic\" (default = logistic)").defaultValue("").required(false)
128    val stepSize=new PropertyDescriptor().name("stepSize").displayName("STEP_SIZE").description("Param for Step size (a.k.a. learning rate) in interval (0, 1] for shrinking the contribution of each estimator. (default = 0.1)").defaultValue("").required(false)
129    descriptor = training_data_path :: descriptor
130    descriptor = model_save_path :: descriptor
131    descriptor = maxBins :: descriptor
132    descriptor = maxDepth :: descriptor
133    descriptor = minInfoGain :: descriptor
134    descriptor = minInstancesPerNode :: descriptor
135    descriptor = impurity :: descriptor
136    descriptor = subSamplingRate::descriptor
137    descriptor = lossType :: descriptor
138    descriptor = stepSize :: descriptor
139    descriptor
140  }
141
142  override def getIcon(): Array[Byte] = {
143    ImageUtil.getImage("icon/ml_classification/GBTTraining.png")
144  }
145
146  override def getGroup(): List[String] = {
147    List(StopGroup.MLGroup.toString)
148  }
149
150}