Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_feature/WordToVec.scala

1package cn.piflow.bundle.ml_feature
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.feature.Word2Vec
8import org.apache.spark.ml.feature.Word2VecModel
9import org.apache.spark.sql.SparkSession
10
11class WordToVec extends ConfigurableStop{
12  val authorEmail: String = "06whuxx@163.com"
13  val description: String = "Transfer word to vector"
14  val inportList: List[String] = List(Port.DefaultPort.toString)
15  val outportList: List[String] = List(Port.DefaultPort.toString)
16  var maxIter:String=_
17  var maxSentenceLength:String=_
18  var minCount:String=_
19  var numPartitions:String=_
20  var stepSize:String=_
21  var vectorSize:String=_
22  var colName:String=_
23  var outputCol:String=_
24
25  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
26    val spark = pec.get[SparkSession]()
27    val sqlContext=spark.sqlContext
28    val df=in.read()
29    df.createOrReplaceTempView("doc")
30    sqlContext.udf.register("split",(str:String)=>str.split(" "))
31    val sqlText:String="select split("+colName+") as "+colName+"_new from doc"
32    val dfNew=sqlContext.sql(sqlText)
33    dfNew.show()
34
35    //Param for maximum number of iterations (>= 0)
36    var maxIterValue:Int=50
37    if(maxIter!=""){
38      maxIterValue=maxIter.toInt
39    }
40
41    //Sets the maximum length (in words) of each sentence in the input data. Any sentence longer than this threshold will be divided into chunks of up to maxSentenceLength size. Default: 1000
42    var maxSentenceLengthValue:Int=1000
43    if(maxSentenceLength!=""){
44      maxSentenceLengthValue=maxSentenceLength.toInt
45    }
46
47    //The minimum number of times a token must appear to be included in the word2vec model's vocabulary. Default: 5
48    var minCountValue:Int=1
49    if(minCount!=""){
50      minCountValue=minCount.toInt
51    }
52
53    var numPartitionsValue:Int=1
54    if(numPartitions!=""){
55      numPartitionsValue=numPartitions.toInt
56    }
57
58    //Param for Step size to be used for each iteration of optimization (> 0).
59    var stepSizeValue:Int=5
60    if(stepSize!=""){
61      stepSizeValue=stepSize.toInt
62    }
63
64    //The dimension of the code that you want to transform from words. Default: 100
65    var vectorSizeValue:Int=5
66    if(vectorSize!=""){
67      vectorSizeValue=vectorSize.toInt
68    }
69
70    //clustering with kmeans algorithm
71    val word2vec=new Word2Vec()
72        .setMaxIter(maxIterValue)
73        .setMaxSentenceLength(maxSentenceLengthValue)
74        .setMinCount(minCountValue)
75        .setNumPartitions(numPartitionsValue)
76        .setStepSize(stepSizeValue)
77        .setVectorSize(vectorSizeValue)
78        .setInputCol(colName+"_new")
79        .setOutputCol(outputCol)
80        .fit(dfNew)
81
82    import spark.implicits._
83    val dfOut=word2vec.transform(dfNew)
84    dfOut.show()
85    out.write(dfOut)
86
87  }
88
89  def initialize(ctx: ProcessContext): Unit = {
90
91  }
92
93
94  def setProperties(map: Map[String, Any]): Unit = {
95    maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
96    vectorSize=MapUtil.get(map,key="vectorSize").toString
97    maxSentenceLength=MapUtil.get(map,key="maxSentenceLength").toString
98    minCount=MapUtil.get(map,key="minCount").toString
99    numPartitions=MapUtil.get(map,key="numPartitions").toString
100    stepSize=MapUtil.get(map,key="stepSize").toString
101    colName=MapUtil.get(map,key="colName").asInstanceOf[String]
102    outputCol=MapUtil.get(map,key="outputCol").asInstanceOf[String]
103  }
104
105  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
106    var descriptor : List[PropertyDescriptor] = List()
107    val vectorSize = new PropertyDescriptor().name("vectorSize").displayName("VECTOR_SIZE").description("The dimension of the code that you want to transform from words. Default: 100").defaultValue("").required(false)
108    val maxSentenceLength = new PropertyDescriptor().name("maxSentenceLength").displayName("MAX_SENTENCE_LENGTH").description("Sets the maximum length (in words) of each sentence in the input data. Any sentence longer than this threshold will be divided into chunks of up to maxSentenceLength size. Default: 1000").defaultValue("").required(false)
109    val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0)").defaultValue("").required(false)
110    val minCount=new PropertyDescriptor().name("minCount").displayName("MIN_COUNT").description("The minimum number of times a token must appear to be included in the word2vec model's vocabulary. Default: 5").defaultValue("").required(false)
111    val stepSize=new PropertyDescriptor().name("stepSize").displayName("STEP_SIZE").defaultValue("").required(false)
112    val numPartitions=new PropertyDescriptor().name("numPartitions").displayName("NUM_PARTITIONS").description("Param for Step size to be used for each iteration of optimization (> 0).").defaultValue("").required(false)
113    val colName=new PropertyDescriptor().name("colName").displayName("INPUT_COL").description("").defaultValue("").required(true)
114    val outputCol=new PropertyDescriptor().name("outputCol").displayName("OUTPUT_COL").description("").defaultValue("").required(true)
115    descriptor = vectorSize :: descriptor
116    descriptor = maxSentenceLength :: descriptor
117    descriptor = maxIter :: descriptor
118    descriptor = minCount :: descriptor
119    descriptor = stepSize :: descriptor
120    descriptor = numPartitions :: descriptor
121    descriptor = colName :: descriptor
122    descriptor = outputCol :: descriptor
123    descriptor
124  }
125
126  override def getIcon(): Array[Byte] = {
127    ImageUtil.getImage("icon/ml_feature/WordToVec.png")
128  }
129
130  override def getGroup(): List[String] = {
131    List(StopGroup.MLGroup.toString)
132  }
133}