upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/ml_feature/WordToVec.scala
1package cn.piflow.bundle.ml_feature
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.ml.feature.Word2Vec
8import org.apache.spark.ml.feature.Word2VecModel
9import org.apache.spark.sql.SparkSession
10
11class WordToVec extends ConfigurableStop{
12 val authorEmail: String = "06whuxx@163.com"
13 val description: String = "Transfer word to vector"
14 val inportList: List[String] = List(Port.DefaultPort.toString)
15 val outportList: List[String] = List(Port.DefaultPort.toString)
16 var maxIter:String=_
17 var maxSentenceLength:String=_
18 var minCount:String=_
19 var numPartitions:String=_
20 var stepSize:String=_
21 var vectorSize:String=_
22 var colName:String=_
23 var outputCol:String=_
24
25 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
26 val spark = pec.get[SparkSession]()
27 val sqlContext=spark.sqlContext
28 val df=in.read()
29 df.createOrReplaceTempView("doc")
30 sqlContext.udf.register("split",(str:String)=>str.split(" "))
31 val sqlText:String="select split("+colName+") as "+colName+"_new from doc"
32 val dfNew=sqlContext.sql(sqlText)
33 dfNew.show()
34
35 //Param for maximum number of iterations (>= 0)
36 var maxIterValue:Int=50
37 if(maxIter!=""){
38 maxIterValue=maxIter.toInt
39 }
40
41 //Sets the maximum length (in words) of each sentence in the input data. Any sentence longer than this threshold will be divided into chunks of up to maxSentenceLength size. Default: 1000
42 var maxSentenceLengthValue:Int=1000
43 if(maxSentenceLength!=""){
44 maxSentenceLengthValue=maxSentenceLength.toInt
45 }
46
47 //The minimum number of times a token must appear to be included in the word2vec model's vocabulary. Default: 5
48 var minCountValue:Int=1
49 if(minCount!=""){
50 minCountValue=minCount.toInt
51 }
52
53 var numPartitionsValue:Int=1
54 if(numPartitions!=""){
55 numPartitionsValue=numPartitions.toInt
56 }
57
58 //Param for Step size to be used for each iteration of optimization (> 0).
59 var stepSizeValue:Int=5
60 if(stepSize!=""){
61 stepSizeValue=stepSize.toInt
62 }
63
64 //The dimension of the code that you want to transform from words. Default: 100
65 var vectorSizeValue:Int=5
66 if(vectorSize!=""){
67 vectorSizeValue=vectorSize.toInt
68 }
69
70 //clustering with kmeans algorithm
71 val word2vec=new Word2Vec()
72 .setMaxIter(maxIterValue)
73 .setMaxSentenceLength(maxSentenceLengthValue)
74 .setMinCount(minCountValue)
75 .setNumPartitions(numPartitionsValue)
76 .setStepSize(stepSizeValue)
77 .setVectorSize(vectorSizeValue)
78 .setInputCol(colName+"_new")
79 .setOutputCol(outputCol)
80 .fit(dfNew)
81
82 import spark.implicits._
83 val dfOut=word2vec.transform(dfNew)
84 dfOut.show()
85 out.write(dfOut)
86
87 }
88
89 def initialize(ctx: ProcessContext): Unit = {
90
91 }
92
93
94 def setProperties(map: Map[String, Any]): Unit = {
95 maxIter=MapUtil.get(map,key="maxIter").asInstanceOf[String]
96 vectorSize=MapUtil.get(map,key="vectorSize").toString
97 maxSentenceLength=MapUtil.get(map,key="maxSentenceLength").toString
98 minCount=MapUtil.get(map,key="minCount").toString
99 numPartitions=MapUtil.get(map,key="numPartitions").toString
100 stepSize=MapUtil.get(map,key="stepSize").toString
101 colName=MapUtil.get(map,key="colName").asInstanceOf[String]
102 outputCol=MapUtil.get(map,key="outputCol").asInstanceOf[String]
103 }
104
105 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
106 var descriptor : List[PropertyDescriptor] = List()
107 val vectorSize = new PropertyDescriptor().name("vectorSize").displayName("VECTOR_SIZE").description("The dimension of the code that you want to transform from words. Default: 100").defaultValue("").required(false)
108 val maxSentenceLength = new PropertyDescriptor().name("maxSentenceLength").displayName("MAX_SENTENCE_LENGTH").description("Sets the maximum length (in words) of each sentence in the input data. Any sentence longer than this threshold will be divided into chunks of up to maxSentenceLength size. Default: 1000").defaultValue("").required(false)
109 val maxIter=new PropertyDescriptor().name("maxIter").displayName("MAX_ITER").description("Param for maximum number of iterations (>= 0)").defaultValue("").required(false)
110 val minCount=new PropertyDescriptor().name("minCount").displayName("MIN_COUNT").description("The minimum number of times a token must appear to be included in the word2vec model's vocabulary. Default: 5").defaultValue("").required(false)
111 val stepSize=new PropertyDescriptor().name("stepSize").displayName("STEP_SIZE").defaultValue("").required(false)
112 val numPartitions=new PropertyDescriptor().name("numPartitions").displayName("NUM_PARTITIONS").description("Param for Step size to be used for each iteration of optimization (> 0).").defaultValue("").required(false)
113 val colName=new PropertyDescriptor().name("colName").displayName("INPUT_COL").description("").defaultValue("").required(true)
114 val outputCol=new PropertyDescriptor().name("outputCol").displayName("OUTPUT_COL").description("").defaultValue("").required(true)
115 descriptor = vectorSize :: descriptor
116 descriptor = maxSentenceLength :: descriptor
117 descriptor = maxIter :: descriptor
118 descriptor = minCount :: descriptor
119 descriptor = stepSize :: descriptor
120 descriptor = numPartitions :: descriptor
121 descriptor = colName :: descriptor
122 descriptor = outputCol :: descriptor
123 descriptor
124 }
125
126 override def getIcon(): Array[Byte] = {
127 ImageUtil.getImage("icon/ml_feature/WordToVec.png")
128 }
129
130 override def getGroup(): List[String] = {
131 List(StopGroup.MLGroup.toString)
132 }
133}