upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/nlp/WordSpliter.scala
1package cn.piflow.bundle.nlp
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import com.huaban.analysis.jieba.JiebaSegmenter.SegMode
8import com.huaban.analysis.jieba._
9import org.apache.spark.rdd.RDD
10import org.apache.spark.sql.types.{StringType, StructField, StructType}
11import org.apache.spark.sql.{DataFrame, Row, SparkSession}
12
13import scala.collection.JavaConverters._
14import scala.collection.mutable.ArrayBuffer
15
16class WordSpliter extends ConfigurableStop {
17
18 val authorEmail: String = "huchuan0901@163.com"
19 val description: String = "Word segmentation"
20 val inportList: List[String] = List(Port.AnyPort.toString)
21 val outportList: List[String] = List(Port.DefaultPort.toString)
22
23 var path:String = _
24
25
26 val jiebaSegmenter = new JiebaSegmenter()
27 var tokenARR:ArrayBuffer[String]=ArrayBuffer()
28
29 def segmenter(str:String): Unit ={
30
31 var strVar = str
32 //delete symbol
33 strVar = strVar.replaceAll( "[\\p{P}+~$`^=|<>~`$^+=|<>¥×+\\s]" , "");
34
35 val tokens = jiebaSegmenter.process(strVar,SegMode.SEARCH).asScala
36
37 for (token: SegToken <- tokens){
38
39 tokenARR += token.word
40
41 }
42 }
43
44 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
45
46 val session: SparkSession = pec.get[SparkSession]()
47
48 //read
49 val strDF = session.read.text(path)
50
51 //segmenter
52 segmenter(strDF.head().getString(0))
53
54 //write df
55 val rows: List[Row] = tokenARR.map(each => {
56 var arr:Array[String]=Array(each)
57 val row: Row = Row.fromSeq(arr)
58 row
59 }).toList
60 val rowRDD: RDD[Row] = session.sparkContext.makeRDD(rows)
61 val schema: StructType = StructType(Array(
62 StructField("words",StringType)
63 ))
64 val df: DataFrame = session.createDataFrame(rowRDD,schema)
65
66 out.write(df)
67 }
68
69 def initialize(ctx: ProcessContext): Unit = {
70
71 }
72
73 def setProperties(map : Map[String, Any]) = {
74 path = MapUtil.get(map,"path").asInstanceOf[String]
75 }
76
77 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
78 var descriptor : List[PropertyDescriptor] = List()
79 val path = new PropertyDescriptor().name("path").displayName("path").description("The path of text file").defaultValue("").required(true)
80 descriptor = path :: descriptor
81 descriptor
82 }
83
84 override def getIcon(): Array[Byte] = {
85 ImageUtil.getImage("icon/nlp/NLP.png")
86 }
87
88 override def getGroup(): List[String] = {
89 List(StopGroup.Alg_NLPGroup.toString)
90 }
91
92}