upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/TitleClean.scala
1package cn.piflow.bundle.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.SparkSession
9import org.apache.spark.sql.types.StructField
10
11class TitleClean extends ConfigurableStop{
12 val authorEmail: String = "06whuxx@163.com"
13 val description: String = "Cleaning title data"
14 val inportList: List[String] = List(Port.DefaultPort)
15 val outportList: List[String] = List(Port.DefaultPort)
16
17 var columnName:String=_
18
19 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20 val spark = pec.get[SparkSession]()
21 val sqlContext=spark.sqlContext
22 val dfOld = in.read()
23 dfOld.createOrReplaceTempView("thesis")
24 sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processTitle(str))
25 val structFields: Array[String] = dfOld.schema.fieldNames
26 val columnNames = columnName.split(",").toSet
27 val sqlNewFieldStr = new StringBuilder
28 columnNames.foreach(c=>{
29 if (columnNames.contains(c)) {
30 sqlNewFieldStr ++= ",regexPro("
31 sqlNewFieldStr ++= c
32 sqlNewFieldStr ++= ") as "
33 sqlNewFieldStr ++= c
34 sqlNewFieldStr ++= "_new "
35 }
36 })
37
38 val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
39
40 val dfNew=sqlContext.sql(sqlText)
41 dfNew.createOrReplaceTempView("thesis")
42 val schemaStr = new StringBuilder
43 structFields.foreach(field => {
44 schemaStr ++= field
45 if (columnNames.contains(field)) {
46 schemaStr ++= "_new as "
47 schemaStr ++= field
48 }
49 schemaStr ++= ","
50 })
51 val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
52 val dfNew1=sqlContext.sql(sqlTextNew)
53
54 out.write(dfNew1)
55 }
56
57 def initialize(ctx: ProcessContext): Unit = {
58
59 }
60
61
62 def setProperties(map: Map[String, Any]): Unit = {
63 columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
64 }
65
66 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
67 var descriptor : List[PropertyDescriptor] = List()
68
69 val columnName = new PropertyDescriptor()
70 .name("columnName")
71 .displayName("Column_Name")
72 .description("Column names are what you want to clean,multiple column names are separated by commas")
73 .defaultValue("")
74 .required(true)
75 .example("title")
76 descriptor = columnName :: descriptor
77
78 descriptor
79 }
80
81 override def getIcon(): Array[Byte] = {
82 ImageUtil.getImage("icon/clean/TitleClean.png")
83 }
84
85 override def getGroup(): List[String] = {
86 List(StopGroup.CleanGroup)
87 }
88
89}