upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/PhoneNumberClean.scala
1package cn.piflow.bundle.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.SparkSession
9import org.apache.spark.sql.types.StructField
10
11class PhoneNumberClean extends ConfigurableStop{
12
13 val authorEmail: String = "06whuxx@163.com"
14 val description: String = "Cleaning data in mobile number format"
15 val inportList: List[String] = List(Port.DefaultPort)
16 val outportList: List[String] = List(Port.DefaultPort)
17
18 var columnName:String=_
19
20 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21 val spark = pec.get[SparkSession]()
22 val sqlContext=spark.sqlContext
23 val dfOld = in.read()
24 dfOld.createOrReplaceTempView("thesis")
25 sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processPhonenum(str))
26 val structFields: Array[String] = dfOld.schema.fieldNames
27 val columnNames = columnName.split(",").toSet
28 val sqlNewFieldStr = new StringBuilder
29 columnNames.foreach(c=>{
30 if (columnNames.contains(c)) {
31 sqlNewFieldStr ++= ",regexPro("
32 sqlNewFieldStr ++= c
33 sqlNewFieldStr ++= ") as "
34 sqlNewFieldStr ++= c
35 sqlNewFieldStr ++= "_new "
36 }
37 })
38
39 val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
40
41 val dfNew=sqlContext.sql(sqlText)
42 dfNew.createOrReplaceTempView("thesis")
43 val schemaStr = new StringBuilder
44 structFields.foreach(field => {
45 schemaStr ++= field
46 if (columnNames.contains(field)) {
47 schemaStr ++= "_new as "
48 schemaStr ++= field
49 }
50 schemaStr ++= ","
51 })
52 val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
53 val dfNew1=sqlContext.sql(sqlTextNew)
54
55 out.write(dfNew1)
56 }
57
58
59
60 def initialize(ctx: ProcessContext): Unit = {
61
62 }
63
64
65 def setProperties(map: Map[String, Any]): Unit = {
66 columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
67
68 }
69
70 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
71 var descriptor : List[PropertyDescriptor] = List()
72 val columnName = new PropertyDescriptor()
73 .name("columnName")
74 .displayName("Column_Name")
75 .description("Column names are what you want to clean,multiple column names are separated by commas")
76 .defaultValue("")
77 .required(true)
78 .example("phonenumber")
79 descriptor = columnName :: descriptor
80 descriptor
81 }
82
83 override def getIcon(): Array[Byte] = {
84 ImageUtil.getImage("icon/clean/PhoneNumberClean.png")
85 }
86
87 override def getGroup(): List[String] = {
88 List(StopGroup.CleanGroup)
89 }
90
91}