Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/PhoneNumberClean.scala

1package cn.piflow.bundle.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.SparkSession
9import org.apache.spark.sql.types.StructField
10
11class PhoneNumberClean extends ConfigurableStop{
12
13  val authorEmail: String = "06whuxx@163.com"
14  val description: String = "Cleaning data in mobile number format"
15  val inportList: List[String] = List(Port.DefaultPort)
16  val outportList: List[String] = List(Port.DefaultPort)
17
18  var columnName:String=_
19
20  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21    val spark = pec.get[SparkSession]()
22    val sqlContext=spark.sqlContext
23    val dfOld = in.read()
24    dfOld.createOrReplaceTempView("thesis")
25    sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processPhonenum(str))
26    val structFields: Array[String] = dfOld.schema.fieldNames
27    val columnNames = columnName.split(",").toSet
28    val sqlNewFieldStr = new StringBuilder
29    columnNames.foreach(c=>{
30      if (columnNames.contains(c)) {
31        sqlNewFieldStr ++= ",regexPro("
32        sqlNewFieldStr ++= c
33        sqlNewFieldStr ++= ") as "
34        sqlNewFieldStr ++= c
35        sqlNewFieldStr ++= "_new "
36      }
37    })
38
39    val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
40
41    val dfNew=sqlContext.sql(sqlText)
42    dfNew.createOrReplaceTempView("thesis")
43    val schemaStr = new StringBuilder
44    structFields.foreach(field => {
45      schemaStr ++= field
46      if (columnNames.contains(field)) {
47        schemaStr ++= "_new as "
48        schemaStr ++= field
49      }
50      schemaStr ++= ","
51    })
52    val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
53    val dfNew1=sqlContext.sql(sqlTextNew)
54
55    out.write(dfNew1)
56  }
57
58
59
60  def initialize(ctx: ProcessContext): Unit = {
61
62  }
63
64
65  def setProperties(map: Map[String, Any]): Unit = {
66    columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
67
68  }
69
70  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
71    var descriptor : List[PropertyDescriptor] = List()
72    val columnName = new PropertyDescriptor()
73      .name("columnName")
74      .displayName("Column_Name")
75      .description("Column names are what you want to clean,multiple column names are separated by commas")
76      .defaultValue("")
77      .required(true)
78      .example("phonenumber")
79    descriptor = columnName :: descriptor
80    descriptor
81  }
82
83  override def getIcon(): Array[Byte] = {
84    ImageUtil.getImage("icon/clean/PhoneNumberClean.png")
85  }
86
87  override def getGroup(): List[String] = {
88    List(StopGroup.CleanGroup)
89  }
90
91}