Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/IdentityNumberClean.scala

1package cn.piflow.bundle.clean
2
3
4import cn.piflow.bundle.util.CleanUtil
5import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
6import cn.piflow.conf._
7import cn.piflow.conf.bean.PropertyDescriptor
8import cn.piflow.conf.util.{ImageUtil, MapUtil}
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11
12
13class IdentityNumberClean extends ConfigurableStop{
14
15  val authorEmail: String = "06whuxx@163.com"
16  val description: String = "Cleaning data in ID Card format"
17  val inportList: List[String] = List(Port.DefaultPort)
18  val outportList: List[String] = List(Port.DefaultPort)
19
20  var columnName:String=_
21
22  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23    val spark = pec.get[SparkSession]()
24    val sqlContext=spark.sqlContext
25    val dfOld = in.read()
26    dfOld.createOrReplaceTempView("thesis")
27    sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processCardCode(str))
28    val structFields: Array[String] = dfOld.schema.fieldNames
29    val columnNames = columnName.split(",").toSet
30    val sqlNewFieldStr = new StringBuilder
31    columnNames.foreach(c=>{
32      if (columnNames.contains(c)) {
33        sqlNewFieldStr ++= ",regexPro("
34        sqlNewFieldStr ++= c
35        sqlNewFieldStr ++= ") as "
36        sqlNewFieldStr ++= c
37        sqlNewFieldStr ++= "_new "
38      }
39    })
40
41    val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
42
43    val dfNew=sqlContext.sql(sqlText)
44    dfNew.createOrReplaceTempView("thesis")
45    val schemaStr = new StringBuilder
46    structFields.foreach(field => {
47      schemaStr ++= field
48      if (columnNames.contains(field)) {
49        schemaStr ++= "_new as "
50        schemaStr ++= field
51      }
52      schemaStr ++= ","
53    })
54    val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
55    val dfNew1=sqlContext.sql(sqlTextNew)
56
57    out.write(dfNew1)
58  }
59
60
61  def initialize(ctx: ProcessContext): Unit = {
62
63  }
64
65
66  def setProperties(map: Map[String, Any]): Unit = {
67    columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
68  }
69
70  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
71    var descriptor : List[PropertyDescriptor] = List()
72    val columnName = new PropertyDescriptor()
73      .name("columnName")
74      .displayName("Column_Name")
75      .description("Column names are what you want to clean,multiple column names are separated by commas")
76      .defaultValue("")
77      .required(true)
78      .example("IdCard")
79
80    descriptor = columnName :: descriptor
81    descriptor
82  }
83
84  override def getIcon(): Array[Byte] = {
85    ImageUtil.getImage("icon/clean/IdentityNumberClean.png")
86  }
87
88  override def getGroup(): List[String] = {
89    List(StopGroup.CleanGroup)
90  }
91}