Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/EmailClean.scala

1package cn.piflow.bundle.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.SparkSession
9
10class EmailClean extends ConfigurableStop{
11
12  val authorEmail: String = "songdongze@cnic.cn"
13  val description: String = "Cleaning data in email format"
14  val inportList: List[String] = List(Port.DefaultPort)
15  val outportList: List[String] = List(Port.DefaultPort)
16
17  var columnName:String= _
18
19  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20    val spark = pec.get[SparkSession]()
21    val sqlContext=spark.sqlContext
22    val dfOld = in.read()
23    dfOld.createOrReplaceTempView("thesis")
24    sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processEmail(str))
25    val structFields: Array[String] = dfOld.schema.fieldNames
26    val columnNames = columnName.split(",").toSet
27    val sqlNewFieldStr = new StringBuilder
28    columnNames.foreach(c=>{
29      if (columnNames.contains(c)) {
30        sqlNewFieldStr ++= ",regexPro("
31        sqlNewFieldStr ++= c
32        sqlNewFieldStr ++= ") as "
33        sqlNewFieldStr ++= c
34        sqlNewFieldStr ++= "_new "
35      }
36    })
37
38    val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
39
40    val dfNew=sqlContext.sql(sqlText)
41    dfNew.createOrReplaceTempView("thesis")
42    val schemaStr = new StringBuilder
43    structFields.foreach(field => {
44      schemaStr ++= field
45      if (columnNames.contains(field)) {
46        schemaStr ++= "_new as "
47        schemaStr ++= field
48      }
49      schemaStr ++= ","
50    })
51    val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
52    val dfNew1=sqlContext.sql(sqlTextNew)
53
54    out.write(dfNew1)
55  }
56
57  def initialize(ctx: ProcessContext): Unit = {
58
59  }
60
61
62  def setProperties(map: Map[String, Any]): Unit = {
63    columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
64
65  }
66
67  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
68    var descriptor : List[PropertyDescriptor] = List()
69    val columnName = new PropertyDescriptor()
70      .name("columnName")
71      .displayName("Column Name")
72      .description("Column names are what you want to clean,multiple column names are separated by commas")
73      .defaultValue("")
74      .required(true)
75      .example("email")
76
77    descriptor = columnName :: descriptor
78    descriptor
79  }
80
81  override def getIcon(): Array[Byte] = {
82    ImageUtil.getImage("icon/clean/EmailClean.png")
83
84  }
85
86  override def getGroup(): List[String] = {
87    List(StopGroup.CleanGroup)
88  }
89
90}