Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/ProvinceClean.scala

1package cn.piflow.bundle.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import org.apache.spark.sql.SparkSession
9
10class ProvinceClean extends ConfigurableStop{
11  val authorEmail: String = "songdongze@cnic.cn"
12  val description: String = "Cleaning province data"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15
16  var columnName:String=_
17
18  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
19    val spark = pec.get[SparkSession]()
20    val sqlContext=spark.sqlContext
21    val dfOld = in.read()
22    dfOld.createOrReplaceTempView("thesis")
23    sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processProvince(str))
24    val structFields: Array[String] = dfOld.schema.fieldNames
25    val columnNames = columnName.split(",").toSet
26    val sqlNewFieldStr = new StringBuilder
27    columnNames.foreach(c=>{
28      if (columnNames.contains(c)) {
29        sqlNewFieldStr ++= ",regexPro("
30        sqlNewFieldStr ++= c
31        sqlNewFieldStr ++= ") as "
32        sqlNewFieldStr ++= c
33        sqlNewFieldStr ++= "_new "
34      }
35    })
36
37    val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
38
39    val dfNew=sqlContext.sql(sqlText)
40    dfNew.createOrReplaceTempView("thesis")
41    val schemaStr = new StringBuilder
42    structFields.foreach(field => {
43      schemaStr ++= field
44      if (columnNames.contains(field)) {
45        schemaStr ++= "_new as "
46        schemaStr ++= field
47      }
48      schemaStr ++= ","
49    })
50    val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
51    val dfNew1=sqlContext.sql(sqlTextNew)
52
53    out.write(dfNew1)
54  }
55
56
57  def initialize(ctx: ProcessContext): Unit = {
58
59  }
60
61
62  def setProperties(map: Map[String, Any]): Unit = {
63    columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
64
65  }
66
67  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
68    var descriptor : List[PropertyDescriptor] = List()
69    val columnName = new PropertyDescriptor()
70      .name("columnName")
71      .displayName("Column_Name")
72      .description("Column names are what you want to clean,multiple column names are separated by commas")
73      .defaultValue("")
74      .required(true)
75      .example("province")
76    descriptor = columnName :: descriptor
77
78    descriptor
79  }
80
81  override def getIcon(): Array[Byte] = {
82    ImageUtil.getImage("icon/clean/ProvinceClean.png")
83
84  }
85
86  override def getGroup(): List[String] = {
87    List(StopGroup.CleanGroup)
88  }
89
90}