Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/clean/TitleClean.scala

1package cn.piflow.bundle.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.SparkSession
9import org.apache.spark.sql.types.StructField
10
11class TitleClean extends ConfigurableStop{
12  val authorEmail: String = "06whuxx@163.com"
13  val description: String = "Cleaning title data"
14  val inportList: List[String] = List(Port.DefaultPort)
15  val outportList: List[String] = List(Port.DefaultPort)
16
17  var columnName:String=_
18
19  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20    val spark = pec.get[SparkSession]()
21    val sqlContext=spark.sqlContext
22    val dfOld = in.read()
23    dfOld.createOrReplaceTempView("thesis")
24    sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processTitle(str))
25    val structFields: Array[String] = dfOld.schema.fieldNames
26    val columnNames = columnName.split(",").toSet
27    val sqlNewFieldStr = new StringBuilder
28    columnNames.foreach(c=>{
29      if (columnNames.contains(c)) {
30        sqlNewFieldStr ++= ",regexPro("
31        sqlNewFieldStr ++= c
32        sqlNewFieldStr ++= ") as "
33        sqlNewFieldStr ++= c
34        sqlNewFieldStr ++= "_new "
35      }
36    })
37
38    val sqlText:String="select * " + sqlNewFieldStr + " from thesis"
39
40    val dfNew=sqlContext.sql(sqlText)
41    dfNew.createOrReplaceTempView("thesis")
42    val schemaStr = new StringBuilder
43    structFields.foreach(field => {
44      schemaStr ++= field
45      if (columnNames.contains(field)) {
46        schemaStr ++= "_new as "
47        schemaStr ++= field
48      }
49      schemaStr ++= ","
50    })
51    val sqlTextNew:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
52    val dfNew1=sqlContext.sql(sqlTextNew)
53
54    out.write(dfNew1)
55  }
56
57  def initialize(ctx: ProcessContext): Unit = {
58
59  }
60
61
62  def setProperties(map: Map[String, Any]): Unit = {
63    columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
64  }
65
66  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
67    var descriptor : List[PropertyDescriptor] = List()
68
69    val columnName = new PropertyDescriptor()
70      .name("columnName")
71      .displayName("Column_Name")
72      .description("Column names are what you want to clean,multiple column names are separated by commas")
73      .defaultValue("")
74      .required(true)
75      .example("title")
76    descriptor = columnName :: descriptor
77
78    descriptor
79  }
80
81  override def getIcon(): Array[Byte] = {
82    ImageUtil.getImage("icon/clean/TitleClean.png")
83  }
84
85  override def getGroup(): List[String] = {
86    List(StopGroup.CleanGroup)
87  }
88
89}