Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/clean/CardCodeClean.scala

1package cn.piflow.bundle.nsfc.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.conf.bean.PropertyDescriptor
5import cn.piflow.conf.util.{ImageUtil, MapUtil}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import org.apache.spark.sql.SparkSession
9
10class CardCodeClean extends ConfigurableStop {
11    val authorEmail: String = "songdongze@cnic.cn"
12    val description: String = "Clean Id Card data."
13    val inportList: List[String] = List(Port.DefaultPort.toString)
14    val outportList: List[String] = List(Port.DefaultPort.toString)
15    //var regex:String =_
16    var cardCodeField:String=_
17    var cardTypeField:String=_
18    //var replaceStr:String=_
19
20    def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21        val spark = pec.get[SparkSession]()
22        val sqlContext=spark.sqlContext
23        val dfOld = in.read()
24        dfOld.createOrReplaceTempView("thesis")
25        sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processCardCode(str))
26        val structFields: Array[String] = dfOld.schema.fieldNames
27        val columnNames = cardCodeField.split(",").map(x => x.trim).toSet
28        val sqlNewFieldStr = new StringBuilder
29
30
31        val sqlText:String="select *, if("+ cardTypeField +"='1',regexPro(" + cardCodeField + "),"+cardCodeField+") as " + cardCodeField + "_new"  + " from thesis"
32
33        val dfNew=sqlContext.sql(sqlText)
34        dfNew.createOrReplaceTempView("thesis")
35        val schemaStr = new StringBuilder
36        structFields.foreach(field => {
37                schemaStr ++= field
38        if (cardCodeField.equals(field)) {
39            schemaStr ++= "_new as "
40            schemaStr ++= field
41        }
42        schemaStr ++= ","
43    })
44        val sqlText1:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
45        val dfNew1=sqlContext.sql(sqlText1)
46
47        //dfNew.show()
48        out.write(dfNew1)
49    }
50
51
52    def initialize(ctx: ProcessContext): Unit = {
53
54    }
55
56
57    def setProperties(map: Map[String, Any]): Unit = {
58        cardTypeField=MapUtil.get(map,key="cardTypeField").asInstanceOf[String]
59        cardCodeField=MapUtil.get(map,key="cardCodeField").asInstanceOf[String]
60    }
61
62    override def getPropertyDescriptor(): List[PropertyDescriptor] = {
63        var descriptor : List[PropertyDescriptor] = List()
64        val cardTypeField = new PropertyDescriptor().name("cardTypeField").displayName("CARD TYPE FIELD NAME").description("The card type for card code ").defaultValue("card_type").required(true)
65        descriptor = cardTypeField :: descriptor
66        val cardCodeField = new PropertyDescriptor().name("cardCodeField").displayName("CARD CODE FIELD NAME").description("The card code  you want to clean").defaultValue("card_code").required(true)
67        descriptor = cardCodeField :: descriptor
68        descriptor
69    }
70
71    override def getIcon(): Array[Byte] = {
72        ImageUtil.getImage("icon/clean/IdentityNumberClean.png")
73    }
74
75    override def getGroup(): List[String] = {
76        List(StopGroup.NSFC.toString)
77    }
78}