upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/clean/CardCodeClean.scala
1package cn.piflow.bundle.nsfc.clean
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.conf.bean.PropertyDescriptor
5import cn.piflow.conf.util.{ImageUtil, MapUtil}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import org.apache.spark.sql.SparkSession
9
10class CardCodeClean extends ConfigurableStop {
11 val authorEmail: String = "songdongze@cnic.cn"
12 val description: String = "Clean Id Card data."
13 val inportList: List[String] = List(Port.DefaultPort.toString)
14 val outportList: List[String] = List(Port.DefaultPort.toString)
15 //var regex:String =_
16 var cardCodeField:String=_
17 var cardTypeField:String=_
18 //var replaceStr:String=_
19
20 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21 val spark = pec.get[SparkSession]()
22 val sqlContext=spark.sqlContext
23 val dfOld = in.read()
24 dfOld.createOrReplaceTempView("thesis")
25 sqlContext.udf.register("regexPro",(str:String)=>CleanUtil.processCardCode(str))
26 val structFields: Array[String] = dfOld.schema.fieldNames
27 val columnNames = cardCodeField.split(",").map(x => x.trim).toSet
28 val sqlNewFieldStr = new StringBuilder
29
30
31 val sqlText:String="select *, if("+ cardTypeField +"='1',regexPro(" + cardCodeField + "),"+cardCodeField+") as " + cardCodeField + "_new" + " from thesis"
32
33 val dfNew=sqlContext.sql(sqlText)
34 dfNew.createOrReplaceTempView("thesis")
35 val schemaStr = new StringBuilder
36 structFields.foreach(field => {
37 schemaStr ++= field
38 if (cardCodeField.equals(field)) {
39 schemaStr ++= "_new as "
40 schemaStr ++= field
41 }
42 schemaStr ++= ","
43 })
44 val sqlText1:String = "select " + schemaStr.substring(0,schemaStr.length -1) + " from thesis"
45 val dfNew1=sqlContext.sql(sqlText1)
46
47 //dfNew.show()
48 out.write(dfNew1)
49 }
50
51
52 def initialize(ctx: ProcessContext): Unit = {
53
54 }
55
56
57 def setProperties(map: Map[String, Any]): Unit = {
58 cardTypeField=MapUtil.get(map,key="cardTypeField").asInstanceOf[String]
59 cardCodeField=MapUtil.get(map,key="cardCodeField").asInstanceOf[String]
60 }
61
62 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
63 var descriptor : List[PropertyDescriptor] = List()
64 val cardTypeField = new PropertyDescriptor().name("cardTypeField").displayName("CARD TYPE FIELD NAME").description("The card type for card code ").defaultValue("card_type").required(true)
65 descriptor = cardTypeField :: descriptor
66 val cardCodeField = new PropertyDescriptor().name("cardCodeField").displayName("CARD CODE FIELD NAME").description("The card code you want to clean").defaultValue("card_code").required(true)
67 descriptor = cardCodeField :: descriptor
68 descriptor
69 }
70
71 override def getIcon(): Array[Byte] = {
72 ImageUtil.getImage("icon/clean/IdentityNumberClean.png")
73 }
74
75 override def getGroup(): List[String] = {
76 List(StopGroup.NSFC.toString)
77 }
78}