upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/util/NSFCUtil.scala
1package cn.piflow.bundle.util
2
3import java.text.SimpleDateFormat
4import java.util.{Date, UUID}
5
6import org.apache.spark.sql.Row
7import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType}
8
9object NSFCUtil {
10 val dateFormat: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
11 val add_field = Array(
12 StructField("0", StringType, nullable = true),
13 StructField("identity_card", StringType, nullable = true),
14 StructField("military_id", StringType, nullable = true),
15 StructField("passport", StringType, nullable = true),
16 StructField("4", StringType, nullable = true),
17 StructField("home_return_permit", StringType, nullable = true),
18 StructField("mainland_travel_permit_for_taiwan_residents", StringType, nullable = true)
19 )
20 def buildNewOPersonRow (beforeRowSeq : Seq[Any], beforeSchema:StructType, afterSchema:StructType, idTypeField : String, idField :String, source:String): Row = {
21 var afterSeq = scala.collection.mutable.ArraySeq[Any]()
22 var afterMap = scala.collection.mutable.HashMap[String, Any]()
23
24 var card_type: String = ""
25 var card_code: String = ""
26
27 var zero = "null"
28 var one = "null"
29 var two = "null"
30 var three = "null"
31 var four = "null"
32 var five = "null"
33 var six = "null"
34 //Seq[V] -> Map[K,V]
35 for (index <- 0 until beforeSchema.length) {
36 val name = beforeSchema(index).name
37 name match {
38 case `idTypeField` => if (beforeRowSeq(index) == null)
39 card_type = "null" else card_type = String.valueOf(beforeRowSeq(index))
40 case `idField` => if (beforeRowSeq(index) == null)
41 card_code = "null" else card_code = String.valueOf(beforeRowSeq(index))
42 case _ => afterMap.put(beforeSchema(index).name, beforeRowSeq(index))
43 }
44 }
45 card_type match {
46 case id_type.MAINLAND.id => six = card_code //6
47 case id_type.FOUR.id => four = card_code //4
48 case id_type.HOMERETURN.id => five = card_code //5
49 case id_type.MILITARY.id => two = card_code //2
50 case id_type.IDENTITY_CARD.id => one = card_code //1
51 case id_type.PASSPORT.id => three = card_code //3
52 case id_type.ZERO.id => zero = card_code //0
53 case _ =>
54 }
55 afterMap.put("0", ifNUll(zero))
56 afterMap.put("identity_card", ifNUll(one))
57 afterMap.put("military_id", ifNUll(two))
58 afterMap.put("passport", ifNUll(three))
59 afterMap.put("4", ifNUll(four))
60 afterMap.put("home_return_permit", ifNUll(five))
61 afterMap.put("mainland_travel_permit_for_taiwan_residents", ifNUll(six))
62 afterMap.put("source", source) // 加入source字段
63 afterMap.put("uuid", UUID.randomUUID().toString)
64 afterMap.put("id_hash", (card_code + card_type).##.toString)
65 for (index <- 0 until afterSchema.length) {
66 if (!afterMap.keySet.contains(afterSchema(index).name)) {
67 afterMap.put(afterSchema(index).name, null)
68 }
69 }
70 for (field <- afterSchema) {
71 afterSeq = afterSeq.:+(afterMap(field.name))
72 }
73
74 Row.fromSeq(afterSeq)
75 }
76
77 def ifNUll(value : String): Any = {
78 if (value.equals("null")) null else value
79 }
80
81 def mkPersonSchemaWithID(beforeSchema : StructType, idTypeField : String, idField :String) : StructType ={
82 var afterSchema = new StructType(beforeSchema.filter(field => {field.name != idField && field.name != idTypeField}).toArray)
83 add_field.foreach(f => {
84 afterSchema = afterSchema.add(f.name, f.dataType,nullable = true)
85 })
86 afterSchema = afterSchema.add("source", StringType, nullable = true)
87 afterSchema = afterSchema.add("uuid", StringType, nullable = true)
88 afterSchema.add("id_hash", StringType, nullable = true)
89 }
90
91
92 def getTime(row : Row, timeIndex : Int) : Date = {
93 if (row.isNullAt(timeIndex)) new Date(0)
94 else {
95 var date:java.util.Date = null
96 val s = row.getString(timeIndex)
97 try {
98 date = dateFormat.parse(s)
99 } catch {
100 case _ => date = new Date(0)
101 }
102 date
103 }
104 }
105 def mkRowKey(schema_result:StructType, row: Row, key : String): String = {
106 var hasNull = false
107 var s = ""
108 if (key.contains("&")) {
109 val sl = key.split("&")
110 sl.foreach(s_ => {
111 val index = schema_result.fieldIndex(s_)
112 if (!row.isNullAt(index)) {
113 s += row.getAs[String](index)
114 } else {
115 hasNull = true
116 }
117 })
118 } else {
119 val index = schema_result.fieldIndex(key)
120 if (!row.isNullAt(index)) {
121 s = row.getAs[String](index)
122 } else {
123 hasNull = true
124 }
125 }
126 if (hasNull) {
127 s = generateUUID() // key 为空则生产uuid
128 }
129 s
130 }
131 def generateUUID(): String = {
132 UUID.randomUUID().toString
133 }
134}
135object id_type extends Enumeration {
136 val MAINLAND: id = id("6", "mainland_travel_permit_for_taiwan_residents")
137 val IDENTITY_CARD: id = id("1", "identity_card")
138 val MILITARY: id = id("2", "military_id")
139 val PASSPORT: id = id("3", "passport")
140 val HOMERETURN: id = id("5", "home_return_permit")
141 val ZERO: id = id("0", "ZERO")
142 val FOUR: id = id("4", "FOUR")
143 type id_type = id
144}
145case class id(id :String, name :String) extends Serializable {
146 override def equals(obj: scala.Any): Boolean = {
147 if (this == obj) return true
148 obj match {
149 case anotherId: id => this.id == anotherId.id
150 case _ => false
151 }
152 }
153}