Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/util/NSFCUtil.scala

1package cn.piflow.bundle.util
2
3import java.text.SimpleDateFormat
4import java.util.{Date, UUID}
5
6import org.apache.spark.sql.Row
7import org.apache.spark.sql.types.{LongType, StringType, StructField, StructType}
8
9object NSFCUtil {
10  val dateFormat: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
11  val add_field = Array(
12    StructField("0", StringType, nullable = true),
13    StructField("identity_card", StringType, nullable = true),
14    StructField("military_id", StringType, nullable = true),
15    StructField("passport", StringType, nullable = true),
16    StructField("4", StringType, nullable = true),
17    StructField("home_return_permit", StringType, nullable = true),
18    StructField("mainland_travel_permit_for_taiwan_residents", StringType, nullable = true)
19  )
20  def buildNewOPersonRow (beforeRowSeq : Seq[Any], beforeSchema:StructType, afterSchema:StructType, idTypeField : String, idField :String, source:String): Row = {
21    var afterSeq = scala.collection.mutable.ArraySeq[Any]()
22    var afterMap = scala.collection.mutable.HashMap[String, Any]()
23
24    var card_type: String = ""
25    var card_code: String = ""
26
27    var zero = "null"
28    var one = "null"
29    var two = "null"
30    var three = "null"
31    var four = "null"
32    var five = "null"
33    var six = "null"
34    //Seq[V] -> Map[K,V]
35    for (index <- 0 until beforeSchema.length) {
36      val name = beforeSchema(index).name
37      name match {
38        case `idTypeField` => if (beforeRowSeq(index) == null)
39          card_type = "null" else card_type = String.valueOf(beforeRowSeq(index))
40        case `idField` => if (beforeRowSeq(index) == null)
41          card_code = "null" else card_code = String.valueOf(beforeRowSeq(index))
42        case _ => afterMap.put(beforeSchema(index).name, beforeRowSeq(index))
43      }
44    }
45    card_type match {
46      case id_type.MAINLAND.id => six = card_code //6
47      case id_type.FOUR.id => four = card_code //4
48      case id_type.HOMERETURN.id => five = card_code //5
49      case id_type.MILITARY.id => two = card_code //2
50      case id_type.IDENTITY_CARD.id => one = card_code //1
51      case id_type.PASSPORT.id => three = card_code //3
52      case id_type.ZERO.id => zero = card_code //0
53      case _ =>
54    }
55    afterMap.put("0", ifNUll(zero))
56    afterMap.put("identity_card", ifNUll(one))
57    afterMap.put("military_id", ifNUll(two))
58    afterMap.put("passport", ifNUll(three))
59    afterMap.put("4", ifNUll(four))
60    afterMap.put("home_return_permit", ifNUll(five))
61    afterMap.put("mainland_travel_permit_for_taiwan_residents", ifNUll(six))
62    afterMap.put("source", source) // 加入source字段
63    afterMap.put("uuid", UUID.randomUUID().toString)
64    afterMap.put("id_hash", (card_code + card_type).##.toString)
65    for (index <- 0 until afterSchema.length) {
66      if (!afterMap.keySet.contains(afterSchema(index).name)) {
67        afterMap.put(afterSchema(index).name, null)
68      }
69    }
70    for (field <- afterSchema) {
71      afterSeq = afterSeq.:+(afterMap(field.name))
72    }
73
74    Row.fromSeq(afterSeq)
75  }
76
77  def ifNUll(value : String): Any = {
78    if (value.equals("null")) null else value
79  }
80
81  def mkPersonSchemaWithID(beforeSchema : StructType, idTypeField : String, idField :String) : StructType ={
82    var afterSchema = new StructType(beforeSchema.filter(field => {field.name != idField && field.name != idTypeField}).toArray)
83    add_field.foreach(f => {
84      afterSchema = afterSchema.add(f.name, f.dataType,nullable = true)
85    })
86    afterSchema = afterSchema.add("source", StringType, nullable = true)
87    afterSchema = afterSchema.add("uuid", StringType, nullable = true)
88    afterSchema.add("id_hash", StringType, nullable = true)
89  }
90
91
92  def getTime(row : Row, timeIndex : Int) : Date = {
93    if (row.isNullAt(timeIndex)) new Date(0)
94    else {
95      var date:java.util.Date = null
96      val s = row.getString(timeIndex)
97      try {
98        date = dateFormat.parse(s)
99      } catch {
100        case _ => date = new Date(0)
101      }
102      date
103    }
104  }
105  def mkRowKey(schema_result:StructType, row: Row, key : String): String = {
106    var hasNull = false
107    var s = ""
108    if (key.contains("&")) {
109      val sl = key.split("&")
110      sl.foreach(s_ => {
111        val index = schema_result.fieldIndex(s_)
112        if (!row.isNullAt(index)) {
113          s += row.getAs[String](index)
114        } else {
115          hasNull = true
116        }
117      })
118    } else {
119      val index = schema_result.fieldIndex(key)
120      if (!row.isNullAt(index)) {
121        s = row.getAs[String](index)
122      } else {
123        hasNull = true
124      }
125    }
126    if (hasNull) {
127      s = generateUUID() // key 为空则生产uuid
128    }
129    s
130  }
131  def generateUUID(): String = {
132    UUID.randomUUID().toString
133  }
134}
135object id_type extends Enumeration {
136  val MAINLAND: id = id("6", "mainland_travel_permit_for_taiwan_residents")
137  val IDENTITY_CARD: id = id("1", "identity_card")
138  val MILITARY: id = id("2", "military_id")
139  val PASSPORT: id = id("3", "passport")
140  val HOMERETURN: id = id("5", "home_return_permit")
141  val ZERO: id = id("0", "ZERO")
142  val FOUR: id = id("4", "FOUR")
143  type id_type = id
144}
145case class id(id :String, name :String) extends Serializable {
146  override def equals(obj: scala.Any): Boolean = {
147    if (this == obj) return true
148    obj match {
149      case anotherId: id => this.id == anotherId.id
150      case _ => false
151    }
152  }
153}