Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/distinct/PersonIdExpansion.scala

1package cn.piflow.bundle.nsfc.distinct
2
3import cn.piflow.bundle.util.NSFCUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.{Row, SparkSession}
10
11/**
12  * expand the id from id:idtype to id:typea;_:typeb
13  * */
14class PersonIdExpansion extends ConfigurableStop{
15  override val authorEmail: String = "xiaomeng7890@gmail.com"
16  override val description: String = "nothing to say bro"
17  override val inportList: List[String] = List(Port.DefaultPort.toString)
18  override val outportList: List[String] = List(Port.DefaultPort.toString)
19
20  var expandIDFieldName : String = _
21  var expandIDTypeName : String = _
22  var useSchemaTable : Boolean = _
23  var schemaTable : String = _
24  var sourceTable :String = _
25
26  override def setProperties(map: Map[String, Any]): Unit = {
27    expandIDFieldName = MapUtil.get(map,"expandIDField").asInstanceOf[String]
28    expandIDTypeName = MapUtil.get(map,"expandIDType").asInstanceOf[String]
29    schemaTable = MapUtil.get(map,"schemaTableName").asInstanceOf[String]
30    useSchemaTable = MapUtil.get(map,"usedTableSchema").asInstanceOf[Boolean]
31    sourceTable = MapUtil.get(map,"source").asInstanceOf[String]
32  }
33
34  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
35    var descriptor : List[PropertyDescriptor] = List()
36
37    val expandIDFieldName = new PropertyDescriptor().
38      name("expandIDField").
39      displayName("expand ID field").
40      description("the id field you need to expand in your dataFrame").
41      defaultValue("card_code").
42      required(true)
43    descriptor = expandIDFieldName :: descriptor
44
45    val expandIDTypeName = new PropertyDescriptor().
46      name("expandIDType").
47      displayName("expand ID type").
48      description("the id type field you need to expand in your dataFrame").
49      defaultValue("card_type").
50      required(true)
51    descriptor = expandIDTypeName :: descriptor
52
53    val useSchemaTable = new PropertyDescriptor().
54      name("usedTableSchema").
55      displayName("is table schema used").
56      description("if you try to use another table to be the after schema of this DataFrame").
57      defaultValue("false").
58      required(true).
59      allowableValues(Set("true", "false"))
60
61    descriptor = useSchemaTable :: descriptor
62
63    val schemaTable = new PropertyDescriptor().
64      name("schemaTableName").
65      displayName("the table which you can get the schema").
66      description("is usedTableSchema is true, this should be a hive table name").
67      defaultValue("middle.m_person").
68      required(true)
69    descriptor = schemaTable :: descriptor
70
71    val source = new PropertyDescriptor().
72      name("source").
73      displayName("source").
74      description("source table name").
75      defaultValue("temp.t_person").
76      required(true)
77    descriptor = source :: descriptor
78
79    descriptor
80  }
81
82  override def getIcon(): Array[Byte] = ImageUtil.getImage("icon/hive/SelectHiveQL.png")
83
84  override def getGroup(): List[String] =  List(StopGroup.NSFC.toString, "sha0w", "distinct")
85
86  override def initialize(ctx: ProcessContext): Unit = {}
87
88  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
89    if (in.isEmpty()) throw new Exception("in DF is indispensable")
90    val sc: SparkSession = pec.get[SparkSession]()
91    val inDF = in.read()
92    val beforeSchema = inDF.schema
93    val afterSchema = if (useSchemaTable) {
94      sc.read.table(schemaTable).schema
95    } else {
96      NSFCUtil.mkPersonSchemaWithID(beforeSchema, expandIDFieldName, expandIDTypeName)
97    }
98    val outRow: RDD[Row] = inDF.rdd.map(r => {
99      r.toSeq
100    }).map(seq => NSFCUtil.buildNewOPersonRow(seq, beforeSchema, afterSchema, expandIDTypeName, expandIDFieldName, sourceTable))
101    out.write(sc.createDataFrame(outRow, afterSchema))
102  }
103}