Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParserColumns.scala

1package cn.piflow.bundle.xml
2
3import cn.piflow._
4import cn.piflow.bundle.util.XmlToJson
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11
12class XmlParserColumns extends ConfigurableStop {
13
14  val authorEmail: String = "ygang@cnic.cn"
15  val description: String = "Parse xml data in columns in upstream data"
16  val inportList: List[String] = List(Port.DefaultPort)
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var xmlColumns:String = _
20
21  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
22
23    val spark = pec.get[SparkSession]()
24
25    val df = in.read()
26
27    spark.sqlContext.udf.register("xmlToJson",(str:String)=>{
28      XmlToJson.xmlParse(str.replaceAll("\n","\t"))
29    })
30    val columns: Array[String] = xmlColumns.toLowerCase.split(",").map(x => x.trim)
31
32    val fields: Array[String] = df.schema.fieldNames
33    var fieldString = new StringBuilder
34    fields.foreach(x=>{
35      if (columns.contains(x.toLowerCase)){
36        fieldString.append(s"xmlToJson(${x}) as ${x} ,")
37      } else {
38        fieldString.append(s"${x},")
39      }
40    })
41
42    df.createOrReplaceTempView("temp")
43    val sqlText = "select " +fieldString.stripSuffix(",")+ " from temp"
44    val frame: DataFrame = spark.sql(sqlText)
45
46    val rdd: RDD[String] = frame.toJSON.rdd.map(x => {
47      x.toString().replace("\\n", "").replace("}\"", "}").replace(":\"{", ":{").replace("\\","")
48    })
49
50    val outDF: DataFrame = spark.read.json(rdd)
51    outDF.printSchema()
52
53    out.write(outDF)
54
55  }
56
57  def initialize(ctx: ProcessContext): Unit = {
58
59  }
60
61  def setProperties(map : Map[String, Any]) = {
62    xmlColumns = MapUtil.get(map,"xmlColumns").asInstanceOf[String]
63  }
64
65  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
66    var descriptor : List[PropertyDescriptor] = List()
67    val xmlColumns = new PropertyDescriptor()
68      .name("xmlColumns")
69      .displayName("xmlColumns")
70      .description("Parsed column names containing xml,Multiple columns separated by commas")
71      .defaultValue("")
72      .required(true)
73        .example("product_xml")
74    descriptor = xmlColumns :: descriptor
75    descriptor
76  }
77
78  override def getIcon(): Array[Byte] = {
79    ImageUtil.getImage("icon/xml/XmlParser.png")
80  }
81
82  override def getGroup(): List[String] = {
83    List(StopGroup.XmlGroup)
84  }
85
86
87
88}