upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParserColumns.scala
1package cn.piflow.bundle.xml
2
3import cn.piflow._
4import cn.piflow.bundle.util.XmlToJson
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11
12class XmlParserColumns extends ConfigurableStop {
13
14 val authorEmail: String = "ygang@cnic.cn"
15 val description: String = "Parse xml data in columns in upstream data"
16 val inportList: List[String] = List(Port.DefaultPort)
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var xmlColumns:String = _
20
21 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
22
23 val spark = pec.get[SparkSession]()
24
25 val df = in.read()
26
27 spark.sqlContext.udf.register("xmlToJson",(str:String)=>{
28 XmlToJson.xmlParse(str.replaceAll("\n","\t"))
29 })
30 val columns: Array[String] = xmlColumns.toLowerCase.split(",").map(x => x.trim)
31
32 val fields: Array[String] = df.schema.fieldNames
33 var fieldString = new StringBuilder
34 fields.foreach(x=>{
35 if (columns.contains(x.toLowerCase)){
36 fieldString.append(s"xmlToJson(${x}) as ${x} ,")
37 } else {
38 fieldString.append(s"${x},")
39 }
40 })
41
42 df.createOrReplaceTempView("temp")
43 val sqlText = "select " +fieldString.stripSuffix(",")+ " from temp"
44 val frame: DataFrame = spark.sql(sqlText)
45
46 val rdd: RDD[String] = frame.toJSON.rdd.map(x => {
47 x.toString().replace("\\n", "").replace("}\"", "}").replace(":\"{", ":{").replace("\\","")
48 })
49
50 val outDF: DataFrame = spark.read.json(rdd)
51 outDF.printSchema()
52
53 out.write(outDF)
54
55 }
56
57 def initialize(ctx: ProcessContext): Unit = {
58
59 }
60
61 def setProperties(map : Map[String, Any]) = {
62 xmlColumns = MapUtil.get(map,"xmlColumns").asInstanceOf[String]
63 }
64
65 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
66 var descriptor : List[PropertyDescriptor] = List()
67 val xmlColumns = new PropertyDescriptor()
68 .name("xmlColumns")
69 .displayName("xmlColumns")
70 .description("Parsed column names containing xml,Multiple columns separated by commas")
71 .defaultValue("")
72 .required(true)
73 .example("product_xml")
74 descriptor = xmlColumns :: descriptor
75 descriptor
76 }
77
78 override def getIcon(): Array[Byte] = {
79 ImageUtil.getImage("icon/xml/XmlParser.png")
80 }
81
82 override def getGroup(): List[String] = {
83 List(StopGroup.XmlGroup)
84 }
85
86
87
88}