upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/FlattenXmlParser.scala
1package cn.piflow.bundle.xml
2
3import java.net.URI
4import java.util
5
6import cn.piflow.bundle.util.JsonUtil
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
9import cn.piflow.conf.bean.PropertyDescriptor
10import cn.piflow.conf.util.{ImageUtil, MapUtil}
11import org.apache.hadoop.conf.Configuration
12import org.apache.hadoop.fs.{FSDataInputStream, FileSystem, Path}
13import org.apache.spark.rdd.RDD
14import org.apache.spark.sql.types.{StringType, StructField, StructType}
15import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
16import org.dom4j.io.SAXReader
17import org.dom4j.{Document, DocumentHelper, Element}
18
19import scala.collection.JavaConverters._
20import scala.collection.mutable.{ArrayBuffer, ListBuffer}
21
22
23class FlattenXmlParser extends ConfigurableStop{
24 override val authorEmail: String = "yangqidong@cnic.cn"
25 val inportList: List[String] = List(PortEnum.NonePort.toString)
26 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
27 override val description: String = "Parse the XML file and expand the label you need"
28
29
30 var xmlpath:String = _
31 var tagPath:String = _
32 var openTag: String = _
33 var returnField: String = _
34
35
36 def javaIterToScalaIter(utilIter: util.Iterator[_]):Iterator[Element]={
37 utilIter.asInstanceOf[util.Iterator[Element]].asScala
38 }
39
40 var ss: SparkSession =_
41 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
42
43
44
45 ss=pec.get[SparkSession]()
46 val conf = new Configuration()
47 val fs: FileSystem = FileSystem.get(URI.create(xmlpath), conf)
48 val hdfsInputStream: FSDataInputStream = fs.open(new Path(xmlpath))
49 val saxReader: SAXReader = new SAXReader()
50 val document: Document = saxReader.read(hdfsInputStream)
51 val rootElt: Element = document.getRootElement
52
53 var finalDF:DataFrame=null
54
55 var arrbuffer:ArrayBuffer[Element]=ArrayBuffer()
56 arrbuffer+=rootElt
57
58 val arrLabel: Array[String] = tagPath.split(",")
59 for(x<-(1 until arrLabel.length)){
60 var ele: Element =arrbuffer(x-1).element(arrLabel(x))
61 arrbuffer+=ele
62 }
63
64 val FatherElement: Element = arrbuffer(arrbuffer.size-1)
65 val FatherInterator: util.Iterator[_] = FatherElement.elementIterator()
66 val scalalInterator: Iterator[Element] = javaIterToScalaIter(FatherInterator)
67
68 var relationKey:String=""
69 var relationValue:String=""
70
71 var valueArr:ArrayBuffer[ArrayBuffer[String]]=ArrayBuffer()
72 var value:ArrayBuffer[String]=ArrayBuffer()
73 var keyArr:ArrayBuffer[String]=ArrayBuffer()
74
75 while (scalalInterator.hasNext){
76 val ele: Element = scalalInterator.next()
77 if(relationKey.size==0&&relationValue.size==0){
78 relationKey=ele.getName
79 relationValue=ele.getStringValue
80 }
81 keyArr+=ele.getName
82 value+=ele.getStringValue
83 }
84 valueArr+=value
85
86 val valRows: List[Row] = valueArr.toList.map(x => {
87 val seq = x.toSeq
88 val row: Row = Row.fromSeq(seq)
89 row
90 })
91 val fie: Array[StructField] = keyArr.toList.map(d=>StructField(d.toString,StringType,nullable = true)).toArray
92 val schame: StructType = StructType(fie)
93 finalDF = ss.createDataFrame(ss.sparkContext.makeRDD(valRows),schame)
94
95
96
97 if(returnField.size>0){
98 val returnARR: Array[String] = returnField.split(",")
99 val seq: Seq[Column] = returnARR.map(x=>finalDF(x)).toSeq
100 val df: DataFrame = finalDF.select(seq : _*)
101 finalDF=df
102
103 }
104
105
106 var smallDF: DataFrame = null
107 if(openTag.size>0){
108 val eleExplode: Element = FatherElement.element(openTag)
109
110 val eleExplodeIterator: util.Iterator[_] = eleExplode.elementIterator()
111 val explodeScalaIterator: Iterator[Element] = javaIterToScalaIter(eleExplodeIterator)
112
113 var explodeValueArr:ArrayBuffer[ArrayBuffer[String]]=ArrayBuffer()
114 var sonKeyArr:ArrayBuffer[String]=ArrayBuffer()
115 while (explodeScalaIterator.hasNext){
116 val ele: Element = explodeScalaIterator.next()
117 val sonEle: util.Iterator[_] = ele.elementIterator()
118 val sonScalaInterator = javaIterToScalaIter(sonEle)
119 var sonValueArr:ArrayBuffer[String]=ArrayBuffer()
120 sonValueArr+=relationValue
121 sonKeyArr.clear()
122 sonKeyArr +=(relationKey+"_")
123 while (sonScalaInterator.hasNext){
124 val each: Element = sonScalaInterator.next()
125 sonValueArr+=each.getStringValue
126 sonKeyArr+=(openTag+"_"+each.getName)
127 }
128 explodeValueArr+=sonValueArr
129 }
130 val valueRows: List[Row] = explodeValueArr.toList.map(x => {
131 val seq = x.toSeq
132 val row: Row = Row.fromSeq(seq)
133 row
134 })
135 val field: Array[StructField] = sonKeyArr.toList.map(d=>StructField(d,StringType,nullable = true)).toArray
136 val sonSchame: StructType = StructType(field)
137 smallDF = ss.createDataFrame(ss.sparkContext.makeRDD(valueRows),sonSchame)
138
139 val df: DataFrame = smallDF.join(finalDF,finalDF(relationKey)===smallDF(relationKey+"_"),"left")
140 finalDF= df.drop(relationKey+"_")
141 }
142
143
144 //println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
145 //finalDF.show(20)
146 //println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
147
148 out.write(finalDF)
149 }
150
151
152 override def setProperties(map: Map[String, Any]): Unit = {
153 xmlpath = MapUtil.get(map,"xmlpath").asInstanceOf[String]
154 tagPath = MapUtil.get(map,"tagPath").asInstanceOf[String]
155 openTag = MapUtil.get(map,"openTag").asInstanceOf[String]
156 returnField = MapUtil.get(map,"returnField").asInstanceOf[String]
157 }
158 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
159 var descriptor : List[PropertyDescriptor] = List()
160 val xmlpath = new PropertyDescriptor().name("xmlpath").displayName("xmlpath").description("the xml String").defaultValue("").required(true)
161 descriptor = xmlpath :: descriptor
162 val tagPath = new PropertyDescriptor().name("tagPath").displayName("tagPath").description("The tag path you want to parse,the delimiter is ,").defaultValue("").required(true)
163 descriptor = tagPath :: descriptor
164 val openTag = new PropertyDescriptor().name("openTag").displayName("openTag").description("The tag you want to expand").defaultValue("").required(false)
165 descriptor = openTag :: descriptor
166 val returnField = new PropertyDescriptor().name("returnField").displayName("returnField").description("The name of the field you want to return,the delimiter is ,").defaultValue("").required(false)
167 descriptor = returnField :: descriptor
168 descriptor
169 }
170 override def getIcon(): Array[Byte] = {
171 ImageUtil.getImage("icon/xml/FlattenXmlParser.png")
172 }
173 override def getGroup(): List[String] = {
174 List(StopGroup.XmlGroup.toString)
175 }
176 override def initialize(ctx: ProcessContext): Unit = {
177
178 }
179
180
181}