Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/FlattenXmlParser.scala

1package cn.piflow.bundle.xml
2
3import java.net.URI
4import java.util
5
6import cn.piflow.bundle.util.JsonUtil
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
9import cn.piflow.conf.bean.PropertyDescriptor
10import cn.piflow.conf.util.{ImageUtil, MapUtil}
11import org.apache.hadoop.conf.Configuration
12import org.apache.hadoop.fs.{FSDataInputStream, FileSystem, Path}
13import org.apache.spark.rdd.RDD
14import org.apache.spark.sql.types.{StringType, StructField, StructType}
15import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession}
16import org.dom4j.io.SAXReader
17import org.dom4j.{Document, DocumentHelper, Element}
18
19import scala.collection.JavaConverters._
20import scala.collection.mutable.{ArrayBuffer, ListBuffer}
21
22
23class FlattenXmlParser extends ConfigurableStop{
24  override val authorEmail: String = "yangqidong@cnic.cn"
25  val inportList: List[String] = List(PortEnum.NonePort.toString)
26  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
27  override val description: String = "Parse the XML file and expand the label you need"
28
29
30  var xmlpath:String = _
31  var tagPath:String = _
32  var openTag: String = _
33  var returnField: String = _
34
35
36  def javaIterToScalaIter(utilIter: util.Iterator[_]):Iterator[Element]={
37    utilIter.asInstanceOf[util.Iterator[Element]].asScala
38  }
39
40  var ss: SparkSession =_
41  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
42
43
44
45    ss=pec.get[SparkSession]()
46    val conf = new Configuration()
47    val fs: FileSystem = FileSystem.get(URI.create(xmlpath), conf)
48    val hdfsInputStream: FSDataInputStream = fs.open(new Path(xmlpath))
49    val saxReader: SAXReader = new SAXReader()
50    val document: Document = saxReader.read(hdfsInputStream)
51    val rootElt: Element = document.getRootElement
52
53    var finalDF:DataFrame=null
54
55    var arrbuffer:ArrayBuffer[Element]=ArrayBuffer()
56    arrbuffer+=rootElt
57
58    val arrLabel: Array[String] = tagPath.split(",")
59    for(x<-(1 until arrLabel.length)){
60      var ele: Element =arrbuffer(x-1).element(arrLabel(x))
61      arrbuffer+=ele
62    }
63
64    val FatherElement: Element = arrbuffer(arrbuffer.size-1)
65    val FatherInterator: util.Iterator[_] = FatherElement.elementIterator()
66    val scalalInterator: Iterator[Element] = javaIterToScalaIter(FatherInterator)
67
68      var relationKey:String=""
69      var relationValue:String=""
70
71    var valueArr:ArrayBuffer[ArrayBuffer[String]]=ArrayBuffer()
72    var value:ArrayBuffer[String]=ArrayBuffer()
73    var keyArr:ArrayBuffer[String]=ArrayBuffer()
74
75    while (scalalInterator.hasNext){
76      val ele: Element = scalalInterator.next()
77      if(relationKey.size==0&&relationValue.size==0){
78        relationKey=ele.getName
79        relationValue=ele.getStringValue
80      }
81      keyArr+=ele.getName
82      value+=ele.getStringValue
83    }
84    valueArr+=value
85
86    val valRows: List[Row] = valueArr.toList.map(x => {
87      val seq = x.toSeq
88      val row: Row = Row.fromSeq(seq)
89      row
90    })
91    val fie: Array[StructField] = keyArr.toList.map(d=>StructField(d.toString,StringType,nullable = true)).toArray
92    val schame: StructType = StructType(fie)
93    finalDF = ss.createDataFrame(ss.sparkContext.makeRDD(valRows),schame)
94
95
96
97    if(returnField.size>0){
98      val returnARR: Array[String] = returnField.split(",")
99      val seq: Seq[Column] = returnARR.map(x=>finalDF(x)).toSeq
100      val df: DataFrame = finalDF.select(seq : _*)
101      finalDF=df
102
103    }
104
105
106   var smallDF: DataFrame = null
107     if(openTag.size>0){
108      val eleExplode: Element = FatherElement.element(openTag)
109
110      val eleExplodeIterator: util.Iterator[_] = eleExplode.elementIterator()
111      val explodeScalaIterator: Iterator[Element] = javaIterToScalaIter(eleExplodeIterator)
112
113      var explodeValueArr:ArrayBuffer[ArrayBuffer[String]]=ArrayBuffer()
114      var sonKeyArr:ArrayBuffer[String]=ArrayBuffer()
115      while (explodeScalaIterator.hasNext){
116        val ele: Element = explodeScalaIterator.next()
117        val sonEle: util.Iterator[_] = ele.elementIterator()
118        val sonScalaInterator = javaIterToScalaIter(sonEle)
119        var sonValueArr:ArrayBuffer[String]=ArrayBuffer()
120        sonValueArr+=relationValue
121        sonKeyArr.clear()
122        sonKeyArr +=(relationKey+"_")
123        while (sonScalaInterator.hasNext){
124          val each: Element = sonScalaInterator.next()
125          sonValueArr+=each.getStringValue
126          sonKeyArr+=(openTag+"_"+each.getName)
127        }
128        explodeValueArr+=sonValueArr
129      }
130      val valueRows: List[Row] = explodeValueArr.toList.map(x => {
131        val seq = x.toSeq
132        val row: Row = Row.fromSeq(seq)
133        row
134      })
135      val field: Array[StructField] = sonKeyArr.toList.map(d=>StructField(d,StringType,nullable = true)).toArray
136      val sonSchame: StructType = StructType(field)
137      smallDF = ss.createDataFrame(ss.sparkContext.makeRDD(valueRows),sonSchame)
138
139       val df: DataFrame = smallDF.join(finalDF,finalDF(relationKey)===smallDF(relationKey+"_"),"left")
140       finalDF=  df.drop(relationKey+"_")
141    }
142
143
144    //println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
145    //finalDF.show(20)
146    //println("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
147
148    out.write(finalDF)
149  }
150
151
152  override def setProperties(map: Map[String, Any]): Unit = {
153    xmlpath = MapUtil.get(map,"xmlpath").asInstanceOf[String]
154    tagPath = MapUtil.get(map,"tagPath").asInstanceOf[String]
155    openTag = MapUtil.get(map,"openTag").asInstanceOf[String]
156    returnField = MapUtil.get(map,"returnField").asInstanceOf[String]
157  }
158  override def getPropertyDescriptor(): List[PropertyDescriptor] =  {
159    var descriptor : List[PropertyDescriptor] = List()
160    val xmlpath = new PropertyDescriptor().name("xmlpath").displayName("xmlpath").description("the xml String").defaultValue("").required(true)
161    descriptor = xmlpath :: descriptor
162    val tagPath = new PropertyDescriptor().name("tagPath").displayName("tagPath").description("The tag path you want to parse,the delimiter is ,").defaultValue("").required(true)
163    descriptor = tagPath :: descriptor
164    val openTag = new PropertyDescriptor().name("openTag").displayName("openTag").description("The tag you want to expand").defaultValue("").required(false)
165    descriptor = openTag :: descriptor
166    val returnField = new PropertyDescriptor().name("returnField").displayName("returnField").description("The name of the field you want to return,the delimiter is ,").defaultValue("").required(false)
167    descriptor = returnField :: descriptor
168    descriptor
169  }
170  override def getIcon(): Array[Byte] = {
171    ImageUtil.getImage("icon/xml/FlattenXmlParser.png")
172  }
173  override def getGroup(): List[String] = {
174    List(StopGroup.XmlGroup.toString)
175  }
176  override def initialize(ctx: ProcessContext): Unit = {
177
178  }
179
180
181}