Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlStringParser.scala

1package cn.piflow.bundle.xml
2
3import java.util
4
5import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
6import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
7import cn.piflow.conf.bean.PropertyDescriptor
8import cn.piflow.conf.util.{ImageUtil, MapUtil}
9import org.apache.spark.rdd.RDD
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11import org.apache.spark.sql.types.{StringType, StructField, StructType}
12import org.dom4j.{Document, DocumentHelper, Element}
13
14import scala.collection.JavaConverters._
15import scala.collection.mutable.{ArrayBuffer, ListBuffer}
16
17class XmlStringParser extends ConfigurableStop {
18  override val authorEmail: String = "yangqidong@cnic.cn"
19  val inportList: List[String] = List(Port.DefaultPort)
20  val outportList: List[String] = List(Port.DefaultPort)
21  override val description: String = "Parse xml string"
22
23  var XmlString:String=_
24  var label:String=_
25  var schema: String = _
26
27
28  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
29    val spark = pec.get[SparkSession]()
30
31    val doc: Document = DocumentHelper.parseText(XmlString)
32    val rootElt: Element = doc.getRootElement
33    var arrbuffer:ArrayBuffer[Element]=ArrayBuffer()
34    arrbuffer+=rootElt
35
36    val arrLabel: Array[String] = label.split(",").map(x => x.trim)
37    for(x<-(1 until arrLabel.length)){
38      var ele: Element =null
39      if(x==1){
40        ele = rootElt.element(arrLabel(x).toString)
41      }else{
42        ele = arrbuffer(x-2).element(arrLabel(x).toString)
43      }
44      arrbuffer+=ele
45    }
46
47
48    val FatherElement: Element = arrbuffer(arrbuffer.size-2)
49
50    val arrSchame: Array[String] = schema.split(",").map(x => x.trim)
51
52    var list:ListBuffer[String]=ListBuffer()
53
54
55    val FatherInterator: util.Iterator[_] = FatherElement.elementIterator(arrbuffer.last.getName)
56    val scalaIterator: Iterator[Element] = FatherInterator.asInstanceOf[util.Iterator[Element]].asScala
57
58    while (scalaIterator.hasNext){
59      val value: Element = scalaIterator.next()
60      var text: String =""
61      for(each<-arrSchame){
62        text += value.element(each).getText+","
63      }
64      list.+=(text.substring(0,text.length-1))
65    }
66
67
68    val listRows: List[Row] = list.toList.map(line => {
69      val seq: Seq[String] = line.split(",").toSeq
70      val row = Row.fromSeq(seq)
71      row
72    })
73    val rowRDD: RDD[Row] = spark.sparkContext.makeRDD(listRows)
74
75
76    val fields: Array[StructField] = arrSchame.map(p => {
77      StructField(p, StringType, nullable = true)
78    })
79    val structType: StructType = StructType(fields)
80
81    val Fdf: DataFrame = spark.createDataFrame(rowRDD,structType)
82
83    //Fdf.show(20)
84    out.write(Fdf)
85
86
87  }
88
89  def StrToFile(str: String): Unit = {
90
91
92  }
93
94  override def setProperties(map: Map[String, Any]): Unit = {
95    XmlString = MapUtil.get(map,"XmlString").asInstanceOf[String]
96    label = MapUtil.get(map,"label").asInstanceOf[String]
97    schema = MapUtil.get(map,"schema").asInstanceOf[String]
98  }
99
100  override def getPropertyDescriptor(): List[PropertyDescriptor] =  {
101    var descriptor : List[PropertyDescriptor] = List()
102    val XmlString = new PropertyDescriptor()
103      .name("XmlString")
104      .displayName("XmlString")
105      .description("the xml String")
106      .defaultValue("")
107      .required(true)
108      .example("<sites>\n    <site>\n        <name>菜鸟教程</name>\n        <url>www.runoob.com</url>\n    </site>\n    <site>\n        <name>Google</name>\n        <url>www.google.com</url>\n    </site>\n    <site>\n        <name>淘宝</name>\n        <url>www.taobao.com</url>\n    </site>\n</sites>")
109
110    descriptor = XmlString :: descriptor
111    val label = new PropertyDescriptor()
112      .name("label")
113      .displayName("label")
114      .description("Parsed label path")
115      .defaultValue("")
116      .required(true)
117        .example("sites,site")
118
119    descriptor = label :: descriptor
120    val schema = new PropertyDescriptor()
121      .name("schema")
122      .displayName("schema")
123      .description("Parsed tag name")
124      .defaultValue("")
125      .required(true)
126        .example("name,url")
127    descriptor = schema :: descriptor
128    descriptor
129  }
130
131  override def getIcon(): Array[Byte] = {
132    ImageUtil.getImage("icon/xml/XmlStringParser.png")
133  }
134
135  override def getGroup(): List[String] = {
136    List(StopGroup.XmlGroup)
137  }
138
139  override def initialize(ctx: ProcessContext): Unit = {
140
141  }
142
143}