upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlStringParser.scala
1package cn.piflow.bundle.xml
2
3import java.util
4
5import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
6import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
7import cn.piflow.conf.bean.PropertyDescriptor
8import cn.piflow.conf.util.{ImageUtil, MapUtil}
9import org.apache.spark.rdd.RDD
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11import org.apache.spark.sql.types.{StringType, StructField, StructType}
12import org.dom4j.{Document, DocumentHelper, Element}
13
14import scala.collection.JavaConverters._
15import scala.collection.mutable.{ArrayBuffer, ListBuffer}
16
17class XmlStringParser extends ConfigurableStop {
18 override val authorEmail: String = "yangqidong@cnic.cn"
19 val inportList: List[String] = List(Port.DefaultPort)
20 val outportList: List[String] = List(Port.DefaultPort)
21 override val description: String = "Parse xml string"
22
23 var XmlString:String=_
24 var label:String=_
25 var schema: String = _
26
27
28 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
29 val spark = pec.get[SparkSession]()
30
31 val doc: Document = DocumentHelper.parseText(XmlString)
32 val rootElt: Element = doc.getRootElement
33 var arrbuffer:ArrayBuffer[Element]=ArrayBuffer()
34 arrbuffer+=rootElt
35
36 val arrLabel: Array[String] = label.split(",").map(x => x.trim)
37 for(x<-(1 until arrLabel.length)){
38 var ele: Element =null
39 if(x==1){
40 ele = rootElt.element(arrLabel(x).toString)
41 }else{
42 ele = arrbuffer(x-2).element(arrLabel(x).toString)
43 }
44 arrbuffer+=ele
45 }
46
47
48 val FatherElement: Element = arrbuffer(arrbuffer.size-2)
49
50 val arrSchame: Array[String] = schema.split(",").map(x => x.trim)
51
52 var list:ListBuffer[String]=ListBuffer()
53
54
55 val FatherInterator: util.Iterator[_] = FatherElement.elementIterator(arrbuffer.last.getName)
56 val scalaIterator: Iterator[Element] = FatherInterator.asInstanceOf[util.Iterator[Element]].asScala
57
58 while (scalaIterator.hasNext){
59 val value: Element = scalaIterator.next()
60 var text: String =""
61 for(each<-arrSchame){
62 text += value.element(each).getText+","
63 }
64 list.+=(text.substring(0,text.length-1))
65 }
66
67
68 val listRows: List[Row] = list.toList.map(line => {
69 val seq: Seq[String] = line.split(",").toSeq
70 val row = Row.fromSeq(seq)
71 row
72 })
73 val rowRDD: RDD[Row] = spark.sparkContext.makeRDD(listRows)
74
75
76 val fields: Array[StructField] = arrSchame.map(p => {
77 StructField(p, StringType, nullable = true)
78 })
79 val structType: StructType = StructType(fields)
80
81 val Fdf: DataFrame = spark.createDataFrame(rowRDD,structType)
82
83 //Fdf.show(20)
84 out.write(Fdf)
85
86
87 }
88
89 def StrToFile(str: String): Unit = {
90
91
92 }
93
94 override def setProperties(map: Map[String, Any]): Unit = {
95 XmlString = MapUtil.get(map,"XmlString").asInstanceOf[String]
96 label = MapUtil.get(map,"label").asInstanceOf[String]
97 schema = MapUtil.get(map,"schema").asInstanceOf[String]
98 }
99
100 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
101 var descriptor : List[PropertyDescriptor] = List()
102 val XmlString = new PropertyDescriptor()
103 .name("XmlString")
104 .displayName("XmlString")
105 .description("the xml String")
106 .defaultValue("")
107 .required(true)
108 .example("<sites>\n <site>\n <name>菜鸟教程</name>\n <url>www.runoob.com</url>\n </site>\n <site>\n <name>Google</name>\n <url>www.google.com</url>\n </site>\n <site>\n <name>淘宝</name>\n <url>www.taobao.com</url>\n </site>\n</sites>")
109
110 descriptor = XmlString :: descriptor
111 val label = new PropertyDescriptor()
112 .name("label")
113 .displayName("label")
114 .description("Parsed label path")
115 .defaultValue("")
116 .required(true)
117 .example("sites,site")
118
119 descriptor = label :: descriptor
120 val schema = new PropertyDescriptor()
121 .name("schema")
122 .displayName("schema")
123 .description("Parsed tag name")
124 .defaultValue("")
125 .required(true)
126 .example("name,url")
127 descriptor = schema :: descriptor
128 descriptor
129 }
130
131 override def getIcon(): Array[Byte] = {
132 ImageUtil.getImage("icon/xml/XmlStringParser.png")
133 }
134
135 override def getGroup(): List[String] = {
136 List(StopGroup.XmlGroup)
137 }
138
139 override def initialize(ctx: ProcessContext): Unit = {
140
141 }
142
143}