upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParserFolder.scala
1package cn.piflow.bundle.xml
2
3import java.net.URI
4
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf._
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.hadoop.conf.Configuration
10import org.apache.hadoop.fs.{FileSystem, Path}
11import org.apache.spark.sql.{DataFrame, SparkSession}
12
13import scala.collection.mutable.ArrayBuffer
14import scala.util.control.Breaks._
15
16/**
17 * Created by admin on 2018/8/27.
18 */
19class XmlParserFolder extends ConfigurableStop{
20 val authorEmail: String = "lijie"
21 val description: String = "Parse xml folder"
22 val inportList: List[String] = List(Port.DefaultPort)
23 val outportList: List[String] = List(Port.DefaultPort)
24
25 var rowTag:String = _
26 var xmlpath:String = _
27 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
28
29 val spark = pec.get[SparkSession]()
30 val pathArr = getFileName(xmlpath)
31 val xmlDF = getResDf(pathArr,spark)
32 //xmlDF.show(30)
33 out.write(xmlDF)
34 }
35 override def setProperties(map: Map[String, Any]): Unit = {
36 xmlpath = MapUtil.get(map,"xmlpath").asInstanceOf[String]
37 rowTag = MapUtil.get(map,"rowTag").asInstanceOf[String]
38 }
39
40 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
41 var descriptor : List[PropertyDescriptor] = List()
42 val xmlpath = new PropertyDescriptor()
43 .name("xmlpath")
44 .displayName("xmlpath")
45 .defaultValue("")
46 .required(true)
47 .example("/work/test/xml/")
48
49 val rowTag = new PropertyDescriptor()
50 .name("rowTag")
51 .displayName("rowTag")
52 .description("the tag you want to parse in xml file")
53 .defaultValue("")
54 .required(true)
55 .example("name,url")
56
57 descriptor = xmlpath :: descriptor
58 descriptor = rowTag :: descriptor
59 descriptor
60 }
61
62 override def getIcon(): Array[Byte] = {
63 ImageUtil.getImage("icon/xml/FolderXmlParser.png")
64 }
65
66 override def getGroup(): List[String] = {
67 List(StopGroup.XmlGroup.toString)
68 }
69
70 override def initialize(ctx: ProcessContext): Unit ={}
71
72 //获取.xml所有文件路径
73 def getFileName(path:String):ArrayBuffer[String]={
74 var arr = ArrayBuffer[String]()
75 val conf = new Configuration()
76 val fs = FileSystem.get(URI.create(path), conf)
77 val statuses = fs.listStatus(new Path(path))
78 for (i <- statuses) {
79 if(!i.isDirectory){
80 if (i.getPath().getName.endsWith(".xml")) {
81 arr+=i.getPath.toString
82 }
83 }else{
84 val arr1 = getFileName(i.getPath.toString)
85 arr=arr++(arr1)
86 }
87 }
88 arr
89 }
90 //获取每个xml得dataframe
91 def getDf(path:String,sparkSession: SparkSession):DataFrame={
92 val df = sparkSession.read.format("com.databricks.spark.xml")
93 .option("rowTag", rowTag)
94 .option("treatEmptyValuesAsNulls", true)
95 .load(path)
96 df
97 }
98 //获取文件夹最终得dataframe
99 def getResDf(pathArr:ArrayBuffer[String],spark:SparkSession):DataFrame={
100 var index =0
101 breakable{
102 for(i <- 0 until pathArr.length){
103 if(getDf(pathArr(i),spark).count()!=0){
104 index=i
105 break
106 }
107 }
108 }
109 var df = spark.read.format("com.databricks.spark.xml")
110 .option("rowTag", rowTag)
111 .option("treatEmptyValuesAsNulls", true)
112 .load(pathArr(index))
113 for(d <- index+1 until(pathArr.length)){
114 if(getDf(pathArr(d),spark).count()!=0){
115 val df1 = spark.read.format("com.databricks.spark.xml")
116 .option("rowTag", rowTag)
117 .option("treatEmptyValuesAsNulls", true)
118 .load(pathArr(d))
119 df = df.union(df1)
120 }
121 }
122 df
123 }
124}
125