Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParserFolder.scala

1package cn.piflow.bundle.xml
2
3import java.net.URI
4
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf._
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.hadoop.conf.Configuration
10import org.apache.hadoop.fs.{FileSystem, Path}
11import org.apache.spark.sql.{DataFrame, SparkSession}
12
13import scala.collection.mutable.ArrayBuffer
14import scala.util.control.Breaks._
15
16/**
17  * Created by admin on 2018/8/27.
18  */
19class XmlParserFolder extends ConfigurableStop{
20  val authorEmail: String = "lijie"
21  val description: String = "Parse xml folder"
22  val inportList: List[String] = List(Port.DefaultPort)
23  val outportList: List[String] = List(Port.DefaultPort)
24
25  var rowTag:String = _
26  var xmlpath:String = _
27  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
28
29    val spark = pec.get[SparkSession]()
30    val pathArr = getFileName(xmlpath)
31    val xmlDF = getResDf(pathArr,spark)
32    //xmlDF.show(30)
33    out.write(xmlDF)
34  }
35  override def setProperties(map: Map[String, Any]): Unit = {
36    xmlpath = MapUtil.get(map,"xmlpath").asInstanceOf[String]
37    rowTag = MapUtil.get(map,"rowTag").asInstanceOf[String]
38  }
39
40  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
41    var descriptor : List[PropertyDescriptor] = List()
42    val xmlpath = new PropertyDescriptor()
43      .name("xmlpath")
44      .displayName("xmlpath")
45      .defaultValue("")
46      .required(true)
47      .example("/work/test/xml/")
48
49    val rowTag = new PropertyDescriptor()
50      .name("rowTag")
51      .displayName("rowTag")
52      .description("the tag you want to parse in xml file")
53      .defaultValue("")
54      .required(true)
55        .example("name,url")
56
57    descriptor = xmlpath :: descriptor
58    descriptor = rowTag :: descriptor
59    descriptor
60  }
61
62  override def getIcon(): Array[Byte] = {
63    ImageUtil.getImage("icon/xml/FolderXmlParser.png")
64  }
65
66  override def getGroup(): List[String] = {
67    List(StopGroup.XmlGroup.toString)
68  }
69
70  override def initialize(ctx: ProcessContext): Unit ={}
71
72  //获取.xml所有文件路径
73  def getFileName(path:String):ArrayBuffer[String]={
74    var arr = ArrayBuffer[String]()
75      val conf = new Configuration()
76      val fs = FileSystem.get(URI.create(path), conf)
77      val statuses = fs.listStatus(new Path(path))
78      for (i <- statuses) {
79        if(!i.isDirectory){
80        if (i.getPath().getName.endsWith(".xml")) {
81          arr+=i.getPath.toString
82        }
83        }else{
84          val arr1 = getFileName(i.getPath.toString)
85          arr=arr++(arr1)
86        }
87      }
88    arr
89  }
90  //获取每个xml得dataframe
91  def getDf(path:String,sparkSession: SparkSession):DataFrame={
92    val df = sparkSession.read.format("com.databricks.spark.xml")
93      .option("rowTag", rowTag)
94      .option("treatEmptyValuesAsNulls", true)
95      .load(path)
96    df
97  }
98  //获取文件夹最终得dataframe
99  def getResDf(pathArr:ArrayBuffer[String],spark:SparkSession):DataFrame={
100    var index =0
101    breakable{
102      for(i <- 0 until pathArr.length){
103        if(getDf(pathArr(i),spark).count()!=0){
104          index=i
105          break
106        }
107      }
108    }
109    var df = spark.read.format("com.databricks.spark.xml")
110      .option("rowTag", rowTag)
111      .option("treatEmptyValuesAsNulls", true)
112      .load(pathArr(index))
113    for(d <- index+1 until(pathArr.length)){
114      if(getDf(pathArr(d),spark).count()!=0){
115        val df1 = spark.read.format("com.databricks.spark.xml")
116          .option("rowTag", rowTag)
117          .option("treatEmptyValuesAsNulls", true)
118          .load(pathArr(d))
119        df = df.union(df1)
120      }
121    }
122    df
123  }
124}
125