Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParser.scala

1package cn.piflow.bundle.xml
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.SparkSession
8import org.apache.spark.sql.types.StructType
9
10import scala.beans.BeanProperty
11
12class XmlParser extends ConfigurableStop {
13
14  val authorEmail: String = "xjzhu@cnic.cn"
15  val description: String = "Parse xml file"
16  val inportList: List[String] = List(Port.DefaultPort)
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var xmlpath:String = _
20  var rowTag:String = _
21
22  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23
24    val spark = pec.get[SparkSession]()
25
26    val xmlDF = spark.read.format("com.databricks.spark.xml")
27      .option("rowTag",rowTag)
28      .option("treatEmptyValuesAsNulls",true)
29      .load(xmlpath)
30
31    out.write(xmlDF)
32  }
33
34  def initialize(ctx: ProcessContext): Unit = {
35
36  }
37
38  def setProperties(map : Map[String, Any]) = {
39    xmlpath = MapUtil.get(map,"xmlpath").asInstanceOf[String]
40    rowTag = MapUtil.get(map,"rowTag").asInstanceOf[String]
41  }
42
43  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
44    var descriptor : List[PropertyDescriptor] = List()
45    val xmlpath = new PropertyDescriptor()
46      .name("xmlpath")
47      .displayName("xmlpath")
48      .description("the path of xml file")
49      .defaultValue("")
50      .required(true)
51      .example("hdfs://192.168.3.138:8020/work/test/testxml.xml")
52
53    val rowTag = new PropertyDescriptor()
54      .name("rowTag")
55      .displayName("rowTag")
56      .description("the tag you want to parse in xml file")
57      .defaultValue("")
58      .required(true)
59        .example("name")
60
61    descriptor = xmlpath :: descriptor
62    descriptor = rowTag :: descriptor
63    descriptor
64  }
65
66  override def getIcon(): Array[Byte] = {
67    ImageUtil.getImage("icon/xml/XmlParser.png")
68  }
69
70  override def getGroup(): List[String] = {
71    List(StopGroup.XmlGroup)
72  }
73
74}