upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/xml/XmlParser.scala
1package cn.piflow.bundle.xml
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.SparkSession
8import org.apache.spark.sql.types.StructType
9
10import scala.beans.BeanProperty
11
12class XmlParser extends ConfigurableStop {
13
14 val authorEmail: String = "xjzhu@cnic.cn"
15 val description: String = "Parse xml file"
16 val inportList: List[String] = List(Port.DefaultPort)
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var xmlpath:String = _
20 var rowTag:String = _
21
22 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23
24 val spark = pec.get[SparkSession]()
25
26 val xmlDF = spark.read.format("com.databricks.spark.xml")
27 .option("rowTag",rowTag)
28 .option("treatEmptyValuesAsNulls",true)
29 .load(xmlpath)
30
31 out.write(xmlDF)
32 }
33
34 def initialize(ctx: ProcessContext): Unit = {
35
36 }
37
38 def setProperties(map : Map[String, Any]) = {
39 xmlpath = MapUtil.get(map,"xmlpath").asInstanceOf[String]
40 rowTag = MapUtil.get(map,"rowTag").asInstanceOf[String]
41 }
42
43 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
44 var descriptor : List[PropertyDescriptor] = List()
45 val xmlpath = new PropertyDescriptor()
46 .name("xmlpath")
47 .displayName("xmlpath")
48 .description("the path of xml file")
49 .defaultValue("")
50 .required(true)
51 .example("hdfs://192.168.3.138:8020/work/test/testxml.xml")
52
53 val rowTag = new PropertyDescriptor()
54 .name("rowTag")
55 .displayName("rowTag")
56 .description("the tag you want to parse in xml file")
57 .defaultValue("")
58 .required(true)
59 .example("name")
60
61 descriptor = xmlpath :: descriptor
62 descriptor = rowTag :: descriptor
63 descriptor
64 }
65
66 override def getIcon(): Array[Byte] = {
67 ImageUtil.getImage("icon/xml/XmlParser.png")
68 }
69
70 override def getGroup(): List[String] = {
71 List(StopGroup.XmlGroup)
72 }
73
74}