upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/XmlParserGenerateNewField.scala
1package cn.piflow.bundle.nsfc
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.SparkSession
8
9import scala.xml.{Elem, NodeSeq, XML}
10
11class XmlParserGenerateNewField extends ConfigurableStop {
12
13 val authorEmail: String = "songdongze@cnic.cn"
14 val description: String = "parse xml string from a hive field, and generate a new field from a xml tag"
15 val inportList: List[String] = List(Port.AnyPort.toString)
16 val outportList: List[String] = List(Port.DefaultPort.toString)
17
18 var decoderField: String = _
19 var tagXPath: String = _
20 var newField: String = _
21
22 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23
24 val spark = pec.get[SparkSession]()
25 val sqlContext=spark.sqlContext
26 val dfOld = in.read()
27 dfOld.createOrReplaceTempView("thesis")
28 val s1: Array[String] = tagXPath.split("/");
29 sqlContext.udf.register("regexPro",(str:String)=>{
30 val xml: Elem = XML.loadString(str)
31 var type_id: NodeSeq = xml
32 for (i <- 2 until s1.length) {
33 type_id = type_id \ s1(i)
34 }
35 type_id.text
36 })
37 val sqlText:String="select *, regexPro(" + decoderField + ") as " + newField + " from thesis"
38
39 val dfNew=sqlContext.sql(sqlText)
40 dfNew.show()
41 out.write(dfNew)
42 }
43
44 def initialize(ctx: ProcessContext): Unit = {
45
46 }
47
48 def setProperties(map : Map[String, Any]) = {
49 decoderField = MapUtil.get(map,"decoderField").asInstanceOf[String]
50 tagXPath = MapUtil.get(map,"tagXPath").asInstanceOf[String]
51 newField = MapUtil.get(map,"newField").asInstanceOf[String]
52
53 }
54
55 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
56 var descriptor : List[PropertyDescriptor] = List()
57 val decoderField = new PropertyDescriptor().name("decoderField").displayName("decoder field").description("decoder field").defaultValue("").required(true)
58 val tagXPath = new PropertyDescriptor().name("tagXPath").displayName("xml tag XPath").description("the tag you want to parse in xml file, XPath").defaultValue("").required(true)
59 val newField = new PropertyDescriptor().name("newField").displayName("new Field name").description("generate a new field from tag").defaultValue("").required(true)
60 descriptor = decoderField :: descriptor
61 descriptor = tagXPath :: descriptor
62 descriptor = newField :: descriptor
63 descriptor
64 }
65
66 override def getIcon(): Array[Byte] = {
67 ImageUtil.getImage("icon/xml/XmlParser.png")
68 }
69
70 override def getGroup(): List[String] = {
71 List(StopGroup.XmlGroup.toString)
72 }
73
74}