Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/nsfc/XmlParserGenerateNewField.scala

1package cn.piflow.bundle.nsfc
2
3import cn.piflow._
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.SparkSession
8
9import scala.xml.{Elem, NodeSeq, XML}
10
11class XmlParserGenerateNewField extends ConfigurableStop {
12
13  val authorEmail: String = "songdongze@cnic.cn"
14  val description: String = "parse xml string from a hive field, and generate a new field from a xml tag"
15  val inportList: List[String] = List(Port.AnyPort.toString)
16  val outportList: List[String] = List(Port.DefaultPort.toString)
17
18  var decoderField: String = _
19  var tagXPath: String = _
20  var newField: String = _
21
22  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23
24    val spark = pec.get[SparkSession]()
25    val sqlContext=spark.sqlContext
26    val dfOld = in.read()
27    dfOld.createOrReplaceTempView("thesis")
28    val s1: Array[String] = tagXPath.split("/");
29    sqlContext.udf.register("regexPro",(str:String)=>{
30      val xml: Elem = XML.loadString(str)
31      var type_id: NodeSeq = xml
32      for (i <- 2 until s1.length) {
33        type_id = type_id \ s1(i)
34      }
35      type_id.text
36    })
37    val sqlText:String="select *, regexPro(" + decoderField + ") as " + newField + " from thesis"
38
39    val dfNew=sqlContext.sql(sqlText)
40    dfNew.show()
41    out.write(dfNew)
42  }
43
44  def initialize(ctx: ProcessContext): Unit = {
45
46  }
47
48  def setProperties(map : Map[String, Any]) = {
49    decoderField = MapUtil.get(map,"decoderField").asInstanceOf[String]
50    tagXPath = MapUtil.get(map,"tagXPath").asInstanceOf[String]
51    newField = MapUtil.get(map,"newField").asInstanceOf[String]
52
53  }
54
55  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
56    var descriptor : List[PropertyDescriptor] = List()
57    val decoderField = new PropertyDescriptor().name("decoderField").displayName("decoder field").description("decoder field").defaultValue("").required(true)
58    val tagXPath = new PropertyDescriptor().name("tagXPath").displayName("xml tag XPath").description("the tag you want to parse in xml file, XPath").defaultValue("").required(true)
59    val newField = new PropertyDescriptor().name("newField").displayName("new Field name").description("generate a new field from tag").defaultValue("").required(true)
60    descriptor = decoderField :: descriptor
61    descriptor = tagXPath :: descriptor
62    descriptor = newField :: descriptor
63    descriptor
64  }
65
66  override def getIcon(): Array[Byte] = {
67    ImageUtil.getImage("icon/xml/XmlParser.png")
68  }
69
70  override def getGroup(): List[String] = {
71    List(StopGroup.XmlGroup.toString)
72  }
73
74}