Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/EvaluateJsonPath.scala

1package cn.piflow.bundle.json
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8
9class EvaluateJsonPath extends ConfigurableStop{
10  override val authorEmail: String = "yangqidong@cnic.cn"
11  val inportList: List[String] = List(PortEnum.NonePort.toString)
12  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
13  override val description: String = "Parse multiple json files"
14
15
16  var jsonPathes: String = _
17  var tag : String = _
18
19
20
21  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
22    val spark = pec.get[SparkSession]()
23
24    val jsonPathARR: Array[String] = jsonPathes.split(";")
25
26    val jsonDF = spark.read.option("multiline","true").json(jsonPathARR(0))
27    var FinalDF: DataFrame = jsonDF.select(tag)
28
29   if(jsonPathARR.length>1){
30     for(x<-(1 until jsonPathARR.length)){
31       var jDF: DataFrame = spark.read.option("multiline","true").json(jsonPathARR(x)).select(tag)
32       val ff: DataFrame = FinalDF.union(jDF).toDF()
33       FinalDF=ff
34     }
35   }
36
37
38    out.write(jsonDF)
39  }
40
41  override def setProperties(map: Map[String, Any]): Unit = {
42    jsonPathes = MapUtil.get(map,"jsonPathes").asInstanceOf[String]
43    tag = MapUtil.get(map,"tag").asInstanceOf[String]
44  }
45  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
46    var descriptor : List[PropertyDescriptor] = List()
47    val jsonPathes = new PropertyDescriptor().name("jsonPathes").displayName("jsonPathes").description("The path of the json file,the delimiter is ;").defaultValue("").required(true)
48    val tag=new PropertyDescriptor().name("tag").displayName("tag").description("The tag you want to parse").defaultValue("").required(true)
49    descriptor = jsonPathes :: descriptor
50    descriptor = tag :: descriptor
51    descriptor
52  }
53  override def getIcon(): Array[Byte] = {
54    ImageUtil.getImage("icon/json/EvaluateJsonPATH .png")
55  }
56
57  override def getGroup(): List[String] = {
58    List(StopGroup.JsonGroup.toString)
59  }
60  override def initialize(ctx: ProcessContext): Unit = {
61
62  }
63
64
65}