upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/EvaluateJsonPath.scala
1package cn.piflow.bundle.json
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8
9class EvaluateJsonPath extends ConfigurableStop{
10 override val authorEmail: String = "yangqidong@cnic.cn"
11 val inportList: List[String] = List(PortEnum.NonePort.toString)
12 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
13 override val description: String = "Parse multiple json files"
14
15
16 var jsonPathes: String = _
17 var tag : String = _
18
19
20
21 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
22 val spark = pec.get[SparkSession]()
23
24 val jsonPathARR: Array[String] = jsonPathes.split(";")
25
26 val jsonDF = spark.read.option("multiline","true").json(jsonPathARR(0))
27 var FinalDF: DataFrame = jsonDF.select(tag)
28
29 if(jsonPathARR.length>1){
30 for(x<-(1 until jsonPathARR.length)){
31 var jDF: DataFrame = spark.read.option("multiline","true").json(jsonPathARR(x)).select(tag)
32 val ff: DataFrame = FinalDF.union(jDF).toDF()
33 FinalDF=ff
34 }
35 }
36
37
38 out.write(jsonDF)
39 }
40
41 override def setProperties(map: Map[String, Any]): Unit = {
42 jsonPathes = MapUtil.get(map,"jsonPathes").asInstanceOf[String]
43 tag = MapUtil.get(map,"tag").asInstanceOf[String]
44 }
45 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
46 var descriptor : List[PropertyDescriptor] = List()
47 val jsonPathes = new PropertyDescriptor().name("jsonPathes").displayName("jsonPathes").description("The path of the json file,the delimiter is ;").defaultValue("").required(true)
48 val tag=new PropertyDescriptor().name("tag").displayName("tag").description("The tag you want to parse").defaultValue("").required(true)
49 descriptor = jsonPathes :: descriptor
50 descriptor = tag :: descriptor
51 descriptor
52 }
53 override def getIcon(): Array[Byte] = {
54 ImageUtil.getImage("icon/json/EvaluateJsonPATH .png")
55 }
56
57 override def getGroup(): List[String] = {
58 List(StopGroup.JsonGroup.toString)
59 }
60 override def initialize(ctx: ProcessContext): Unit = {
61
62 }
63
64
65}