Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/MultiFolderJsonParser.scala

1package cn.piflow.bundle.json
2
3import cn.piflow.bundle.util.JsonUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11import scala.util.control.Breaks.{break, breakable}
12
13class MultiFolderJsonParser extends ConfigurableStop{
14  val authorEmail: String = "yangqidong@cnic.cn"
15  val inportList: List[String] = List(PortEnum.NonePort.toString)
16  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
17  val description: String = "Parse json folders"
18
19  var jsonPathes: String = _
20  var tag : String = _
21
22  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23    val ss = pec.get[SparkSession]()
24
25    val arrPath: Array[String] = jsonPathes.split(";")
26
27    var index: Int = 0
28    breakable {
29      for (i <- 0 until arrPath.length) {
30        if (ss.read.json(arrPath(i)).count() != 0) {
31          index = i
32          break
33        }
34      }
35    }
36
37    var FinalDF = ss.read.option("multiline","true").json(arrPath(index))
38
39    for(d <- index+1 until(arrPath.length)){
40      if(ss.read.json(arrPath(d)).count()!=0){
41        val df1: DataFrame = ss.read.option("multiline","true").json(arrPath(d))
42        val df2: DataFrame = FinalDF.union(df1).toDF()
43        FinalDF=df2
44      }
45    }
46
47    if(tag.length>0){
48      val writeDF: DataFrame = JsonUtil.ParserJsonDF(FinalDF,tag)
49      FinalDF=writeDF
50    }
51
52    out.write(FinalDF)
53  }
54
55  override def setProperties(map: Map[String, Any]): Unit = {
56    jsonPathes = MapUtil.get(map,"jsonPathes").asInstanceOf[String]
57    tag = MapUtil.get(map,"tag").asInstanceOf[String]
58  }
59  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
60    var descriptor : List[PropertyDescriptor] = List()
61    val jsonPathes = new PropertyDescriptor().name("jsonPathes").displayName("jsonPathes").description("The path of the json folder,the delimiter is ;").defaultValue("").required(true)
62    val tag = new PropertyDescriptor().name("tag").displayName("tag").description("The tag you want to parse,If you want to open an array field,you have to write it like this:links_name(MasterField_ChildField)").defaultValue("").required(false)
63    descriptor = jsonPathes :: descriptor
64    descriptor = tag :: descriptor
65    descriptor
66  }
67  override def getIcon(): Array[Byte] = {
68    ImageUtil.getImage("icon/json/MultiFolderJsonParser.png")
69  }
70
71  override def getGroup(): List[String] = {
72    List(StopGroup.JsonGroup.toString)
73  }
74  override def initialize(ctx: ProcessContext): Unit = {
75
76  }
77
78
79}