upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/MultiFolderJsonParser.scala
1package cn.piflow.bundle.json
2
3import cn.piflow.bundle.util.JsonUtil
4import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
5import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11import scala.util.control.Breaks.{break, breakable}
12
13class MultiFolderJsonParser extends ConfigurableStop{
14 val authorEmail: String = "yangqidong@cnic.cn"
15 val inportList: List[String] = List(PortEnum.NonePort.toString)
16 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
17 val description: String = "Parse json folders"
18
19 var jsonPathes: String = _
20 var tag : String = _
21
22 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23 val ss = pec.get[SparkSession]()
24
25 val arrPath: Array[String] = jsonPathes.split(";")
26
27 var index: Int = 0
28 breakable {
29 for (i <- 0 until arrPath.length) {
30 if (ss.read.json(arrPath(i)).count() != 0) {
31 index = i
32 break
33 }
34 }
35 }
36
37 var FinalDF = ss.read.option("multiline","true").json(arrPath(index))
38
39 for(d <- index+1 until(arrPath.length)){
40 if(ss.read.json(arrPath(d)).count()!=0){
41 val df1: DataFrame = ss.read.option("multiline","true").json(arrPath(d))
42 val df2: DataFrame = FinalDF.union(df1).toDF()
43 FinalDF=df2
44 }
45 }
46
47 if(tag.length>0){
48 val writeDF: DataFrame = JsonUtil.ParserJsonDF(FinalDF,tag)
49 FinalDF=writeDF
50 }
51
52 out.write(FinalDF)
53 }
54
55 override def setProperties(map: Map[String, Any]): Unit = {
56 jsonPathes = MapUtil.get(map,"jsonPathes").asInstanceOf[String]
57 tag = MapUtil.get(map,"tag").asInstanceOf[String]
58 }
59 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
60 var descriptor : List[PropertyDescriptor] = List()
61 val jsonPathes = new PropertyDescriptor().name("jsonPathes").displayName("jsonPathes").description("The path of the json folder,the delimiter is ;").defaultValue("").required(true)
62 val tag = new PropertyDescriptor().name("tag").displayName("tag").description("The tag you want to parse,If you want to open an array field,you have to write it like this:links_name(MasterField_ChildField)").defaultValue("").required(false)
63 descriptor = jsonPathes :: descriptor
64 descriptor = tag :: descriptor
65 descriptor
66 }
67 override def getIcon(): Array[Byte] = {
68 ImageUtil.getImage("icon/json/MultiFolderJsonParser.png")
69 }
70
71 override def getGroup(): List[String] = {
72 List(StopGroup.JsonGroup.toString)
73 }
74 override def initialize(ctx: ProcessContext): Unit = {
75
76 }
77
78
79}