upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/JsonParser.scala
1package cn.piflow.bundle.json
2
3import cn.piflow._
4import cn.piflow.bundle.util.JsonUtil
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.{DataFrame, SparkSession}
9
10class JsonParser extends ConfigurableStop{
11
12 val authorEmail: String = "xjzhu@cnic.cn"
13 val description: String = "Parse json file"
14 val inportList: List[String] = List(Port.DefaultPort)
15 val outportList: List[String] = List(Port.DefaultPort)
16
17 var jsonPath: String = _
18 var tag : String = _
19
20 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21
22 val spark = pec.get[SparkSession]()
23
24 val jsonPathARR: Array[String] = jsonPath.split(",").map(x => x.trim)
25
26 var FinalDF = spark.read.option("multiline","true").json(jsonPathARR(0))
27
28 if(jsonPathARR.length>1){
29 for(x<-(1 until jsonPathARR.length)){
30 var jDF: DataFrame = spark.read.option("multiline","true").json(jsonPathARR(x))
31 val ff: DataFrame = FinalDF.union(jDF).toDF()
32 FinalDF=ff
33 }
34 }
35 FinalDF.printSchema()
36
37 if(tag.length>0){
38 val writeDF: DataFrame = JsonUtil.ParserJsonDF(FinalDF,tag)
39 FinalDF=writeDF
40 }
41
42 out.write(FinalDF)
43 }
44
45 def initialize(ctx: ProcessContext): Unit = {
46
47 }
48
49 override def setProperties(map: Map[String, Any]): Unit = {
50 jsonPath = MapUtil.get(map,"jsonPath").asInstanceOf[String]
51 tag = MapUtil.get(map,"tag").asInstanceOf[String]
52 }
53
54 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
55 var descriptor : List[PropertyDescriptor] = List()
56 val jsonPath = new PropertyDescriptor()
57 .name("jsonPath")
58 .displayName("jsonPath")
59 .description("The path of the json file,Separate multiple files with commas")
60 .defaultValue("")
61 .required(true)
62 .example("hdfs://master:9000/work/json/test/example.json,hdfs://master:9000/work/json/test/example1.json")
63
64 val tag = new PropertyDescriptor()
65 .name("tag")
66 .displayName("Tag")
67 .description("The tag you want to parse,If you want to open an array field,you have to write it like this:links_name(MasterField_ChildField)")
68 .defaultValue("")
69 .required(false)
70 .example("name,province_name")
71
72 descriptor = jsonPath :: descriptor
73 descriptor = tag :: descriptor
74 descriptor
75 }
76
77 override def getIcon(): Array[Byte] = {
78 ImageUtil.getImage("icon/json/jsonParser.png")
79 }
80
81 override def getGroup(): List[String] = {
82 List(StopGroup.JsonGroup)
83 }
84
85}
86