Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/JsonParser.scala

1package cn.piflow.bundle.json
2
3import cn.piflow._
4import cn.piflow.bundle.util.JsonUtil
5import cn.piflow.conf._
6import cn.piflow.conf.bean.PropertyDescriptor
7import cn.piflow.conf.util.{ImageUtil, MapUtil}
8import org.apache.spark.sql.{DataFrame, SparkSession}
9
10class JsonParser extends ConfigurableStop{
11
12  val authorEmail: String = "xjzhu@cnic.cn"
13  val description: String = "Parse json file"
14  val inportList: List[String] = List(Port.DefaultPort)
15  val outportList: List[String] = List(Port.DefaultPort)
16
17  var jsonPath: String = _
18  var tag : String = _
19
20  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
21
22    val spark = pec.get[SparkSession]()
23
24    val jsonPathARR: Array[String] = jsonPath.split(",").map(x => x.trim)
25
26    var FinalDF = spark.read.option("multiline","true").json(jsonPathARR(0))
27
28    if(jsonPathARR.length>1){
29      for(x<-(1 until jsonPathARR.length)){
30        var jDF: DataFrame = spark.read.option("multiline","true").json(jsonPathARR(x))
31        val ff: DataFrame = FinalDF.union(jDF).toDF()
32        FinalDF=ff
33      }
34    }
35    FinalDF.printSchema()
36
37    if(tag.length>0){
38      val writeDF: DataFrame = JsonUtil.ParserJsonDF(FinalDF,tag)
39      FinalDF=writeDF
40    }
41
42    out.write(FinalDF)
43  }
44
45  def initialize(ctx: ProcessContext): Unit = {
46
47  }
48
49  override def setProperties(map: Map[String, Any]): Unit = {
50    jsonPath = MapUtil.get(map,"jsonPath").asInstanceOf[String]
51    tag = MapUtil.get(map,"tag").asInstanceOf[String]
52  }
53
54  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
55    var descriptor : List[PropertyDescriptor] = List()
56    val jsonPath = new PropertyDescriptor()
57      .name("jsonPath")
58      .displayName("jsonPath")
59      .description("The path of the json file,Separate multiple files with commas")
60      .defaultValue("")
61      .required(true)
62      .example("hdfs://master:9000/work/json/test/example.json,hdfs://master:9000/work/json/test/example1.json")
63
64    val tag = new PropertyDescriptor()
65      .name("tag")
66      .displayName("Tag")
67      .description("The tag you want to parse,If you want to open an array field,you have to write it like this:links_name(MasterField_ChildField)")
68      .defaultValue("")
69      .required(false)
70      .example("name,province_name")
71
72    descriptor = jsonPath :: descriptor
73    descriptor = tag :: descriptor
74    descriptor
75  }
76
77  override def getIcon(): Array[Byte] = {
78    ImageUtil.getImage("icon/json/jsonParser.png")
79  }
80
81  override def getGroup(): List[String] = {
82    List(StopGroup.JsonGroup)
83  }
84
85}
86