Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/FolderJsonParser.scala

1package cn.piflow.bundle.json
2
3
4import java.net.URI
5
6import cn.piflow.bundle.util.JsonUtil
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
9import cn.piflow.conf.bean.PropertyDescriptor
10import cn.piflow.conf.util.{ImageUtil, MapUtil}
11import org.apache.hadoop.conf.Configuration
12import org.apache.hadoop.fs.{FileStatus, FileSystem, FileUtil, Path}
13import org.apache.spark.sql._
14
15import scala.collection.mutable.ArrayBuffer
16import scala.util.control.Breaks.{break, breakable}
17import org.apache.spark.sql.{DataFrame, SQLContext}
18
19
20
21class FolderJsonParser extends ConfigurableStop{
22  override val authorEmail: String = "yangqidong@cnic.cn"
23  val inportList: List[String] = List(PortEnum.NonePort.toString)
24  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
25  override val description: String ="Parse json folder"
26
27  var FolderPath:String = _
28  var tag : String = _
29
30  var openArrField:String=""
31  var ArrSchame:String=""
32  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
33    val spark = pec.get[SparkSession]()
34    val sql: SQLContext = spark.sqlContext
35
36    val arrPath: ArrayBuffer[String] = getFileName(FolderPath)
37    var FinalDF: DataFrame = getFinalDF(arrPath,sql)
38
39    if(tag.length>0){
40      val writeDF: DataFrame = JsonUtil.ParserJsonDF(FinalDF,tag)
41      FinalDF=writeDF
42    }
43
44    out.write(FinalDF)
45  }
46
47  def getDf(Path: String,ss:SQLContext): DataFrame ={
48    val frame: DataFrame = ss.read.json(Path)
49    frame
50  }
51
52  def getFinalDF(arrPath: ArrayBuffer[String],ss:SQLContext): DataFrame = {
53    var index: Int = 0
54    breakable {
55      for (i <- 0 until arrPath.length) {
56        if (getDf(arrPath(i), ss).count() != 0) {
57          index = i
58          break
59        }
60      }
61    }
62
63    val df01 = ss.read.option("multiline","true").json(arrPath(index))
64
65    var aaa:String="name"
66    var df: DataFrame = df01
67    for(d <- index+1 until(arrPath.length)){
68      if(getDf(arrPath(d),ss).count()!=0){
69                val df1: DataFrame = ss.read.option("multiline","true").json(arrPath(d))
70        val df2: DataFrame = df.union(df1).toDF()
71        df=df2
72      }
73    }
74    df
75  }
76
77
78  def getFileName(path:String):ArrayBuffer[String]={
79    val conf: Configuration = new Configuration()
80    val hdfs: FileSystem = FileSystem.get(URI.create(path),conf)
81    val fs: Array[FileStatus] = hdfs.listStatus(new Path(path))
82    val arrPath: Array[Path] = FileUtil.stat2Paths(fs)
83    var arrBuffer:ArrayBuffer[String]=ArrayBuffer()
84    for(eachPath<-arrPath){
85      arrBuffer+=(FolderPath+eachPath.getName)
86    }
87    arrBuffer
88  }
89
90
91
92  override def setProperties(map: Map[String, Any]): Unit = {
93    FolderPath = MapUtil.get(map,"FolderPath").asInstanceOf[String]
94    tag = MapUtil.get(map,"tag").asInstanceOf[String]
95
96  }
97
98
99
100  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
101    var descriptor : List[PropertyDescriptor] = List()
102    val FolderPath = new PropertyDescriptor().name("FolderPath").displayName("FolderPath").description("The path of the json folder").defaultValue("").required(true)
103    descriptor = FolderPath :: descriptor
104    val tag = new PropertyDescriptor().name("tag").displayName("tag").description("The tag you want to parse,If you want to open an array field,you have to write it like this:links_name(MasterField_ChildField)").defaultValue("").required(false)
105    descriptor = tag :: descriptor
106
107
108    descriptor
109  }
110
111
112
113  override def getIcon(): Array[Byte] = {
114    ImageUtil.getImage("icon/json/FolderJsonPath.png")
115  }
116
117  override def getGroup(): List[String] = {
118    List(StopGroup.JsonGroup.toString)
119  }
120
121  override def initialize(ctx: ProcessContext): Unit = {
122
123  }
124
125}