upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/json/FolderJsonParser.scala
1package cn.piflow.bundle.json
2
3
4import java.net.URI
5
6import cn.piflow.bundle.util.JsonUtil
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
9import cn.piflow.conf.bean.PropertyDescriptor
10import cn.piflow.conf.util.{ImageUtil, MapUtil}
11import org.apache.hadoop.conf.Configuration
12import org.apache.hadoop.fs.{FileStatus, FileSystem, FileUtil, Path}
13import org.apache.spark.sql._
14
15import scala.collection.mutable.ArrayBuffer
16import scala.util.control.Breaks.{break, breakable}
17import org.apache.spark.sql.{DataFrame, SQLContext}
18
19
20
21class FolderJsonParser extends ConfigurableStop{
22 override val authorEmail: String = "yangqidong@cnic.cn"
23 val inportList: List[String] = List(PortEnum.NonePort.toString)
24 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
25 override val description: String ="Parse json folder"
26
27 var FolderPath:String = _
28 var tag : String = _
29
30 var openArrField:String=""
31 var ArrSchame:String=""
32 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
33 val spark = pec.get[SparkSession]()
34 val sql: SQLContext = spark.sqlContext
35
36 val arrPath: ArrayBuffer[String] = getFileName(FolderPath)
37 var FinalDF: DataFrame = getFinalDF(arrPath,sql)
38
39 if(tag.length>0){
40 val writeDF: DataFrame = JsonUtil.ParserJsonDF(FinalDF,tag)
41 FinalDF=writeDF
42 }
43
44 out.write(FinalDF)
45 }
46
47 def getDf(Path: String,ss:SQLContext): DataFrame ={
48 val frame: DataFrame = ss.read.json(Path)
49 frame
50 }
51
52 def getFinalDF(arrPath: ArrayBuffer[String],ss:SQLContext): DataFrame = {
53 var index: Int = 0
54 breakable {
55 for (i <- 0 until arrPath.length) {
56 if (getDf(arrPath(i), ss).count() != 0) {
57 index = i
58 break
59 }
60 }
61 }
62
63 val df01 = ss.read.option("multiline","true").json(arrPath(index))
64
65 var aaa:String="name"
66 var df: DataFrame = df01
67 for(d <- index+1 until(arrPath.length)){
68 if(getDf(arrPath(d),ss).count()!=0){
69 val df1: DataFrame = ss.read.option("multiline","true").json(arrPath(d))
70 val df2: DataFrame = df.union(df1).toDF()
71 df=df2
72 }
73 }
74 df
75 }
76
77
78 def getFileName(path:String):ArrayBuffer[String]={
79 val conf: Configuration = new Configuration()
80 val hdfs: FileSystem = FileSystem.get(URI.create(path),conf)
81 val fs: Array[FileStatus] = hdfs.listStatus(new Path(path))
82 val arrPath: Array[Path] = FileUtil.stat2Paths(fs)
83 var arrBuffer:ArrayBuffer[String]=ArrayBuffer()
84 for(eachPath<-arrPath){
85 arrBuffer+=(FolderPath+eachPath.getName)
86 }
87 arrBuffer
88 }
89
90
91
92 override def setProperties(map: Map[String, Any]): Unit = {
93 FolderPath = MapUtil.get(map,"FolderPath").asInstanceOf[String]
94 tag = MapUtil.get(map,"tag").asInstanceOf[String]
95
96 }
97
98
99
100 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
101 var descriptor : List[PropertyDescriptor] = List()
102 val FolderPath = new PropertyDescriptor().name("FolderPath").displayName("FolderPath").description("The path of the json folder").defaultValue("").required(true)
103 descriptor = FolderPath :: descriptor
104 val tag = new PropertyDescriptor().name("tag").displayName("tag").description("The tag you want to parse,If you want to open an array field,you have to write it like this:links_name(MasterField_ChildField)").defaultValue("").required(false)
105 descriptor = tag :: descriptor
106
107
108 descriptor
109 }
110
111
112
113 override def getIcon(): Array[Byte] = {
114 ImageUtil.getImage("icon/json/FolderJsonPath.png")
115 }
116
117 override def getGroup(): List[String] = {
118 List(StopGroup.JsonGroup.toString)
119 }
120
121 override def initialize(ctx: ProcessContext): Unit = {
122
123 }
124
125}