upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/excel/ExcelParser.scala
1package cn.piflow.bundle.excel
2
3
4import java.io.{BufferedInputStream, ByteArrayInputStream}
5import cn.piflow._
6import cn.piflow.bundle.util.ExcelToJson
7import cn.piflow.conf._
8import cn.piflow.conf.bean.PropertyDescriptor
9import cn.piflow.conf.util.{ImageUtil, MapUtil}
10import net.sf.json.JSONArray
11import org.apache.hadoop.conf.Configuration
12import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
13import org.apache.spark.sql.{DataFrame, SparkSession}
14
15class ExcelParser extends ConfigurableStop{
16
17 val authorEmail: String = "ygang@cnic.cn"
18 val description: String = "Parse excel file to json"
19 val inportList: List[String] = List(PortEnum.DefaultPort.toString)
20 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
21
22 var cachePath: String = _
23
24
25 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
26
27 val spark = pec.get[SparkSession]()
28 val sc = spark.sparkContext
29 val inDf = in.read()
30
31 val configuration: Configuration = new Configuration()
32 var pathStr: String =inDf.take(1)(0).get(0).asInstanceOf[String]
33 val pathARR: Array[String] = pathStr.split("\\/")
34 var hdfsUrl:String=""
35 for (x <- (0 until 3)){
36 hdfsUrl+=(pathARR(x) +"/")
37 }
38
39 configuration.set("fs.defaultFS",hdfsUrl)
40 var fs: FileSystem = FileSystem.get(configuration)
41
42 var hdfsPathJsonCache:String = ""
43
44 hdfsPathJsonCache = hdfsUrl+cachePath+"/excelCache/excelCache.json"
45
46 val path: Path = new Path(hdfsPathJsonCache)
47 if(fs.exists(path)){
48 fs.delete(path)
49 }
50 fs.create(path).close()
51
52 var fdosOut: FSDataOutputStream = fs.append(path)
53 var jsonStr: String =""
54 var bisIn: BufferedInputStream =null
55
56
57 var count = 0 ;
58 inDf.collect().foreach(row=>{
59 val pathStr = row.get(0).asInstanceOf[String]
60
61 if (pathStr.endsWith(".xls") || pathStr.endsWith("xlsx")){
62
63 val array: JSONArray = ExcelToJson.readExcel(pathStr,hdfsUrl)
64
65 println(array.size())
66
67 for (i <- 0 until array.size()){
68 jsonStr = array.get(i).toString
69
70
71 bisIn = new BufferedInputStream(new ByteArrayInputStream((jsonStr+"\n").getBytes()))
72
73 val buff: Array[Byte] = new Array[Byte](1048576)
74
75 var num: Int = bisIn.read(buff)
76 while (num != -1) {
77 fdosOut.write(buff, 0, num)
78 fdosOut.flush()
79 num = bisIn.read(buff)
80 }
81 fdosOut.flush()
82 bisIn = null
83
84 }
85 }
86 })
87
88 fdosOut.close()
89
90 val df: DataFrame = spark.read.json(hdfsPathJsonCache)
91
92 out.write(df)
93
94
95 }
96
97 def initialize(ctx: ProcessContext): Unit = {
98
99 }
100
101 def setProperties(map : Map[String, Any]): Unit = {
102 cachePath = MapUtil.get(map,"cachePath").asInstanceOf[String]
103 }
104
105 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
106 var descriptor : List[PropertyDescriptor] = List()
107 val jsonSavePath = new PropertyDescriptor().name("cachePath").displayName("cachePath").description("save path of json").defaultValue("").required(true)
108 descriptor = jsonSavePath :: descriptor
109 descriptor
110 }
111
112 override def getIcon(): Array[Byte] = {
113 ImageUtil.getImage("icon/excel/excelParse.png")
114 }
115
116 override def getGroup(): List[String] = {
117 List(StopGroup.ExcelGroup.toString)
118 }
119
120}
121