Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/excel/ExcelParser.scala

1package cn.piflow.bundle.excel
2
3
4import java.io.{BufferedInputStream, ByteArrayInputStream}
5import cn.piflow._
6import cn.piflow.bundle.util.ExcelToJson
7import cn.piflow.conf._
8import cn.piflow.conf.bean.PropertyDescriptor
9import cn.piflow.conf.util.{ImageUtil, MapUtil}
10import net.sf.json.JSONArray
11import org.apache.hadoop.conf.Configuration
12import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
13import org.apache.spark.sql.{DataFrame, SparkSession}
14
15class ExcelParser extends ConfigurableStop{
16
17  val authorEmail: String = "ygang@cnic.cn"
18  val description: String = "Parse excel file to json"
19  val inportList: List[String] = List(PortEnum.DefaultPort.toString)
20  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
21
22  var cachePath: String = _
23
24
25  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
26
27    val spark = pec.get[SparkSession]()
28    val sc = spark.sparkContext
29    val inDf = in.read()
30
31    val configuration: Configuration = new Configuration()
32    var pathStr: String =inDf.take(1)(0).get(0).asInstanceOf[String]
33    val pathARR: Array[String] = pathStr.split("\\/")
34    var hdfsUrl:String=""
35    for (x <- (0 until 3)){
36      hdfsUrl+=(pathARR(x) +"/")
37    }
38
39    configuration.set("fs.defaultFS",hdfsUrl)
40    var fs: FileSystem = FileSystem.get(configuration)
41
42    var hdfsPathJsonCache:String = ""
43
44    hdfsPathJsonCache = hdfsUrl+cachePath+"/excelCache/excelCache.json"
45
46    val path: Path = new Path(hdfsPathJsonCache)
47    if(fs.exists(path)){
48      fs.delete(path)
49    }
50    fs.create(path).close()
51
52    var fdosOut: FSDataOutputStream = fs.append(path)
53    var jsonStr: String =""
54    var bisIn: BufferedInputStream =null
55
56
57    var count = 0 ;
58    inDf.collect().foreach(row=>{
59      val pathStr = row.get(0).asInstanceOf[String]
60
61      if (pathStr.endsWith(".xls") || pathStr.endsWith("xlsx")){
62
63        val array: JSONArray = ExcelToJson.readExcel(pathStr,hdfsUrl)
64
65        println(array.size())
66
67        for (i <- 0 until array.size()){
68          jsonStr = array.get(i).toString
69
70
71          bisIn = new BufferedInputStream(new ByteArrayInputStream((jsonStr+"\n").getBytes()))
72
73          val buff: Array[Byte] = new Array[Byte](1048576)
74
75          var num: Int = bisIn.read(buff)
76          while (num != -1) {
77            fdosOut.write(buff, 0, num)
78            fdosOut.flush()
79            num = bisIn.read(buff)
80          }
81          fdosOut.flush()
82          bisIn = null
83
84        }
85      }
86    })
87
88    fdosOut.close()
89
90    val df: DataFrame = spark.read.json(hdfsPathJsonCache)
91
92    out.write(df)
93
94
95  }
96
97  def initialize(ctx: ProcessContext): Unit = {
98
99  }
100
101  def setProperties(map : Map[String, Any]): Unit = {
102    cachePath = MapUtil.get(map,"cachePath").asInstanceOf[String]
103  }
104
105  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
106    var descriptor : List[PropertyDescriptor] = List()
107    val jsonSavePath = new PropertyDescriptor().name("cachePath").displayName("cachePath").description("save path of json").defaultValue("").required(true)
108    descriptor = jsonSavePath :: descriptor
109    descriptor
110  }
111
112  override def getIcon(): Array[Byte] = {
113    ImageUtil.getImage("icon/excel/excelParse.png")
114  }
115
116  override def getGroup(): List[String] = {
117    List(StopGroup.ExcelGroup.toString)
118  }
119
120}
121