Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/csv/FolderCsvParser.scala

1package cn.piflow.bundle.csv
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.SparkContext
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.types.{StringType, StructField, StructType}
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11
12class FolderCsvParser extends ConfigurableStop{
13  override val authorEmail: String = "yangqidong@cnic.cn"
14  val inportList: List[String] = List(PortEnum.NonePort.toString)
15  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
16  override val description: String = "Parse csv folder"
17
18  var FolderPath:String=_
19  var delimiter: String = _
20  var schema: String = _
21
22  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23    val session: SparkSession = pec.get[SparkSession]
24    val context: SparkContext = session.sparkContext
25
26    val StrRDD: RDD[String] = context.textFile(FolderPath)
27    var num:Int=0
28    var s:String=delimiter
29
30    val rowRDD: RDD[Row] = StrRDD.map(line => {
31      val seqSTR: Seq[String] = line.split(s).toSeq
32      num=seqSTR.size
33      val row = Row.fromSeq(seqSTR)
34      row
35    })
36
37    val schameARR: Array[String] = schema.split(",")
38    val fields: Array[StructField] = schameARR.map(d=>StructField(d.trim,StringType,nullable = true))
39
40    val NewSchema: StructType = StructType(fields)
41    val frame: DataFrame = session.createDataFrame(rowRDD,NewSchema)
42
43    val Fdf = frame.filter(row=>{
44      var bool: Boolean =true
45      for(x<-(0 until num)){
46        bool=row.get(x).equals(schameARR(x))
47      }
48      !bool
49    }).toDF()
50
51    out.write(Fdf)
52
53  }
54
55  override def setProperties(map: Map[String, Any]): Unit = {
56    FolderPath = MapUtil.get(map,"csvPath").asInstanceOf[String]
57    delimiter = MapUtil.get(map,"delimiter").asInstanceOf[String]
58    schema = MapUtil.get(map,"schema").asInstanceOf[String]
59  }
60
61  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
62    var descriptor : List[PropertyDescriptor] = List()
63    val FolderPath = new PropertyDescriptor().name("FolderPath").displayName("FolderPath").description("The path of csv Folder").defaultValue("").required(true)
64    descriptor = FolderPath :: descriptor
65    val delimiter = new PropertyDescriptor().name("delimiter").displayName("delimiter").description("The delimiter of csv file").defaultValue("").required(true)
66    descriptor = delimiter :: descriptor
67    val schema = new PropertyDescriptor().name("schema").displayName("schema").description("CSV string field description information, please use, split").defaultValue("").required(true)
68    descriptor = schema :: descriptor
69    descriptor
70  }
71
72  override def getIcon(): Array[Byte] = {
73    ImageUtil.getImage("icon/csv/FolderCsvParser.png")
74  }
75
76  override def getGroup(): List[String] = {
77    List(StopGroup.CsvGroup.toString)
78  }
79
80  override def initialize(ctx: ProcessContext): Unit = {
81
82  }
83
84}