upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/csv/FolderCsvParser.scala
1package cn.piflow.bundle.csv
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, PortEnum, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.SparkContext
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.types.{StringType, StructField, StructType}
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11
12class FolderCsvParser extends ConfigurableStop{
13 override val authorEmail: String = "yangqidong@cnic.cn"
14 val inportList: List[String] = List(PortEnum.NonePort.toString)
15 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
16 override val description: String = "Parse csv folder"
17
18 var FolderPath:String=_
19 var delimiter: String = _
20 var schema: String = _
21
22 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
23 val session: SparkSession = pec.get[SparkSession]
24 val context: SparkContext = session.sparkContext
25
26 val StrRDD: RDD[String] = context.textFile(FolderPath)
27 var num:Int=0
28 var s:String=delimiter
29
30 val rowRDD: RDD[Row] = StrRDD.map(line => {
31 val seqSTR: Seq[String] = line.split(s).toSeq
32 num=seqSTR.size
33 val row = Row.fromSeq(seqSTR)
34 row
35 })
36
37 val schameARR: Array[String] = schema.split(",")
38 val fields: Array[StructField] = schameARR.map(d=>StructField(d.trim,StringType,nullable = true))
39
40 val NewSchema: StructType = StructType(fields)
41 val frame: DataFrame = session.createDataFrame(rowRDD,NewSchema)
42
43 val Fdf = frame.filter(row=>{
44 var bool: Boolean =true
45 for(x<-(0 until num)){
46 bool=row.get(x).equals(schameARR(x))
47 }
48 !bool
49 }).toDF()
50
51 out.write(Fdf)
52
53 }
54
55 override def setProperties(map: Map[String, Any]): Unit = {
56 FolderPath = MapUtil.get(map,"csvPath").asInstanceOf[String]
57 delimiter = MapUtil.get(map,"delimiter").asInstanceOf[String]
58 schema = MapUtil.get(map,"schema").asInstanceOf[String]
59 }
60
61 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
62 var descriptor : List[PropertyDescriptor] = List()
63 val FolderPath = new PropertyDescriptor().name("FolderPath").displayName("FolderPath").description("The path of csv Folder").defaultValue("").required(true)
64 descriptor = FolderPath :: descriptor
65 val delimiter = new PropertyDescriptor().name("delimiter").displayName("delimiter").description("The delimiter of csv file").defaultValue("").required(true)
66 descriptor = delimiter :: descriptor
67 val schema = new PropertyDescriptor().name("schema").displayName("schema").description("CSV string field description information, please use, split").defaultValue("").required(true)
68 descriptor = schema :: descriptor
69 descriptor
70 }
71
72 override def getIcon(): Array[Byte] = {
73 ImageUtil.getImage("icon/csv/FolderCsvParser.png")
74 }
75
76 override def getGroup(): List[String] = {
77 List(StopGroup.CsvGroup.toString)
78 }
79
80 override def initialize(ctx: ProcessContext): Unit = {
81
82 }
83
84}