Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/csv/CsvStringParser.scala

1package cn.piflow.bundle.generic.csv
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.SparkContext
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.types.{StringType, StructField, StructType}
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11
12class CsvStringParser extends ConfigurableStop{
13  override val authorEmail: String = "yangqidong@cnic.cn"
14  val inportList: List[String] = List(Port.DefaultPort)
15  val outportList: List[String] = List(Port.DefaultPort)
16  override val description: String = "Parse csv string"
17
18
19  var string:String=_
20  var delimiter: String = _
21  var schema: String = _
22
23  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24
25    val session: SparkSession = pec.get[SparkSession]
26    val context: SparkContext = session.sparkContext
27
28    val arrStr: Array[String] = string.split("\n").map(x => x.trim)
29
30    var num:Int=0
31    val listROW: List[Row] = arrStr.map(line => {
32      val seqSTR: Seq[String] = line.split(delimiter).map(x=>x.trim).toSeq
33      num=seqSTR.size
34      val row = Row.fromSeq(seqSTR)
35      row
36    }).toList
37    val rowRDD: RDD[Row] = context.makeRDD(listROW)
38
39    var Fdf: DataFrame =null
40    if(schema.length==0){
41      (0 until num).foreach(x=>{
42        schema+=("_"+x+",")
43      })
44    }
45    val fields: Array[StructField] = schema.split(",").map(d=>StructField(d.trim,StringType,nullable = true))
46    val NewSchema: StructType = StructType(fields)
47    Fdf = session.createDataFrame(rowRDD,NewSchema)
48    out.write(Fdf)
49  }
50
51
52  override def setProperties(map: Map[String, Any]): Unit = {
53    string = MapUtil.get(map,"string").asInstanceOf[String]
54    delimiter = MapUtil.get(map,"delimiter").asInstanceOf[String]
55    schema = MapUtil.get(map,"schema").asInstanceOf[String]
56  }
57
58  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
59    var descriptor : List[PropertyDescriptor] = List()
60
61    val string = new PropertyDescriptor()
62      .name("string")
63      .displayName("String")
64      .defaultValue("")
65      .required(true)
66      .example("1,zs\n2,ls\n3,ww")
67    descriptor = string :: descriptor
68
69    val delimiter = new PropertyDescriptor()
70      .name("delimiter")
71      .displayName("Delimiter")
72      .description("The delimiter of CSV string")
73      .defaultValue("")
74      .required(true)
75      .example(",")
76    descriptor = delimiter :: descriptor
77
78    val schema = new PropertyDescriptor()
79      .name("schema")
80      .displayName("Schema")
81      .description("The schema of CSV string")
82      .defaultValue("")
83      .required(false)
84      .example("")
85    descriptor = schema :: descriptor
86
87    descriptor
88
89  }
90
91  override def getIcon(): Array[Byte] = {
92    ImageUtil.getImage("icon/csv/CsvStringParser.png")
93  }
94
95  override def getGroup(): List[String] = {
96    List(StopGroup.CsvGroup)
97  }
98
99  override def initialize(ctx: ProcessContext): Unit = {
100
101  }
102
103}