upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/csv/CsvStringParser.scala
1package cn.piflow.bundle.generic.csv
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.SparkContext
8import org.apache.spark.rdd.RDD
9import org.apache.spark.sql.types.{StringType, StructField, StructType}
10import org.apache.spark.sql.{DataFrame, Row, SparkSession}
11
12class CsvStringParser extends ConfigurableStop{
13 override val authorEmail: String = "yangqidong@cnic.cn"
14 val inportList: List[String] = List(Port.DefaultPort)
15 val outportList: List[String] = List(Port.DefaultPort)
16 override val description: String = "Parse csv string"
17
18
19 var string:String=_
20 var delimiter: String = _
21 var schema: String = _
22
23 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24
25 val session: SparkSession = pec.get[SparkSession]
26 val context: SparkContext = session.sparkContext
27
28 val arrStr: Array[String] = string.split("\n").map(x => x.trim)
29
30 var num:Int=0
31 val listROW: List[Row] = arrStr.map(line => {
32 val seqSTR: Seq[String] = line.split(delimiter).map(x=>x.trim).toSeq
33 num=seqSTR.size
34 val row = Row.fromSeq(seqSTR)
35 row
36 }).toList
37 val rowRDD: RDD[Row] = context.makeRDD(listROW)
38
39 var Fdf: DataFrame =null
40 if(schema.length==0){
41 (0 until num).foreach(x=>{
42 schema+=("_"+x+",")
43 })
44 }
45 val fields: Array[StructField] = schema.split(",").map(d=>StructField(d.trim,StringType,nullable = true))
46 val NewSchema: StructType = StructType(fields)
47 Fdf = session.createDataFrame(rowRDD,NewSchema)
48 out.write(Fdf)
49 }
50
51
52 override def setProperties(map: Map[String, Any]): Unit = {
53 string = MapUtil.get(map,"string").asInstanceOf[String]
54 delimiter = MapUtil.get(map,"delimiter").asInstanceOf[String]
55 schema = MapUtil.get(map,"schema").asInstanceOf[String]
56 }
57
58 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
59 var descriptor : List[PropertyDescriptor] = List()
60
61 val string = new PropertyDescriptor()
62 .name("string")
63 .displayName("String")
64 .defaultValue("")
65 .required(true)
66 .example("1,zs\n2,ls\n3,ww")
67 descriptor = string :: descriptor
68
69 val delimiter = new PropertyDescriptor()
70 .name("delimiter")
71 .displayName("Delimiter")
72 .description("The delimiter of CSV string")
73 .defaultValue("")
74 .required(true)
75 .example(",")
76 descriptor = delimiter :: descriptor
77
78 val schema = new PropertyDescriptor()
79 .name("schema")
80 .displayName("Schema")
81 .description("The schema of CSV string")
82 .defaultValue("")
83 .required(false)
84 .example("")
85 descriptor = schema :: descriptor
86
87 descriptor
88
89 }
90
91 override def getIcon(): Array[Byte] = {
92 ImageUtil.getImage("icon/csv/CsvStringParser.png")
93 }
94
95 override def getGroup(): List[String] = {
96 List(StopGroup.CsvGroup)
97 }
98
99 override def initialize(ctx: ProcessContext): Unit = {
100
101 }
102
103}