upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/common/MockData.scala
1package cn.piflow.bundle.generic.common
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8import org.apache.spark.sql.types._
9import org.json4s
10import org.json4s.JsonAST._
11import org.json4s.jackson.JsonMethods._
12
13import scala.util.Random
14
15class MockData extends ConfigurableStop{
16 override val authorEmail: String = "xjzhu@cnic.cn"
17 override val description: String = "Mock dataframe."
18 override val inportList: List[String] = List(Port.DefaultPort)
19 override val outportList: List[String] = List(Port.DefaultPort)
20
21 var schema: String = _
22 var count: Int = _
23
24 override def setProperties(map: Map[String, Any]): Unit = {
25 schema = MapUtil.get(map,"schema").asInstanceOf[String]
26 count = MapUtil.get(map,"count").asInstanceOf[String].toInt
27 }
28
29 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
30 var descriptor : List[PropertyDescriptor] = List()
31 val schema = new PropertyDescriptor()
32 .name("schema")
33 .displayName("Schema")
34 .description("The schema of mock data, format is column:columnType:isNullable. " +
35 "Separate multiple fields with commas. " +
36 "columnType can be String/Int/Long/Float/Double/Boolean. " +
37 "isNullable can be left blank, the default value is false. ")
38 .defaultValue("")
39 .required(true)
40 .example("id:String,name:String,age:Int")
41 descriptor = schema :: descriptor
42
43 val count = new PropertyDescriptor()
44 .name("count")
45 .displayName("Count")
46 .description("The count of dataframe")
47 .defaultValue("")
48 .required(true)
49 .example("10")
50 descriptor = count :: descriptor
51
52 descriptor
53 }
54
55 override def getIcon(): Array[Byte] = {
56 ImageUtil.getImage("icon/common/MockData.png")
57 }
58
59 override def getGroup(): List[String] = {
60 List(StopGroup.CommonGroup)
61 }
62
63 override def initialize(ctx: ProcessContext): Unit = {}
64
65 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
66 val spark = pec.get[SparkSession]()
67 import spark.implicits._
68
69 val field = this.schema.split(",")
70 val structFieldArray : Array[StructField] = new Array[StructField](field.size)
71 for(i <- 0 to field.size - 1){
72 val columnInfo = field(i).trim.split(":")
73 val column = columnInfo(0).trim
74 val columnType = columnInfo(1).trim
75 var isNullable = false
76 if(columnInfo.size == 3){
77 isNullable = columnInfo(2).trim.toBoolean
78 }
79 columnType match {
80 case "String"=> structFieldArray(i) = new StructField(column, StringType, isNullable)
81 case "Int"=>structFieldArray(i) = new StructField(column, IntegerType, isNullable)
82 case "Double"=>structFieldArray(i) = new StructField(column, DoubleType, isNullable)
83 case "Float"=>structFieldArray(i) = new StructField(column, FloatType, isNullable)
84 case "Long"=>structFieldArray(i) = new StructField(column, LongType, isNullable)
85 case "Boolean"=>structFieldArray(i) = new StructField(column, BooleanType, isNullable)
86 //case "Date"=>structFieldArray(i) = new StructField(column, DateType, nullable = true)
87 //case "Timestamp"=>structFieldArray(i) = new StructField(column, TimestampType, nullable = true)
88 }
89 }
90 val schemaStructType = StructType(structFieldArray)
91 val rnd : Random = new Random()
92 val df = spark.read.schema(schemaStructType).json((0 to count -1 ).map{ _ => compact(randomJson(rnd,schemaStructType))}.toDS())
93 out.write(df)
94 }
95
96 private def randomJson( rnd: Random, dataType : DataType): JValue ={
97
98 val alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
99 dataType match {
100 case v:DoubleType =>
101 json4s.JDouble(rnd.nextDouble())
102 case v:StringType =>
103 JString((1 to 10).map(x => alpha(Random.nextInt.abs % alpha.size)).mkString)
104 case v:IntegerType =>
105 JInt(rnd.nextInt(100))
106 case v:LongType =>
107 JInt(rnd.nextLong())
108 case v:FloatType =>
109 JDouble(rnd.nextFloat())
110 case v:BooleanType =>
111 JBool(rnd.nextBoolean())
112 case v:ArrayType =>
113 val size = rnd.nextInt(10)
114 JArray(
115 (0 to size).map(_ => randomJson(rnd, v.elementType)).toList
116 )
117 case v:StructType =>
118 JObject(
119 v.fields.flatMap{
120 f =>
121 if( f.nullable && rnd.nextBoolean())
122 None
123 else
124 Some(JField(f.name, randomJson(rnd, f.dataType)))
125 }.toList
126 )
127 }
128
129 }
130}