Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/generic/common/MockData.scala

1package cn.piflow.bundle.generic.common
2
3import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
4import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import org.apache.spark.sql.{DataFrame, SparkSession}
8import org.apache.spark.sql.types._
9import org.json4s
10import org.json4s.JsonAST._
11import org.json4s.jackson.JsonMethods._
12
13import scala.util.Random
14
15class MockData extends ConfigurableStop{
16  override val authorEmail: String = "xjzhu@cnic.cn"
17  override val description: String = "Mock dataframe."
18  override val inportList: List[String] = List(Port.DefaultPort)
19  override val outportList: List[String] = List(Port.DefaultPort)
20
21  var schema: String = _
22  var count: Int = _
23
24  override def setProperties(map: Map[String, Any]): Unit = {
25    schema = MapUtil.get(map,"schema").asInstanceOf[String]
26    count = MapUtil.get(map,"count").asInstanceOf[String].toInt
27  }
28
29  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
30    var descriptor : List[PropertyDescriptor] = List()
31    val schema = new PropertyDescriptor()
32      .name("schema")
33      .displayName("Schema")
34      .description("The schema of mock data, format is column:columnType:isNullable. " +
35        "Separate multiple fields with commas. " +
36        "columnType can be String/Int/Long/Float/Double/Boolean. " +
37        "isNullable can be left blank, the default value is false. ")
38      .defaultValue("")
39      .required(true)
40      .example("id:String,name:String,age:Int")
41    descriptor = schema :: descriptor
42
43    val count = new PropertyDescriptor()
44      .name("count")
45      .displayName("Count")
46      .description("The count of dataframe")
47      .defaultValue("")
48      .required(true)
49      .example("10")
50    descriptor = count :: descriptor
51
52    descriptor
53  }
54
55  override def getIcon(): Array[Byte] = {
56    ImageUtil.getImage("icon/common/MockData.png")
57  }
58
59  override def getGroup(): List[String] = {
60    List(StopGroup.CommonGroup)
61  }
62
63  override def initialize(ctx: ProcessContext): Unit = {}
64
65  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
66    val spark = pec.get[SparkSession]()
67    import spark.implicits._
68
69    val field = this.schema.split(",")
70    val structFieldArray : Array[StructField] = new Array[StructField](field.size)
71    for(i <- 0 to field.size - 1){
72      val columnInfo = field(i).trim.split(":")
73      val column = columnInfo(0).trim
74      val columnType = columnInfo(1).trim
75      var isNullable = false
76      if(columnInfo.size == 3){
77        isNullable = columnInfo(2).trim.toBoolean
78      }
79      columnType match {
80        case "String"=> structFieldArray(i) = new StructField(column, StringType, isNullable)
81        case "Int"=>structFieldArray(i) = new StructField(column, IntegerType, isNullable)
82        case "Double"=>structFieldArray(i) = new StructField(column, DoubleType, isNullable)
83        case "Float"=>structFieldArray(i) = new StructField(column, FloatType, isNullable)
84        case "Long"=>structFieldArray(i) = new StructField(column, LongType, isNullable)
85        case "Boolean"=>structFieldArray(i) = new StructField(column, BooleanType, isNullable)
86        //case "Date"=>structFieldArray(i) = new StructField(column, DateType, nullable = true)
87        //case "Timestamp"=>structFieldArray(i) = new StructField(column, TimestampType, nullable = true)
88      }
89    }
90    val schemaStructType = StructType(structFieldArray)
91    val rnd : Random = new Random()
92    val df = spark.read.schema(schemaStructType).json((0 to count -1 ).map{ _ => compact(randomJson(rnd,schemaStructType))}.toDS())
93    out.write(df)
94  }
95
96  private def randomJson( rnd: Random, dataType : DataType): JValue ={
97
98    val alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
99    dataType match {
100      case v:DoubleType =>
101        json4s.JDouble(rnd.nextDouble())
102      case v:StringType =>
103        JString((1 to 10).map(x => alpha(Random.nextInt.abs % alpha.size)).mkString)
104      case v:IntegerType =>
105        JInt(rnd.nextInt(100))
106      case v:LongType =>
107        JInt(rnd.nextLong())
108      case v:FloatType =>
109        JDouble(rnd.nextFloat())
110      case v:BooleanType =>
111        JBool(rnd.nextBoolean())
112      case v:ArrayType =>
113        val size = rnd.nextInt(10)
114        JArray(
115          (0 to size).map(_ => randomJson(rnd, v.elementType)).toList
116        )
117      case v:StructType =>
118        JObject(
119          v.fields.flatMap{
120            f =>
121              if( f.nullable && rnd.nextBoolean())
122                None
123              else
124                Some(JField(f.name, randomJson(rnd, f.dataType)))
125          }.toList
126        )
127    }
128
129  }
130}