Source code

Download source archive ↓

src/AddDayID.scala

1package cn.piflow.bundle.flux.pretreatment
2
3
4import cn.piflow.bundle.flux.util.FluxUtil_1
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.spark.sql.SparkSession
10
11class AddDayID extends ConfigurableStop{
12  val authorEmail: String = "ygang@cnic.cn"
13  val description: String = "按照year、month、day、hour 添加自增id,且根据year、month、day添加 day_id,"
14  val inportList: List[String] = List(Port.DefaultPort)
15  val outportList: List[String] = List(Port.DefaultPort)
16
17  var partitionNum: String = _
18
19  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20    val spark = pec.get[SparkSession]()
21
22//    val partitionNum = in.read().rdd.getNumPartitions
23
24    val flux = new FluxUtil_1
25    val df = flux.AddIdWithDay(spark, in.read(),partitionNum.toInt)
26
27    out.write(df)
28  }
29
30  override def setProperties(map: Map[String, Any]): Unit = {
31    partitionNum = MapUtil.get(map,"partitionNum").asInstanceOf[String]
32  }
33
34  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
35    var descriptor : List[PropertyDescriptor] = List()
36
37    val partitionNum = new PropertyDescriptor()
38      .name("partitionNum")
39      .displayName("partitionNum")
40      .description("Set the number of partitions")
41      .defaultValue("20")
42      .required(true)
43      .example("20")
44    descriptor = partitionNum :: descriptor
45
46    descriptor
47  }
48
49  override def getIcon(): Array[Byte] = {
50    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
51  }
52
53  override def getGroup(): List[String] = {
54    List("Flux_pretreatment")
55  }
56
57  override def initialize(ctx: ProcessContext): Unit = {
58
59  }
60
61}