Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/flux/pretreatment/AddDayID.scala

1package cn.piflow.bundle.flux.pretreatment
2
3import cn.piflow.bundle.flux.util.FluxUtil_1
4import cn.piflow.conf.bean.PropertyDescriptor
5import cn.piflow.conf.util.{ImageUtil, MapUtil}
6import cn.piflow.conf.{ConfigurableStop, Port}
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import org.apache.spark.sql.SparkSession
9
10class AddDayID extends ConfigurableStop{
11  val authorEmail: String = "ygang@cnic.cn"
12  val description: String = "按照year、month、day、hour 添加自增id,且根据year、month、day添加 day_id,"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15
16  var partitionNum: String = _
17
18  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
19    val spark = pec.get[SparkSession]()
20
21//    val partitionNum = in.read().rdd.getNumPartitions
22
23    val flux = new FluxUtil_1
24    val df = flux.AddIdWithDay(spark, in.read(),partitionNum.toInt)
25
26    out.write(df)
27  }
28
29  override def setProperties(map: Map[String, Any]): Unit = {
30    partitionNum = MapUtil.get(map,"partitionNum").asInstanceOf[String]
31  }
32
33  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
34    var descriptor : List[PropertyDescriptor] = List()
35
36    val partitionNum = new PropertyDescriptor()
37      .name("partitionNum")
38      .displayName("partitionNum")
39      .description("Set the number of partitions")
40      .defaultValue("20")
41      .required(true)
42      .example("20")
43    descriptor = partitionNum :: descriptor
44
45    descriptor
46  }
47
48  override def getIcon(): Array[Byte] = {
49    ImageUtil.getImage("png/Flux/flux.png",this.getClass.getName)
50  }
51
52  override def getGroup(): List[String] = {
53    List("Flux_pretreatment")
54  }
55
56  override def initialize(ctx: ProcessContext): Unit = {
57
58  }
59
60}