Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/common/DataSort.scala

1package cn.piflow.bundle.common
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.SparkSession
8import org.apache.spark.sql.functions.col
9
10class DataSort extends ConfigurableStop{
11  val authorEmail: String = "ygang@cnic.cn"
12  val description: String = "数据排序"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] =  List(Port.DefaultPort)
15
16  var sortField: String = _
17  var sortOrder:String = _
18
19  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20    val spark = pec.get[SparkSession]()
21    var dataSortDF = in.read()
22
23    val cols: Seq[String] = Seq(sortField.split(","): _*)
24
25    if(sortOrder.equals("asc")){
26      dataSortDF  =  dataSortDF.repartition(1).orderBy(cols.map(col).map(_.asc): _*)
27    } else {
28      dataSortDF  =  dataSortDF.repartition(1).orderBy(cols.map(col).map(_.desc): _*)
29    }
30
31    out.write(dataSortDF)
32
33  }
34
35  override def setProperties(map: Map[String, Any]): Unit = {
36    sortField = MapUtil.get(map,"sortField").asInstanceOf[String]
37    sortOrder = MapUtil.get(map,"sortOrder").asInstanceOf[String]
38  }
39
40  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
41    var descriptor : List[PropertyDescriptor] = List()
42
43    val sortField = new PropertyDescriptor()
44      .name("sortField")
45      .displayName("sortField")
46      .description("排序字段")
47      .defaultValue("")
48      .required(true)
49      .example("id")
50    descriptor = sortField :: descriptor
51
52    val sortOrder = new PropertyDescriptor()
53      .name("sortOrder")
54      .displayName("sortOrder")
55      .description("排序方式:默认升序")
56      .defaultValue("asc")
57      .allowableValues(Set("asc","desc"))
58      .required(true)
59      .example("asc")
60    descriptor = sortOrder :: descriptor
61
62    descriptor
63  }
64
65  override def getIcon(): Array[Byte] = {
66    ImageUtil.getImage("icon/csv/CsvParser.png")
67  }
68
69  override def getGroup(): List[String] = {
70    List(StopGroup.CommonGroup)
71  }
72
73  override def initialize(ctx: ProcessContext): Unit = {
74
75  }
76
77}