upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/common/DataSort.scala
1package cn.piflow.bundle.common
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.SparkSession
8import org.apache.spark.sql.functions.col
9
10class DataSort extends ConfigurableStop{
11 val authorEmail: String = "ygang@cnic.cn"
12 val description: String = "数据排序"
13 val inportList: List[String] = List(Port.DefaultPort)
14 val outportList: List[String] = List(Port.DefaultPort)
15
16 var sortField: String = _
17 var sortOrder:String = _
18
19 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20 val spark = pec.get[SparkSession]()
21 var dataSortDF = in.read()
22
23 val cols: Seq[String] = Seq(sortField.split(","): _*)
24
25 if(sortOrder.equals("asc")){
26 dataSortDF = dataSortDF.repartition(1).orderBy(cols.map(col).map(_.asc): _*)
27 } else {
28 dataSortDF = dataSortDF.repartition(1).orderBy(cols.map(col).map(_.desc): _*)
29 }
30
31 out.write(dataSortDF)
32
33 }
34
35 override def setProperties(map: Map[String, Any]): Unit = {
36 sortField = MapUtil.get(map,"sortField").asInstanceOf[String]
37 sortOrder = MapUtil.get(map,"sortOrder").asInstanceOf[String]
38 }
39
40 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
41 var descriptor : List[PropertyDescriptor] = List()
42
43 val sortField = new PropertyDescriptor()
44 .name("sortField")
45 .displayName("sortField")
46 .description("排序字段")
47 .defaultValue("")
48 .required(true)
49 .example("id")
50 descriptor = sortField :: descriptor
51
52 val sortOrder = new PropertyDescriptor()
53 .name("sortOrder")
54 .displayName("sortOrder")
55 .description("排序方式:默认升序")
56 .defaultValue("asc")
57 .allowableValues(Set("asc","desc"))
58 .required(true)
59 .example("asc")
60 descriptor = sortOrder :: descriptor
61
62 descriptor
63 }
64
65 override def getIcon(): Array[Byte] = {
66 ImageUtil.getImage("icon/csv/CsvParser.png")
67 }
68
69 override def getGroup(): List[String] = {
70 List(StopGroup.CommonGroup)
71 }
72
73 override def initialize(ctx: ProcessContext): Unit = {
74
75 }
76
77}