Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/excel/ExcelWriteSort.scala

1package cn.piflow.bundle.excel
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
6import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
7import org.apache.spark.sql.functions.col
8
9
10class ExcelWriteSort extends ConfigurableStop{
11  val authorEmail: String = "ygang@cnic.cn"
12  val description: String = "Write DataFrame into an Excel file (custom sorting)"
13  val inportList: List[String] = List(Port.DefaultPort)
14  val outportList: List[String] = List(Port.DefaultPort)
15
16  var filePath: String = _
17  var header: String = _
18  var dataAddress: String = _
19  var saveMode: String = _
20  var sortField:String = _
21  var sortOrder:String = _
22
23  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24    var df = in.read()
25
26    val cols: Seq[String] = Seq(sortField.split(","): _*)
27
28    if(sortOrder.equals("asc")){
29      df  =  df.repartition(1).orderBy(cols.map(col).map(_.asc): _*)
30    } else {
31      df  =  df.repartition(1).orderBy(cols.map(col).map(_.desc): _*)
32    }
33    df.write
34      .format("com.crealytics.spark.excel")
35      .option("dataAddress",dataAddress)
36      //      .option("timestampFormat", timestampFormat)
37      .option("header", header)
38      .mode(saveMode)
39      .save(filePath)
40  }
41
42  override def setProperties(map: Map[String, Any]): Unit = {
43    filePath = MapUtil.get(map,"filePath").asInstanceOf[String]
44    header = MapUtil.get(map,"header").asInstanceOf[String]
45    dataAddress = MapUtil.get(map,"dataAddress").asInstanceOf[String]
46    saveMode = MapUtil.get(map,"saveMode").asInstanceOf[String]
47
48    sortField = MapUtil.get(map,"sortField").asInstanceOf[String]
49    sortOrder = MapUtil.get(map,"sortOrder").asInstanceOf[String]
50  }
51
52  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
53    var descriptor : List[PropertyDescriptor] = List()
54
55    val filePath = new PropertyDescriptor()
56      .name("filePath")
57      .displayName("FilePath")
58      .description("The path of excel file")
59      .defaultValue("")
60      .required(true)
61      .example("/test/test.xlsx")
62    descriptor = filePath :: descriptor
63
64    val header = new PropertyDescriptor()
65      .name("header")
66      .displayName("Header")
67      .description("Whether the excel file has a header")
68      .defaultValue("true")
69      .allowableValues(Set("true","false"))
70      .required(true)
71      .example("true")
72    descriptor = header :: descriptor
73
74    val dataAddress = new PropertyDescriptor()
75      .name("dataAddress")
76      .displayName("DataAddress")
77            .description("The location of data to read or write can be specified with the dataAddress option, Currently the following address styles are supported:" +
78              "A1: Start cell of the data. Reading will return all rows below and all columns to the right. Writing will start here and use as many columns and rows as required.\n" +
79              "'My Sheet'!A1: Same as above, but with a specific sheet.\n" +
80              "A1:F35: Cell range of data. Reading will return only rows and columns in the specified range. Writing will start in the first cell (B3 in this example) " +
81              "and use only the specified columns and rows. If there are more rows or columns in the DataFrame to write, they will be truncated. Make sure this is what you want.\n" +
82              "'My Sheet'!A1:F35: Same as above, but with a specific sheet.")
83      .defaultValue("A1")
84      .required(true)
85      .example("A1")
86    descriptor = dataAddress :: descriptor
87
88    val saveMode = new PropertyDescriptor()
89      .name("saveMode")
90      .displayName("saveMode")
91      .description("default: overwrite")
92      .defaultValue("overwrite")
93      .allowableValues(Set("append","overwrite"))
94      .required(true)
95      .example("overwrite")
96    descriptor = saveMode :: descriptor
97
98    val sortField = new PropertyDescriptor()
99      .name("sortField")
100      .displayName("sortField")
101      .description("排序字段")
102      .defaultValue("")
103      .required(true)
104      .example("id")
105    descriptor = sortField :: descriptor
106
107    val sortOrder = new PropertyDescriptor()
108      .name("sortOrder")
109      .displayName("sortOrder")
110      .description("排序方式:默认升序")
111      .defaultValue("asc")
112      .allowableValues(Set("asc","desc"))
113      .required(true)
114      .example("asc")
115    descriptor = sortOrder :: descriptor
116
117    descriptor
118  }
119
120  override def getIcon(): Array[Byte] = {
121    ImageUtil.getImage("icon/excel/excelParse.png")
122  }
123
124  override def getGroup(): List[String] = {
125    List(StopGroup.ExcelGroup)
126  }
127
128  override def initialize(ctx: ProcessContext): Unit = {
129
130  }
131
132}