Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/file/RegexTextProcess.scala

1package cn.piflow.bundle.file
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
6import cn.piflow.conf._
7import org.apache.spark.sql.SparkSession
8
9class RegexTextProcess extends ConfigurableStop{
10    val authorEmail: String = "06whuxx@163.com"
11    val description: String = "Replace values in a column with regex"
12    val inportList: List[String] = List(Port.DefaultPort)
13    val outportList: List[String] = List(Port.DefaultPort)
14
15    var regex:String =_
16    var columnName:String=_
17    var replaceStr:String=_
18
19    def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20      val spark = pec.get[SparkSession]()
21      val sqlContext=spark.sqlContext
22      val dfOld = in.read()
23      val regexText=regex
24      val replaceText=replaceStr
25      dfOld.createOrReplaceTempView("thesis")
26      sqlContext.udf.register("regexPro",(str:String)=>str.replaceAll(regexText,replaceText))
27      val sqlText:String="select *,regexPro("+columnName+") as "+columnName+"_new from thesis"
28      val dfNew=sqlContext.sql(sqlText)
29      out.write(dfNew)
30    }
31
32  def initialize(ctx: ProcessContext): Unit = {
33
34  }
35
36
37  def setProperties(map: Map[String, Any]): Unit = {
38    regex=MapUtil.get(map,key="regex").asInstanceOf[String]
39    columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
40    replaceStr=MapUtil.get(map,key="replaceStr").asInstanceOf[String]
41
42  }
43
44  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
45    var descriptor : List[PropertyDescriptor] = List()
46    val regex = new PropertyDescriptor()
47      .name("regex")
48      .displayName("Regex")
49      .description("regex")
50      .defaultValue("")
51      .required(true)
52      .example("0001")
53    descriptor = regex :: descriptor
54
55    val columnName = new PropertyDescriptor()
56      .name("columnName")
57      .displayName("ColumnName")
58      .description("The columns you want to replace")
59      .defaultValue("")
60      .required(true)
61      .example("id")
62    descriptor = columnName :: descriptor
63
64    val replaceStr = new PropertyDescriptor()
65      .name("replaceStr")
66      .displayName("ReplaceStr")
67      .description("Value after replacement")
68      .defaultValue("")
69      .required(true)
70      .example("1111")
71    descriptor = replaceStr :: descriptor
72    descriptor
73  }
74
75  override def getIcon(): Array[Byte] = {
76    ImageUtil.getImage("icon/file/RegexTextProcess.png")
77  }
78
79  override def getGroup(): List[String] = {
80    List(StopGroup.FileGroup)
81  }
82}