upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/file/RegexTextProcess.scala
1package cn.piflow.bundle.file
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil}
5import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
6import cn.piflow.conf._
7import org.apache.spark.sql.SparkSession
8
9class RegexTextProcess extends ConfigurableStop{
10 val authorEmail: String = "06whuxx@163.com"
11 val description: String = "Replace values in a column with regex"
12 val inportList: List[String] = List(Port.DefaultPort)
13 val outportList: List[String] = List(Port.DefaultPort)
14
15 var regex:String =_
16 var columnName:String=_
17 var replaceStr:String=_
18
19 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
20 val spark = pec.get[SparkSession]()
21 val sqlContext=spark.sqlContext
22 val dfOld = in.read()
23 val regexText=regex
24 val replaceText=replaceStr
25 dfOld.createOrReplaceTempView("thesis")
26 sqlContext.udf.register("regexPro",(str:String)=>str.replaceAll(regexText,replaceText))
27 val sqlText:String="select *,regexPro("+columnName+") as "+columnName+"_new from thesis"
28 val dfNew=sqlContext.sql(sqlText)
29 out.write(dfNew)
30 }
31
32 def initialize(ctx: ProcessContext): Unit = {
33
34 }
35
36
37 def setProperties(map: Map[String, Any]): Unit = {
38 regex=MapUtil.get(map,key="regex").asInstanceOf[String]
39 columnName=MapUtil.get(map,key="columnName").asInstanceOf[String]
40 replaceStr=MapUtil.get(map,key="replaceStr").asInstanceOf[String]
41
42 }
43
44 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
45 var descriptor : List[PropertyDescriptor] = List()
46 val regex = new PropertyDescriptor()
47 .name("regex")
48 .displayName("Regex")
49 .description("regex")
50 .defaultValue("")
51 .required(true)
52 .example("0001")
53 descriptor = regex :: descriptor
54
55 val columnName = new PropertyDescriptor()
56 .name("columnName")
57 .displayName("ColumnName")
58 .description("The columns you want to replace")
59 .defaultValue("")
60 .required(true)
61 .example("id")
62 descriptor = columnName :: descriptor
63
64 val replaceStr = new PropertyDescriptor()
65 .name("replaceStr")
66 .displayName("ReplaceStr")
67 .description("Value after replacement")
68 .defaultValue("")
69 .required(true)
70 .example("1111")
71 descriptor = replaceStr :: descriptor
72 descriptor
73 }
74
75 override def getIcon(): Array[Byte] = {
76 ImageUtil.getImage("icon/file/RegexTextProcess.png")
77 }
78
79 override def getGroup(): List[String] = {
80 List(StopGroup.FileGroup)
81 }
82}