upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/neo4j/AllFieldsCleanNeo4j.scala
1package cn.piflow.bundle.neo4j
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import org.apache.spark.sql.{DataFrame, Row, SparkSession}
9import org.apache.spark.sql.types.StringType
10
11import scala.collection.mutable.ListBuffer
12
13class AllFieldsCleanNeo4j extends ConfigurableStop {
14 val authorEmail: String = "anhong12@cnic.cn"
15 val description: String = "Clean DataFrame for NSFC Neo4j"
16
17 val inportList: List[String] = List(Port.DefaultPort)
18 val outportList: List[String] = List(Port.DefaultPort)
19
20
21//
22// var regex:String=_
23// var replaceStr:String=_
24
25 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
26 val spark = pec.get[SparkSession]()
27
28 val df = in.read()
29 df.createOrReplaceTempView("temp")
30
31 // clean authors
32 spark.sqlContext.udf.register("cleanFields", (fieldStr: String) => {
33 if (fieldStr == null) null
34 else fieldStr.replaceAll("\"", "")
35 })
36
37
38 val fieldNames: Array[String] = df.schema.fieldNames
39 val sqlStringBuilder = new StringBuilder
40 for (i <- 0 until fieldNames.length) {
41 sqlStringBuilder.append("cleanFields(" + fieldNames(i) + ") as "+fieldNames(i) +",")
42 }
43
44 val sqlString = sqlStringBuilder.dropRight(1).toString()
45
46 val frame: DataFrame = spark.sql("select "+sqlString +" from temp")
47
48
49 //dfNew.show()
50 out.write(frame)
51 }
52
53 def initialize(ctx: ProcessContext): Unit = {
54
55 }
56
57
58 def setProperties(map: Map[String, Any]): Unit = {
59// regex=MapUtil.get(map,key="regex").asInstanceOf[String]
60// replaceStr=MapUtil.get(map,key="replaceStr").asInstanceOf[String]
61
62 }
63
64 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
65 var descriptor : List[PropertyDescriptor] = List()
66//
67// val regex = new PropertyDescriptor().name("regex").displayName("regex").description("the regex which to be repalced").defaultValue("\"").required(true)
68// val replaceStr = new PropertyDescriptor().name("repalceStr").displayName("replaceStr").description("repalace what").defaultValue("").required(true)
69// descriptor = regex :: descriptor
70// descriptor = replaceStr :: descriptor
71
72
73
74
75
76 descriptor
77 }
78
79 override def getIcon(): Array[Byte] = {
80 ImageUtil.getImage("icon/clean/TitleClean.png")
81 }
82
83 override def getGroup(): List[String] = {
84 List(StopGroup.Neo4jGroup)
85 }
86
87}