Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/neo4j/AllFieldsCleanNeo4j.scala

1package cn.piflow.bundle.neo4j
2
3import cn.piflow.bundle.util.CleanUtil
4import cn.piflow.conf._
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import org.apache.spark.sql.{DataFrame, Row, SparkSession}
9import org.apache.spark.sql.types.StringType
10
11import scala.collection.mutable.ListBuffer
12
13class AllFieldsCleanNeo4j extends ConfigurableStop {
14  val authorEmail: String = "anhong12@cnic.cn"
15  val description: String = "Clean DataFrame for NSFC Neo4j"
16
17  val inportList: List[String] = List(Port.DefaultPort)
18  val outportList: List[String] = List(Port.DefaultPort)
19
20
21//
22//  var regex:String=_
23//  var replaceStr:String=_
24
25  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
26    val spark = pec.get[SparkSession]()
27
28    val df = in.read()
29    df.createOrReplaceTempView("temp")
30
31    // clean authors
32    spark.sqlContext.udf.register("cleanFields", (fieldStr: String) => {
33      if (fieldStr == null) null
34      else fieldStr.replaceAll("\"", "")
35    })
36
37
38    val fieldNames: Array[String] = df.schema.fieldNames
39    val sqlStringBuilder = new StringBuilder
40    for (i <- 0 until fieldNames.length) {
41      sqlStringBuilder.append("cleanFields(" + fieldNames(i) + ") as "+fieldNames(i) +",")
42    }
43
44    val sqlString = sqlStringBuilder.dropRight(1).toString()
45
46    val frame: DataFrame = spark.sql("select   "+sqlString +"   from temp")
47
48
49    //dfNew.show()
50    out.write(frame)
51  }
52
53  def initialize(ctx: ProcessContext): Unit = {
54
55  }
56
57
58  def setProperties(map: Map[String, Any]): Unit = {
59//    regex=MapUtil.get(map,key="regex").asInstanceOf[String]
60//    replaceStr=MapUtil.get(map,key="replaceStr").asInstanceOf[String]
61
62  }
63
64  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
65    var descriptor : List[PropertyDescriptor] = List()
66//
67//    val regex = new PropertyDescriptor().name("regex").displayName("regex").description("the regex which to be repalced").defaultValue("\"").required(true)
68//    val replaceStr = new PropertyDescriptor().name("repalceStr").displayName("replaceStr").description("repalace what").defaultValue("").required(true)
69//    descriptor = regex :: descriptor
70//    descriptor = replaceStr :: descriptor
71
72
73
74
75
76    descriptor
77  }
78
79  override def getIcon(): Array[Byte] = {
80    ImageUtil.getImage("icon/clean/TitleClean.png")
81  }
82
83  override def getGroup(): List[String] = {
84    List(StopGroup.Neo4jGroup)
85  }
86
87}