upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/SelectFilesByName.scala
1package cn.piflow.bundle.hdfs
2
3import java.util.regex.Pattern
4
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.hadoop.conf.Configuration
10import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
11import org.apache.spark.rdd.RDD
12import org.apache.spark.sql.types.{StringType, StructField, StructType}
13import org.apache.spark.sql.{DataFrame, Row, SparkSession}
14
15import scala.collection.mutable.ArrayBuffer
16
17class SelectFilesByName extends ConfigurableStop{
18 override val authorEmail: String = "yangqidong@cnic.cn"
19 override val description: String = "Select files by file name"
20 override val inportList: List[String] = List(Port.DefaultPort)
21 override val outportList: List[String] = List(Port.DefaultPort)
22
23 var hdfsUrl:String=_
24 var hdfsPath:String=_
25 var selectionConditions:String =_
26
27 var fs: FileSystem=null
28 var pathARR:ArrayBuffer[String]=ArrayBuffer()
29 var selectArr:Array[String]=null
30
31 def selectFile(path: String): Unit = {
32 val statusesARR: Array[FileStatus] = fs.listStatus(new Path(path))
33 for(each <- statusesARR){
34 val pathStr = each.getPath.toString
35 if(each.isFile){
36 val fileName: String = pathStr.split("/").last
37 selectArr = selectionConditions.split(",").map(x => x.trim)
38 var b: Boolean =false
39 for(x <- selectArr){
40 b = Pattern.matches(x,fileName)
41 if(b){
42 pathARR += pathStr
43 }
44 }
45 }else{
46 selectFile(pathStr)
47 }
48 }
49 }
50
51 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
52
53 val session: SparkSession = pec.get[SparkSession]()
54
55 val configuration: Configuration = new Configuration()
56 configuration.set("fs.defaultFS", hdfsUrl)
57 fs = FileSystem.get(configuration)
58
59 selectFile(hdfsPath)
60
61 val rows: List[Row] = pathARR.map(each => {
62 var arr:Array[String]=Array(each)
63 val row: Row = Row.fromSeq(arr)
64 row
65 }).toList
66 val rowRDD: RDD[Row] = session.sparkContext.makeRDD(rows)
67 val fields: Array[StructField] = "path".split("/").map(d=>StructField(d,StringType,nullable = true))
68 val schema: StructType = StructType(fields)
69 val df: DataFrame = session.createDataFrame(rowRDD,schema)
70 df.collect().foreach(println)
71
72 out.write(df)
73 }
74
75 override def setProperties(map: Map[String, Any]): Unit = {
76 hdfsUrl=MapUtil.get(map,key="hdfsUrl").asInstanceOf[String]
77 hdfsPath=MapUtil.get(map,key="hdfsPath").asInstanceOf[String]
78 selectionConditions=MapUtil.get(map,key="selectionConditions").asInstanceOf[String]
79 }
80
81
82 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
83 var descriptor : List[PropertyDescriptor] = List()
84 val hdfsPath = new PropertyDescriptor()
85 .name("hdfsPath")
86 .displayName("HdfsPath")
87 .defaultValue("")
88 .description("File path of HDFS")
89 .required(true)
90 .example("/work/")
91 descriptor = hdfsPath :: descriptor
92
93 val hdfsUrl = new PropertyDescriptor()
94 .name("hdfsUrl")
95 .displayName("HdfsUrl")
96 .defaultValue("")
97 .description("URL address of HDFS")
98 .required(true)
99 .example("hdfs://192.168.3.138:8020")
100 descriptor = hdfsUrl :: descriptor
101
102 val selectionConditions = new PropertyDescriptor()
103 .name("selectionConditions")
104 .displayName("SelectionConditions")
105 .description("To select a conditions,a regular expression needs to be populated in java")
106 .defaultValue("")
107 .required(true)
108 .example(".*.csv")
109 descriptor = selectionConditions :: descriptor
110
111 descriptor
112 }
113
114 override def getIcon(): Array[Byte] = {
115 ImageUtil.getImage("icon/hdfs/SelectFileByName.png")
116 }
117
118 override def getGroup(): List[String] = {
119 List(StopGroup.HdfsGroup)
120 }
121
122 override def initialize(ctx: ProcessContext): Unit = {
123
124 }
125
126}