Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/SelectFilesByName.scala

1package cn.piflow.bundle.hdfs
2
3import java.util.regex.Pattern
4
5import cn.piflow.conf.bean.PropertyDescriptor
6import cn.piflow.conf.util.{ImageUtil, MapUtil}
7import cn.piflow.conf.{ConfigurableStop, Port, StopGroup}
8import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
9import org.apache.hadoop.conf.Configuration
10import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
11import org.apache.spark.rdd.RDD
12import org.apache.spark.sql.types.{StringType, StructField, StructType}
13import org.apache.spark.sql.{DataFrame, Row, SparkSession}
14
15import scala.collection.mutable.ArrayBuffer
16
17class SelectFilesByName extends ConfigurableStop{
18  override val authorEmail: String = "yangqidong@cnic.cn"
19  override val description: String = "Select files by file name"
20  override val inportList: List[String] = List(Port.DefaultPort)
21  override val outportList: List[String] = List(Port.DefaultPort)
22
23  var hdfsUrl:String=_
24  var hdfsPath:String=_
25  var selectionConditions:String =_
26
27  var fs: FileSystem=null
28  var pathARR:ArrayBuffer[String]=ArrayBuffer()
29  var selectArr:Array[String]=null
30
31  def selectFile(path: String): Unit = {
32    val statusesARR: Array[FileStatus] = fs.listStatus(new Path(path))
33    for(each <- statusesARR){
34      val pathStr = each.getPath.toString
35      if(each.isFile){
36        val fileName: String = pathStr.split("/").last
37        selectArr = selectionConditions.split(",").map(x => x.trim)
38        var b: Boolean =false
39        for(x <- selectArr){
40          b = Pattern.matches(x,fileName)
41          if(b){
42            pathARR += pathStr
43          }
44        }
45      }else{
46        selectFile(pathStr)
47      }
48    }
49  }
50
51  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
52
53    val session: SparkSession = pec.get[SparkSession]()
54
55    val configuration: Configuration = new Configuration()
56    configuration.set("fs.defaultFS", hdfsUrl)
57    fs = FileSystem.get(configuration)
58
59    selectFile(hdfsPath)
60
61    val rows: List[Row] = pathARR.map(each => {
62      var arr:Array[String]=Array(each)
63      val row: Row = Row.fromSeq(arr)
64      row
65    }).toList
66    val rowRDD: RDD[Row] = session.sparkContext.makeRDD(rows)
67    val fields: Array[StructField] = "path".split("/").map(d=>StructField(d,StringType,nullable = true))
68    val schema: StructType = StructType(fields)
69    val df: DataFrame = session.createDataFrame(rowRDD,schema)
70    df.collect().foreach(println)
71
72    out.write(df)
73  }
74
75  override def setProperties(map: Map[String, Any]): Unit = {
76    hdfsUrl=MapUtil.get(map,key="hdfsUrl").asInstanceOf[String]
77    hdfsPath=MapUtil.get(map,key="hdfsPath").asInstanceOf[String]
78    selectionConditions=MapUtil.get(map,key="selectionConditions").asInstanceOf[String]
79  }
80
81
82  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
83    var descriptor : List[PropertyDescriptor] = List()
84    val hdfsPath = new PropertyDescriptor()
85      .name("hdfsPath")
86      .displayName("HdfsPath")
87      .defaultValue("")
88      .description("File path of HDFS")
89      .required(true)
90      .example("/work/")
91    descriptor = hdfsPath :: descriptor
92
93    val hdfsUrl = new PropertyDescriptor()
94      .name("hdfsUrl")
95      .displayName("HdfsUrl")
96      .defaultValue("")
97      .description("URL address of HDFS")
98      .required(true)
99      .example("hdfs://192.168.3.138:8020")
100    descriptor = hdfsUrl :: descriptor
101
102    val selectionConditions = new PropertyDescriptor()
103      .name("selectionConditions")
104      .displayName("SelectionConditions")
105      .description("To select a conditions,a regular expression needs to be populated in java")
106      .defaultValue("")
107      .required(true)
108      .example(".*.csv")
109    descriptor = selectionConditions :: descriptor
110
111    descriptor
112  }
113
114  override def getIcon(): Array[Byte] = {
115    ImageUtil.getImage("icon/hdfs/SelectFileByName.png")
116  }
117
118  override def getGroup(): List[String] = {
119    List(StopGroup.HdfsGroup)
120  }
121
122  override def initialize(ctx: ProcessContext): Unit = {
123
124  }
125
126}