Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/UnzipFilesOnHDFS.scala

1package cn.piflow.bundle.hdfs
2
3import java.io._
4import java.util.zip.GZIPInputStream
5
6import cn.piflow.conf._
7import cn.piflow.conf.bean.PropertyDescriptor
8import cn.piflow.conf.util.{ImageUtil, MapUtil}
9import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
10import org.apache.hadoop.conf.Configuration
11import org.apache.hadoop.fs.{FSDataInputStream, FSDataOutputStream, FileSystem, Path}
12import org.apache.spark.rdd.RDD
13import org.apache.spark.sql.types.{StringType, StructField, StructType}
14import org.apache.spark.sql.{DataFrame, Row, SparkSession}
15import org.apache.tools.tar.{TarEntry, TarInputStream}
16
17import scala.collection.mutable.ArrayBuffer
18
19class UnzipFilesOnHDFS extends ConfigurableStop {
20  val authorEmail: String = "yangqidong@cnic.cn"
21  val description: String = "Unzip files on hdfs"
22  val inportList: List[String] = List(PortEnum.DefaultPort.toString)
23  val outportList: List[String] = List(PortEnum.DefaultPort.toString)
24
25  var isCustomize:String=_
26  var hdfsUrl:String=_
27  var filePath:String=_
28  var savePath:String=_
29
30  var session: SparkSession = null
31  var arr:ArrayBuffer[Row]=ArrayBuffer()
32
33
34  def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
35
36    session = pec.get[SparkSession]()
37
38
39    if(isCustomize.equals("true")){
40
41      unzipFile(hdfsUrl+filePath,savePath)
42
43    }else if (isCustomize .equals("false")){
44
45      val inDf: DataFrame = in.read()
46      inDf.collect().foreach(row => {
47        filePath = row.get(0).asInstanceOf[String]
48        unzipFile(filePath,savePath)
49      })
50    }
51
52    val rdd: RDD[Row] = session.sparkContext.makeRDD(arr.toList)
53    val fields: Array[StructField] =Array(StructField("savePath",StringType,nullable = true))
54    val schema: StructType = StructType(fields)
55    val df: DataFrame = session.createDataFrame(rdd,schema)
56
57    out.write(df)
58
59  }
60
61
62  def whatType(p:String): String = {
63    var typeStr:String=""
64    val pathNames: Array[String] = p.split("\\.")
65    val lastStr: String = pathNames.last
66    if(lastStr.equals("gz")){
67      val penultStr: String = pathNames(pathNames.length-2)
68      if(penultStr.equals("tar")){
69        typeStr="tar.gz"
70      }else {
71        typeStr="gz"
72      }
73  }else{
74      throw new RuntimeException("File type fill in error, or do not support this type.")
75    }
76    typeStr
77  }
78
79
80  def getFs(fileHdfsPath: String): FileSystem = {
81    var configuration: Configuration = new Configuration()
82    var fs: FileSystem =null
83    if (isCustomize.equals("false")) {
84      val pathARR: Array[String] = fileHdfsPath.split("\\/")
85      hdfsUrl = ""
86      for (x <- (0 until 3)) {
87        hdfsUrl += (pathARR(x) + "/")
88      }
89    }
90    configuration.set("fs.defaultFS", hdfsUrl)
91    fs = FileSystem.get(configuration)
92    fs
93  }
94
95
96
97
98  def unzipFile(fileHdfsPath: String, saveHdfsPath: String)= {
99    var eachSavePath : String=""
100
101    var unType: String = whatType(fileHdfsPath)
102    var fileName: String = fileHdfsPath.split("\\/").last
103    var fs: FileSystem= getFs(fileHdfsPath)
104
105    var sp:String=""
106    if(saveHdfsPath.length < 1){
107      sp=fileHdfsPath.replace(fileName,"")
108    }else{
109      sp = hdfsUrl + saveHdfsPath
110    }
111
112    val fdis: FSDataInputStream = fs.open(new Path(fileHdfsPath))
113
114
115    if(unType.equals("gz")){
116      val gzip: GZIPInputStream = new GZIPInputStream(fdis)
117      var n = -1
118      val buf=new Array[Byte](10*1024*1024)
119
120      eachSavePath = sp +fileName.replace(".gz","")
121      arr += Row.fromSeq(Array(eachSavePath))
122      val path = new Path(eachSavePath)
123      val fdos = fs.create(path)
124      while((n=gzip.read(buf)) != -1 && n != -1){
125        fdos.write(buf,0,n)
126        fdos.flush()
127      }
128      fdos.close()
129      gzip.close()
130      fdis.close()
131    }else if(unType.equals("tar.gz")){
132
133      try {
134        val gzip = new GZIPInputStream(new BufferedInputStream(fdis))
135        val tarIn = new TarInputStream(gzip, 1024 * 2)
136
137        var entry: TarEntry = null
138
139        while ((entry = tarIn.getNextEntry) != null  && entry !=null) {
140
141          if (entry.isDirectory()) {
142            val outPath = sp + "/" + entry.getName
143            fs.create(new Path(outPath)).close()
144
145          } else {
146            val outPath = sp + "/" + entry.getName
147
148            arr += Row.fromSeq(Array(outPath))
149            val fos: FSDataOutputStream = fs.create(new Path(outPath))
150
151            var lenth = 0
152            val buff = new Array[Byte](1024)
153            while ((lenth = tarIn.read(buff)) != -1 && (lenth != -1)) {
154              fos.write(buff, 0, lenth)
155            }
156            fos.close()
157          }
158        }
159      }catch {
160        case  e: IOException =>
161          e.printStackTrace()
162      }
163
164    }
165  }
166
167
168  def initialize(ctx: ProcessContext): Unit = {
169
170  }
171
172  def setProperties(map : Map[String, Any]) = {
173    isCustomize=MapUtil.get(map,key="isCustomize").asInstanceOf[String]
174    filePath=MapUtil.get(map,key="filePath").asInstanceOf[String]
175    hdfsUrl=MapUtil.get(map,key="hdfsUrl").asInstanceOf[String]
176    savePath=MapUtil.get(map,key="savePath").asInstanceOf[String]
177  }
178
179  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
180    var descriptor : List[PropertyDescriptor] = List()
181
182    val filePath = new PropertyDescriptor().name("filePath").displayName("filePath").description("file path,such as /a/a.gz").defaultValue("").required(false)
183    val hdfsUrl = new PropertyDescriptor().name("hdfsUrl").displayName("hdfsUrl").description("the url of HDFS,such as hdfs://10.0.86.89:9000").defaultValue("").required(false)
184    val savePath = new PropertyDescriptor().name("savePath").displayName("savePath").description("This parameter can specify the location of the decompressed file, you can choose not to fill in, " +
185      "the program will default to save the decompressed file in the folder where the source file is located. If you fill in, you can specify a folder, such as /A/AB/").defaultValue("").required(false)
186    val isCustomize = new PropertyDescriptor().name("isCustomize").displayName("isCustomize").description("Whether to customize the compressed file path, if true, " +
187                                                                                                          "you must specify the path where the compressed file is located . " +
188                                                                                                          "If it is false, it will automatically find the file path data from the upstream port ")
189                                                                                                          .defaultValue("false").allowableValues(Set("true","false")).required(true)
190    descriptor = isCustomize :: descriptor
191    descriptor = filePath :: descriptor
192    descriptor = hdfsUrl :: descriptor
193    descriptor = savePath :: descriptor
194
195    descriptor
196  }
197
198  override def getIcon(): Array[Byte] = {
199    ImageUtil.getImage("icon/hdfs/UnzipFilesOnHDFS.png")
200  }
201
202  override def getGroup(): List[String] = {
203    List(StopGroup.HdfsGroup.toString)
204  }
205}