upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/hdfs/UnzipFilesOnHDFS.scala
1package cn.piflow.bundle.hdfs
2
3import java.io._
4import java.util.zip.GZIPInputStream
5
6import cn.piflow.conf._
7import cn.piflow.conf.bean.PropertyDescriptor
8import cn.piflow.conf.util.{ImageUtil, MapUtil}
9import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
10import org.apache.hadoop.conf.Configuration
11import org.apache.hadoop.fs.{FSDataInputStream, FSDataOutputStream, FileSystem, Path}
12import org.apache.spark.rdd.RDD
13import org.apache.spark.sql.types.{StringType, StructField, StructType}
14import org.apache.spark.sql.{DataFrame, Row, SparkSession}
15import org.apache.tools.tar.{TarEntry, TarInputStream}
16
17import scala.collection.mutable.ArrayBuffer
18
19class UnzipFilesOnHDFS extends ConfigurableStop {
20 val authorEmail: String = "yangqidong@cnic.cn"
21 val description: String = "Unzip files on hdfs"
22 val inportList: List[String] = List(PortEnum.DefaultPort.toString)
23 val outportList: List[String] = List(PortEnum.DefaultPort.toString)
24
25 var isCustomize:String=_
26 var hdfsUrl:String=_
27 var filePath:String=_
28 var savePath:String=_
29
30 var session: SparkSession = null
31 var arr:ArrayBuffer[Row]=ArrayBuffer()
32
33
34 def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
35
36 session = pec.get[SparkSession]()
37
38
39 if(isCustomize.equals("true")){
40
41 unzipFile(hdfsUrl+filePath,savePath)
42
43 }else if (isCustomize .equals("false")){
44
45 val inDf: DataFrame = in.read()
46 inDf.collect().foreach(row => {
47 filePath = row.get(0).asInstanceOf[String]
48 unzipFile(filePath,savePath)
49 })
50 }
51
52 val rdd: RDD[Row] = session.sparkContext.makeRDD(arr.toList)
53 val fields: Array[StructField] =Array(StructField("savePath",StringType,nullable = true))
54 val schema: StructType = StructType(fields)
55 val df: DataFrame = session.createDataFrame(rdd,schema)
56
57 out.write(df)
58
59 }
60
61
62 def whatType(p:String): String = {
63 var typeStr:String=""
64 val pathNames: Array[String] = p.split("\\.")
65 val lastStr: String = pathNames.last
66 if(lastStr.equals("gz")){
67 val penultStr: String = pathNames(pathNames.length-2)
68 if(penultStr.equals("tar")){
69 typeStr="tar.gz"
70 }else {
71 typeStr="gz"
72 }
73 }else{
74 throw new RuntimeException("File type fill in error, or do not support this type.")
75 }
76 typeStr
77 }
78
79
80 def getFs(fileHdfsPath: String): FileSystem = {
81 var configuration: Configuration = new Configuration()
82 var fs: FileSystem =null
83 if (isCustomize.equals("false")) {
84 val pathARR: Array[String] = fileHdfsPath.split("\\/")
85 hdfsUrl = ""
86 for (x <- (0 until 3)) {
87 hdfsUrl += (pathARR(x) + "/")
88 }
89 }
90 configuration.set("fs.defaultFS", hdfsUrl)
91 fs = FileSystem.get(configuration)
92 fs
93 }
94
95
96
97
98 def unzipFile(fileHdfsPath: String, saveHdfsPath: String)= {
99 var eachSavePath : String=""
100
101 var unType: String = whatType(fileHdfsPath)
102 var fileName: String = fileHdfsPath.split("\\/").last
103 var fs: FileSystem= getFs(fileHdfsPath)
104
105 var sp:String=""
106 if(saveHdfsPath.length < 1){
107 sp=fileHdfsPath.replace(fileName,"")
108 }else{
109 sp = hdfsUrl + saveHdfsPath
110 }
111
112 val fdis: FSDataInputStream = fs.open(new Path(fileHdfsPath))
113
114
115 if(unType.equals("gz")){
116 val gzip: GZIPInputStream = new GZIPInputStream(fdis)
117 var n = -1
118 val buf=new Array[Byte](10*1024*1024)
119
120 eachSavePath = sp +fileName.replace(".gz","")
121 arr += Row.fromSeq(Array(eachSavePath))
122 val path = new Path(eachSavePath)
123 val fdos = fs.create(path)
124 while((n=gzip.read(buf)) != -1 && n != -1){
125 fdos.write(buf,0,n)
126 fdos.flush()
127 }
128 fdos.close()
129 gzip.close()
130 fdis.close()
131 }else if(unType.equals("tar.gz")){
132
133 try {
134 val gzip = new GZIPInputStream(new BufferedInputStream(fdis))
135 val tarIn = new TarInputStream(gzip, 1024 * 2)
136
137 var entry: TarEntry = null
138
139 while ((entry = tarIn.getNextEntry) != null && entry !=null) {
140
141 if (entry.isDirectory()) {
142 val outPath = sp + "/" + entry.getName
143 fs.create(new Path(outPath)).close()
144
145 } else {
146 val outPath = sp + "/" + entry.getName
147
148 arr += Row.fromSeq(Array(outPath))
149 val fos: FSDataOutputStream = fs.create(new Path(outPath))
150
151 var lenth = 0
152 val buff = new Array[Byte](1024)
153 while ((lenth = tarIn.read(buff)) != -1 && (lenth != -1)) {
154 fos.write(buff, 0, lenth)
155 }
156 fos.close()
157 }
158 }
159 }catch {
160 case e: IOException =>
161 e.printStackTrace()
162 }
163
164 }
165 }
166
167
168 def initialize(ctx: ProcessContext): Unit = {
169
170 }
171
172 def setProperties(map : Map[String, Any]) = {
173 isCustomize=MapUtil.get(map,key="isCustomize").asInstanceOf[String]
174 filePath=MapUtil.get(map,key="filePath").asInstanceOf[String]
175 hdfsUrl=MapUtil.get(map,key="hdfsUrl").asInstanceOf[String]
176 savePath=MapUtil.get(map,key="savePath").asInstanceOf[String]
177 }
178
179 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
180 var descriptor : List[PropertyDescriptor] = List()
181
182 val filePath = new PropertyDescriptor().name("filePath").displayName("filePath").description("file path,such as /a/a.gz").defaultValue("").required(false)
183 val hdfsUrl = new PropertyDescriptor().name("hdfsUrl").displayName("hdfsUrl").description("the url of HDFS,such as hdfs://10.0.86.89:9000").defaultValue("").required(false)
184 val savePath = new PropertyDescriptor().name("savePath").displayName("savePath").description("This parameter can specify the location of the decompressed file, you can choose not to fill in, " +
185 "the program will default to save the decompressed file in the folder where the source file is located. If you fill in, you can specify a folder, such as /A/AB/").defaultValue("").required(false)
186 val isCustomize = new PropertyDescriptor().name("isCustomize").displayName("isCustomize").description("Whether to customize the compressed file path, if true, " +
187 "you must specify the path where the compressed file is located . " +
188 "If it is false, it will automatically find the file path data from the upstream port ")
189 .defaultValue("false").allowableValues(Set("true","false")).required(true)
190 descriptor = isCustomize :: descriptor
191 descriptor = filePath :: descriptor
192 descriptor = hdfsUrl :: descriptor
193 descriptor = savePath :: descriptor
194
195 descriptor
196 }
197
198 override def getIcon(): Array[Byte] = {
199 ImageUtil.getImage("icon/hdfs/UnzipFilesOnHDFS.png")
200 }
201
202 override def getGroup(): List[String] = {
203 List(StopGroup.HdfsGroup.toString)
204 }
205}