upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/unstructured/HtmlParser.scala
1package cn.piflow.bundle.unstructured
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil, ProcessUtil}
5import cn.piflow.conf.{ConfigurableStop, Port}
6import cn.piflow.util.UnstructuredUtils
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import com.alibaba.fastjson2.{JSON, JSONArray}
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11import scala.collection.mutable.ArrayBuffer
12
13class HtmlParser extends ConfigurableStop {
14 val authorEmail: String = "tianyao@cnic.cn"
15 val description: String = "parse html to structured data."
16 val inportList: List[String] = List(Port.DefaultPort)
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var filePath: String = _
20 var fileSource: String = _
21
22
23 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24 val spark = pec.get[SparkSession]()
25
26 val unstructuredHost: String = UnstructuredUtils.unstructuredHost()
27 val unstructuredPort: String = UnstructuredUtils.unstructuredPort()
28 if (unstructuredHost == null || unstructuredHost.isEmpty) {
29 println("########## Exception: can not parse, unstructured host is null!!!")
30 throw new Exception("########## Exception: can not parse, unstructured host is null!!!")
31 } else if ("127.0.0.1".equals(unstructuredHost) || "localhost".equals(unstructuredHost)) {
32 println("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
33 throw new Exception("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
34 }
35 var localDir = ""
36 if ("hdfs".equals(fileSource)) {
37 //Download the file to the location,
38 localDir = UnstructuredUtils.downloadFilesFromHdfs(filePath)
39 }
40
41 //Create a mutable ArrayBuffer to store the parameters of the curl command
42 println("curl start==========================================================================")
43 val curlCommandParams = new ArrayBuffer[String]()
44 curlCommandParams += "curl"
45 curlCommandParams += "-X"
46 curlCommandParams += "POST"
47 curlCommandParams += s"$unstructuredHost:$unstructuredPort/general/v0/general"
48 curlCommandParams += "-H"
49 curlCommandParams += "accept: application/json"
50 curlCommandParams += "-H"
51 curlCommandParams += "Content-Type: multipart/form-data"
52 var fileListSize = 0;
53 if ("hdfs".equals(fileSource)) {
54 val fileList = UnstructuredUtils.getLocalFilePaths(localDir)
55 fileListSize = fileList.size
56 fileList.foreach { path =>
57 curlCommandParams += "-F"
58 curlCommandParams += s"files=@$path"
59 }
60 }
61 if ("nfs".equals(fileSource)) {
62 val fileList = UnstructuredUtils.getLocalFilePaths(filePath)
63 fileListSize = fileList.size
64 fileList.foreach { path =>
65 curlCommandParams += "-F"
66 curlCommandParams += s"files=@$path"
67 }
68 }
69 val (output, error): (String, String) = ProcessUtil.executeCommand(curlCommandParams.toSeq)
70 if (output.nonEmpty) {
71 // println(output)
72 import spark.implicits._
73 if (fileListSize > 1) {
74 val array: JSONArray = JSON.parseArray(output)
75 var combinedDF: DataFrame = null
76 array.forEach {
77 o =>
78 val jsonString = o.toString
79 val df = spark.read.json(Seq(jsonString).toDS)
80 if (combinedDF == null) {
81 combinedDF = df
82 } else {
83 combinedDF = combinedDF.union(df)
84 }
85 }
86 combinedDF.show(10)
87 out.write(combinedDF)
88 } else {
89 val df = spark.read.json(Seq(output).toDS())
90 df.show(10)
91 out.write(df)
92 }
93 } else {
94 println(s"########## Exception: $error")
95 throw new Exception(s"########## Exception: $error")
96 }
97 //delete local temp file
98 if ("hdfs".equals(fileSource)) {
99 UnstructuredUtils.deleteTempFiles(localDir)
100 }
101 }
102
103 override def setProperties(map: Map[String, Any]): Unit = {
104 filePath = MapUtil.get(map, "filePath").asInstanceOf[String]
105 fileSource = MapUtil.get(map, "fileSource").asInstanceOf[String]
106 }
107
108 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
109 var descriptor: List[PropertyDescriptor] = List()
110 val filePath = new PropertyDescriptor()
111 .name("filePath")
112 .displayName("FilePath")
113 .description("The path of the file(.html/.htm)")
114 .defaultValue("/test/test.html")
115 .required(true)
116 .example("/test/test.html")
117 descriptor = descriptor :+ filePath
118
119 val fileSource = new PropertyDescriptor()
120 .name("fileSource")
121 .displayName("FileSource")
122 .description("The source of the file ")
123 .defaultValue("hdfs")
124 .allowableValues(Set("hdfs", "nfs"))
125 .required(true)
126 .example("hdfs")
127 descriptor = descriptor :+ fileSource
128
129 descriptor
130 }
131
132 override def getIcon(): Array[Byte] = {
133 ImageUtil.getImage("icon/unstructured/HtmlParser.png")
134 }
135
136 override def getGroup(): List[String] = {
137 List("unstructured")
138 }
139
140
141 override def initialize(ctx: ProcessContext): Unit = {
142
143 }
144
145}