upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/unstructured/PdfParser.scala
1package cn.piflow.bundle.unstructured
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil, ProcessUtil}
5import cn.piflow.conf.{ConfigurableStop, Port}
6import cn.piflow.util.UnstructuredUtils
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import com.alibaba.fastjson2.{JSON, JSONArray}
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11import scala.collection.mutable.ArrayBuffer
12
13class PdfParser extends ConfigurableStop {
14 val authorEmail: String = "tianyao@cnic.cn"
15 val description: String = "parse pdf to structured data."
16 val inportList: List[String] = List(Port.DefaultPort)
17 val outportList: List[String] = List(Port.DefaultPort)
18
19 var filePath: String = _
20 var fileSource: String = _
21 var strategy: String = _
22
23
24 override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
25 val spark = pec.get[SparkSession]()
26
27 val unstructuredHost: String = UnstructuredUtils.unstructuredHost()
28 val unstructuredPort: String = UnstructuredUtils.unstructuredPort()
29 if (unstructuredHost == null || unstructuredHost.isEmpty) {
30 println("########## Exception: can not parse, unstructured host is null!!!")
31 throw new Exception("########## Exception: can not parse, unstructured host is null!!!")
32 } else if ("127.0.0.1".equals(unstructuredHost) || "localhost".equals(unstructuredHost)) {
33 println("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
34 throw new Exception("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
35 }
36 var localDir = ""
37 if ("hdfs".equals(fileSource)) {
38 //Download the file to the location,
39 localDir = UnstructuredUtils.downloadFilesFromHdfs(filePath)
40 }
41
42 //Create a mutable ArrayBuffer to store the parameters of the curl command
43 println("curl start==========================================================================")
44 val curlCommandParams = new ArrayBuffer[String]()
45 curlCommandParams += "curl"
46 curlCommandParams += "-X"
47 curlCommandParams += "POST"
48 curlCommandParams += s"$unstructuredHost:$unstructuredPort/general/v0/general"
49 curlCommandParams += "-H"
50 curlCommandParams += "accept: application/json"
51 curlCommandParams += "-H"
52 curlCommandParams += "Content-Type: multipart/form-data"
53 curlCommandParams += "-F"
54 curlCommandParams += "pdf_infer_table_structure=false"
55 curlCommandParams += "-F"
56 curlCommandParams += s"strategy=$strategy"
57 curlCommandParams += "-F"
58 curlCommandParams += "hi_res_model_name=detectron2_lp"
59 var fileListSize = 0;
60 if ("hdfs".equals(fileSource)) {
61 val fileList = UnstructuredUtils.getLocalFilePaths(localDir)
62 fileListSize = fileList.size
63 fileList.foreach { path =>
64 println(s"local path:$path")
65 curlCommandParams += "-F"
66 curlCommandParams += s"files=@$path"
67 }
68 }
69 if ("nfs".equals(fileSource)) {
70 val fileList = UnstructuredUtils.getLocalFilePaths(filePath)
71 fileListSize = fileList.size
72 fileList.foreach { path =>
73 println(s"local path:$path")
74 curlCommandParams += "-F"
75 curlCommandParams += s"files=@$path"
76 }
77 }
78 val (output, error): (String, String) = ProcessUtil.executeCommand(curlCommandParams.toSeq)
79 if (output.nonEmpty) {
80 // println(output)
81 import spark.implicits._
82 if (fileListSize > 1) {
83 val array: JSONArray = JSON.parseArray(output)
84 var combinedDF: DataFrame = null
85 array.forEach {
86 o =>
87 val jsonString = o.toString
88 val df = spark.read.json(Seq(jsonString).toDS)
89 if (combinedDF == null) {
90 combinedDF = df
91 } else {
92 combinedDF = combinedDF.union(df)
93 }
94 }
95 combinedDF.show(10)
96 out.write(combinedDF)
97 } else {
98 val df = spark.read.json(Seq(output).toDS())
99 df.show(10)
100 out.write(df)
101 }
102 } else {
103 println(s"########## Exception: $error")
104 throw new Exception(s"########## Exception: $error")
105 }
106 //delete local temp file
107 if ("hdfs".equals(fileSource)) {
108 UnstructuredUtils.deleteTempFiles(localDir)
109 }
110 }
111
112 override def setProperties(map: Map[String, Any]): Unit = {
113 filePath = MapUtil.get(map, "filePath").asInstanceOf[String]
114 fileSource = MapUtil.get(map, "fileSource").asInstanceOf[String]
115 strategy = MapUtil.get(map, "strategy").asInstanceOf[String]
116 }
117
118 override def getPropertyDescriptor(): List[PropertyDescriptor] = {
119 var descriptor: List[PropertyDescriptor] = List()
120 val filePath = new PropertyDescriptor()
121 .name("filePath")
122 .displayName("FilePath")
123 .description("The path of the file(.pdf)")
124 .defaultValue("")
125 .required(true)
126 .example("/test/test.pdf")
127 descriptor = descriptor :+ filePath
128
129 val fileSource = new PropertyDescriptor()
130 .name("fileSource")
131 .displayName("FileSource")
132 .description("The source of the file ")
133 .defaultValue("true")
134 .allowableValues(Set("hdfs", "nfs"))
135 .required(true)
136 .example("hdfs")
137 descriptor = descriptor :+ fileSource
138
139 val strategy = new PropertyDescriptor()
140 .name("strategy")
141 .displayName("strategy")
142 .description("The method the method that will be used to process the file ")
143 .defaultValue("true")
144 .allowableValues(Set("auto", "hi_res", "ocr_only", "fast"))
145 .required(true)
146 .example("auto")
147 descriptor = descriptor :+ strategy
148
149 descriptor
150 }
151
152 override def getIcon(): Array[Byte] = {
153 ImageUtil.getImage("icon/unstructured/PdfParser.png")
154 }
155
156 override def getGroup(): List[String] = {
157 List("unstructured")
158 }
159
160
161 override def initialize(ctx: ProcessContext): Unit = {
162
163 }
164
165}