Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/unstructured/PdfParser.scala

1package cn.piflow.bundle.unstructured
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil, ProcessUtil}
5import cn.piflow.conf.{ConfigurableStop, Port}
6import cn.piflow.util.UnstructuredUtils
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import com.alibaba.fastjson2.{JSON, JSONArray}
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11import scala.collection.mutable.ArrayBuffer
12
13class PdfParser extends ConfigurableStop {
14  val authorEmail: String = "tianyao@cnic.cn"
15  val description: String = "parse pdf to structured data."
16  val inportList: List[String] = List(Port.DefaultPort)
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var filePath: String = _
20  var fileSource: String = _
21  var strategy: String = _
22
23
24  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
25    val spark = pec.get[SparkSession]()
26
27    val unstructuredHost: String = UnstructuredUtils.unstructuredHost()
28    val unstructuredPort: String = UnstructuredUtils.unstructuredPort()
29    if (unstructuredHost == null || unstructuredHost.isEmpty) {
30      println("########## Exception: can not parse, unstructured host is null!!!")
31      throw new Exception("########## Exception: can not parse, unstructured host is null!!!")
32    } else if ("127.0.0.1".equals(unstructuredHost) || "localhost".equals(unstructuredHost)) {
33      println("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
34      throw new Exception("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
35    }
36    var localDir = ""
37    if ("hdfs".equals(fileSource)) {
38      //Download the file to the location,
39      localDir = UnstructuredUtils.downloadFilesFromHdfs(filePath)
40    }
41
42    //Create a mutable ArrayBuffer to store the parameters of the curl command
43    println("curl start==========================================================================")
44    val curlCommandParams = new ArrayBuffer[String]()
45    curlCommandParams += "curl"
46    curlCommandParams += "-X"
47    curlCommandParams += "POST"
48    curlCommandParams += s"$unstructuredHost:$unstructuredPort/general/v0/general"
49    curlCommandParams += "-H"
50    curlCommandParams += "accept: application/json"
51    curlCommandParams += "-H"
52    curlCommandParams += "Content-Type: multipart/form-data"
53    curlCommandParams += "-F"
54    curlCommandParams += "pdf_infer_table_structure=false"
55    curlCommandParams += "-F"
56    curlCommandParams += s"strategy=$strategy"
57    curlCommandParams += "-F"
58    curlCommandParams += "hi_res_model_name=detectron2_lp"
59    var fileListSize = 0;
60    if ("hdfs".equals(fileSource)) {
61      val fileList = UnstructuredUtils.getLocalFilePaths(localDir)
62      fileListSize = fileList.size
63      fileList.foreach { path =>
64        println(s"local path:$path")
65        curlCommandParams += "-F"
66        curlCommandParams += s"files=@$path"
67      }
68    }
69    if ("nfs".equals(fileSource)) {
70      val fileList = UnstructuredUtils.getLocalFilePaths(filePath)
71      fileListSize = fileList.size
72      fileList.foreach { path =>
73        println(s"local path:$path")
74        curlCommandParams += "-F"
75        curlCommandParams += s"files=@$path"
76      }
77    }
78    val (output, error): (String, String) = ProcessUtil.executeCommand(curlCommandParams.toSeq)
79    if (output.nonEmpty) {
80      //      println(output)
81      import spark.implicits._
82      if (fileListSize > 1) {
83        val array: JSONArray = JSON.parseArray(output)
84        var combinedDF: DataFrame = null
85        array.forEach {
86          o =>
87            val jsonString = o.toString
88            val df = spark.read.json(Seq(jsonString).toDS)
89            if (combinedDF == null) {
90              combinedDF = df
91            } else {
92              combinedDF = combinedDF.union(df)
93            }
94        }
95        combinedDF.show(10)
96        out.write(combinedDF)
97      } else {
98        val df = spark.read.json(Seq(output).toDS())
99        df.show(10)
100        out.write(df)
101      }
102    } else {
103      println(s"########## Exception: $error")
104      throw new Exception(s"########## Exception: $error")
105    }
106    //delete local temp file
107    if ("hdfs".equals(fileSource)) {
108      UnstructuredUtils.deleteTempFiles(localDir)
109    }
110  }
111
112  override def setProperties(map: Map[String, Any]): Unit = {
113    filePath = MapUtil.get(map, "filePath").asInstanceOf[String]
114    fileSource = MapUtil.get(map, "fileSource").asInstanceOf[String]
115    strategy = MapUtil.get(map, "strategy").asInstanceOf[String]
116  }
117
118  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
119    var descriptor: List[PropertyDescriptor] = List()
120    val filePath = new PropertyDescriptor()
121      .name("filePath")
122      .displayName("FilePath")
123      .description("The path of the file(.pdf)")
124      .defaultValue("")
125      .required(true)
126      .example("/test/test.pdf")
127    descriptor = descriptor :+ filePath
128
129    val fileSource = new PropertyDescriptor()
130      .name("fileSource")
131      .displayName("FileSource")
132      .description("The source of the file ")
133      .defaultValue("true")
134      .allowableValues(Set("hdfs", "nfs"))
135      .required(true)
136      .example("hdfs")
137    descriptor = descriptor :+ fileSource
138
139    val strategy = new PropertyDescriptor()
140      .name("strategy")
141      .displayName("strategy")
142      .description("The method the method that will be used to process the file ")
143      .defaultValue("true")
144      .allowableValues(Set("auto", "hi_res", "ocr_only", "fast"))
145      .required(true)
146      .example("auto")
147    descriptor = descriptor :+ strategy
148
149    descriptor
150  }
151
152  override def getIcon(): Array[Byte] = {
153    ImageUtil.getImage("icon/unstructured/PdfParser.png")
154  }
155
156  override def getGroup(): List[String] = {
157    List("unstructured")
158  }
159
160
161  override def initialize(ctx: ProcessContext): Unit = {
162
163  }
164
165}