Source code

Download source archive ↓

upstream/piflow-bundle/src/main/scala/cn/piflow/bundle/unstructured/DocxParser.scala

1package cn.piflow.bundle.unstructured
2
3import cn.piflow.conf.bean.PropertyDescriptor
4import cn.piflow.conf.util.{ImageUtil, MapUtil, ProcessUtil}
5import cn.piflow.conf.{ConfigurableStop, Port}
6import cn.piflow.util.UnstructuredUtils
7import cn.piflow.{JobContext, JobInputStream, JobOutputStream, ProcessContext}
8import com.alibaba.fastjson2.{JSON, JSONArray}
9import org.apache.spark.sql.{DataFrame, SparkSession}
10
11import scala.collection.mutable.ArrayBuffer
12
13class DocxParser extends ConfigurableStop {
14  val authorEmail: String = "tianyao@cnic.cn"
15  val description: String = "parse docx to structured data."
16  val inportList: List[String] = List(Port.DefaultPort)
17  val outportList: List[String] = List(Port.DefaultPort)
18
19  var filePath: String = _
20  var fileSource: String = _
21
22
23  override def perform(in: JobInputStream, out: JobOutputStream, pec: JobContext): Unit = {
24    val spark = pec.get[SparkSession]()
25
26    val unstructuredHost: String = UnstructuredUtils.unstructuredHost()
27    val unstructuredPort: String = UnstructuredUtils.unstructuredPort()
28    if (unstructuredHost == null || unstructuredHost.isEmpty) {
29      println("########## Exception: can not parse, unstructured host is null!!!")
30      throw new Exception("########## Exception: can not parse, unstructured host is null!!!")
31    } else if ("127.0.0.1".equals(unstructuredHost) || "localhost".equals(unstructuredHost)) {
32      println("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
33      throw new Exception("########## Exception: can not parse, the unstructured host cannot be set to localhost!!!")
34    }
35    var localDir = ""
36    if ("hdfs".equals(fileSource)) {
37      //Download the file to the location,
38      localDir = UnstructuredUtils.downloadFilesFromHdfs(filePath)
39    }
40
41    //Create a mutable ArrayBuffer to store the parameters of the curl command
42    println("curl start==========================================================================")
43    val curlCommandParams = new ArrayBuffer[String]()
44    curlCommandParams += "curl"
45    curlCommandParams += "-X"
46    curlCommandParams += "POST"
47    curlCommandParams += s"$unstructuredHost:$unstructuredPort/general/v0/general"
48    curlCommandParams += "-H"
49    curlCommandParams += "accept: application/json"
50    curlCommandParams += "-H"
51    curlCommandParams += "Content-Type: multipart/form-data"
52    var fileListSize = 0;
53    if ("hdfs".equals(fileSource)) {
54      val fileList = UnstructuredUtils.getLocalFilePaths(localDir)
55      fileListSize = fileList.size
56      fileList.foreach { path =>
57        curlCommandParams += "-F"
58        curlCommandParams += s"files=@$path"
59      }
60    }
61    if ("nfs".equals(fileSource)) {
62      val fileList = UnstructuredUtils.getLocalFilePaths(filePath)
63      fileListSize = fileList.size
64      fileList.foreach { path =>
65        curlCommandParams += "-F"
66        curlCommandParams += s"files=@$path"
67      }
68    }
69    val (output, error): (String, String) = ProcessUtil.executeCommand(curlCommandParams.toSeq)
70    if (output.nonEmpty) {
71      //      println(output)
72      import spark.implicits._
73      if (fileListSize > 1) {
74        val array: JSONArray = JSON.parseArray(output)
75        var combinedDF: DataFrame = null
76        array.forEach {
77          o =>
78            val jsonString = o.toString
79            val df = spark.read.json(Seq(jsonString).toDS)
80            if (combinedDF == null) {
81              combinedDF = df
82            } else {
83              combinedDF = combinedDF.union(df)
84            }
85        }
86        combinedDF.show(10)
87        out.write(combinedDF)
88      } else {
89        val df = spark.read.json(Seq(output).toDS())
90        df.show(10)
91        out.write(df)
92      }
93    } else {
94      println(s"########## Exception: $error")
95      throw new Exception(s"########## Exception: $error")
96    }
97    //delete local temp file
98    if ("hdfs".equals(fileSource)) {
99      UnstructuredUtils.deleteTempFiles(localDir)
100    }
101  }
102
103  override def setProperties(map: Map[String, Any]): Unit = {
104    filePath = MapUtil.get(map, "filePath").asInstanceOf[String]
105    fileSource = MapUtil.get(map, "fileSource").asInstanceOf[String]
106  }
107
108  override def getPropertyDescriptor(): List[PropertyDescriptor] = {
109    var descriptor: List[PropertyDescriptor] = List()
110    val filePath = new PropertyDescriptor()
111      .name("filePath")
112      .displayName("FilePath")
113      .description("The path of the file(.docx)")
114      .defaultValue("/test/test.docx")
115      .required(true)
116      .example("/test/test.docx")
117    descriptor = descriptor :+ filePath
118
119    val fileSource = new PropertyDescriptor()
120      .name("fileSource")
121      .displayName("FileSource")
122      .description("The source of the file ")
123      .defaultValue("hdfs")
124      .allowableValues(Set("hdfs", "nfs"))
125      .required(true)
126      .example("hdfs")
127    descriptor = descriptor :+ fileSource
128
129    descriptor
130  }
131
132  override def getIcon(): Array[Byte] = {
133    ImageUtil.getImage("icon/unstructured/DocxParser.png")
134  }
135
136  override def getGroup(): List[String] = {
137    List("unstructured")
138  }
139
140
141  override def initialize(ctx: ProcessContext): Unit = {
142
143  }
144
145}