src/piflow_documents.py
1"""Portable versions of PiFlow's document and archive transforms."""
2
3import gzip
4from html.parser import HTMLParser
5import io
6import re
7import zipfile
8
9import pyarrow as pa
10
11RECORDS_TEXT = pa.schema([("text", pa.string())])
12ARCHIVE_LIMIT = 32 * 1024 * 1024
13
14
15class HtmlText(HTMLParser):
16 blocks = {"article", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "td", "th"}
17 excluded = {"head", "script", "style", "template"}
18
19 def __init__(self):
20 super().__init__(convert_charrefs=True)
21 self.hidden = []
22 self.lines = []
23 self.buffer = []
24
25 def handle_starttag(self, tag, attrs):
26 if tag in self.excluded:
27 self.hidden.append(tag)
28 elif not self.hidden and (tag in self.blocks or tag == "br"):
29 self.flush()
30
31 def handle_endtag(self, tag):
32 if self.hidden:
33 if tag == self.hidden[-1]:
34 self.hidden.pop()
35 elif tag in self.blocks:
36 self.flush()
37
38 def handle_data(self, data):
39 if not self.hidden:
40 self.buffer.append(data)
41
42 def flush(self):
43 text = " ".join("".join(self.buffer).split())
44 if text:
45 self.lines.append({"text": text})
46 self.buffer.clear()
47
48
49def html_text(data):
50 parser = HtmlText()
51 parser.feed(data.decode("utf-8-sig"))
52 parser.close()
53 parser.flush()
54 return pa.Table.from_pylist(parser.lines, schema=RECORDS_TEXT)
55
56
57def office_text(data, operation, parse):
58 with zipfile.ZipFile(io.BytesIO(data)) as source:
59 entries = source.infolist()
60 if len(entries) > 5000:
61 raise ValueError("Office package contains more than 5,000 entries")
62 if operation == "docx_parser":
63 parts = [entry for entry in entries if entry.filename == "word/document.xml"]
64 ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
65 else:
66 pattern = re.compile(r"ppt/slides/slide([1-9][0-9]{0,5})\.xml")
67 parts = [entry for entry in entries if pattern.fullmatch(entry.filename)]
68 parts.sort(key=lambda entry: int(pattern.fullmatch(entry.filename)[1]))
69 ns = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
70 if not parts or len({entry.filename for entry in parts}) != len(parts):
71 raise ValueError("Office package must have unique matching document parts")
72 if sum(entry.file_size for entry in parts) > ARCHIVE_LIMIT:
73 raise ValueError("expanded document parts exceed 32 MiB")
74 rows = []
75 for index, entry in enumerate(parts, start=1):
76 root = parse(source.read(entry))
77 for paragraph in root.iter(ns + "p"):
78 text = "".join(node.text or "" for node in paragraph.iter(ns + "t"))
79 if text:
80 row = {"text": text}
81 if operation == "pptx_parser":
82 row["slide"] = index
83 rows.append(row)
84 schema = RECORDS_TEXT if operation == "docx_parser" else RECORDS_TEXT.append(pa.field("slide", pa.int64()))
85 return pa.Table.from_pylist(rows, schema=schema)
86
87
88def json_field(data, path, parse):
89 table = parse(data)
90 fields = path.split(".")
91 if any(not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_-]*", field) for field in fields):
92 raise ValueError("tag must be a dot-separated JSON field path")
93
94 def field_value(row):
95 for field in fields:
96 if not isinstance(row, dict):
97 return None
98 row = row.get(field)
99 return row
100
101 return pa.table({fields[-1]: [field_value(row) for row in table.to_pylist()]})
102
103
104def tag_path(root, path):
105 tags = path.strip("/").split("/")
106 if len(tags) > 32 or any(not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_.-]*", tag) for tag in tags):
107 raise ValueError("XML path must contain at most 32 plain element names")
108 if tags[0] != root.tag:
109 raise ValueError("XML path must begin with the root tag")
110 return root.findall("/".join(tags[1:])) if len(tags) > 1 else [root]
111
112
113def flatten_xml(root, params):
114 parent = tag_path(root, params["tagPath"].replace(",", "/"))
115 if len(parent) != 1:
116 raise ValueError("tagPath must select one parent element")
117 group = params["openTag"]
118 fields = [part.strip() for part in params["returnField"].split(",") if part.strip()]
119 target = parent[0]
120 shared = {node.tag: "".join(node.itertext()) for node in target if node.tag != group}
121 if fields:
122 shared = {key: shared.get(key) for key in fields}
123 if not group:
124 children = [shared]
125 else:
126 containers = [node for node in target if node.tag == group]
127 children = [
128 {**shared, **{f"{group}_{element.tag}": "".join(element.itertext()) for element in row}}
129 for container in containers for row in container
130 ]
131 names = dict.fromkeys(key for row in children for key in row)
132 return pa.table({key: pa.array([row.get(key) for row in children], type=pa.string()) for key in names})
133
134
135def xml_value(value, path, parse):
136 if value is None:
137 return None
138 return "".join("".join(element.itertext()) for element in tag_path(parse(value), path))
139
140
141def decompress(data):
142 with gzip.GzipFile(fileobj=io.BytesIO(data)) as source:
143 count = 0
144 while chunk := source.read(64 * 1024):
145 count += len(chunk)
146 if count > ARCHIVE_LIMIT:
147 raise ValueError("expanded gzip stream exceeds 32 MiB")
148 yield chunk
149
150
151def pdf_text(data, schema):
152 from pypdf import PdfReader
153
154 reader = PdfReader(io.BytesIO(data), strict=True)
155 if reader.is_encrypted or len(reader.pages) > 100:
156 raise ValueError("PDF input must be unencrypted and at most 100 pages")
157 texts = []
158 for number, page in enumerate(reader.pages, start=1):
159 stream = page.get_contents()
160 if stream is not None and len(stream.get_data()) > 2 * 1024 * 1024:
161 raise ValueError("decoded PDF page content exceeds 2 MiB")
162 texts.append({"page": number, "text": page.extract_text().strip()})
163 return pa.Table.from_pylist(texts, schema=schema)