Source code

Download source archive ↓

src/piflow_documents.py

1"""Portable versions of PiFlow's document and archive transforms."""
2
3import gzip
4from html.parser import HTMLParser
5import io
6import re
7import zipfile
8
9import pyarrow as pa
10
11RECORDS_TEXT = pa.schema([("text", pa.string())])
12ARCHIVE_LIMIT = 32 * 1024 * 1024
13
14
15class HtmlText(HTMLParser):
16    blocks = {"article", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "p", "section", "td", "th"}
17    excluded = {"head", "script", "style", "template"}
18
19    def __init__(self):
20        super().__init__(convert_charrefs=True)
21        self.hidden = []
22        self.lines = []
23        self.buffer = []
24
25    def handle_starttag(self, tag, attrs):
26        if tag in self.excluded:
27            self.hidden.append(tag)
28        elif not self.hidden and (tag in self.blocks or tag == "br"):
29            self.flush()
30
31    def handle_endtag(self, tag):
32        if self.hidden:
33            if tag == self.hidden[-1]:
34                self.hidden.pop()
35        elif tag in self.blocks:
36            self.flush()
37
38    def handle_data(self, data):
39        if not self.hidden:
40            self.buffer.append(data)
41
42    def flush(self):
43        text = " ".join("".join(self.buffer).split())
44        if text:
45            self.lines.append({"text": text})
46        self.buffer.clear()
47
48
49def html_text(data):
50    parser = HtmlText()
51    parser.feed(data.decode("utf-8-sig"))
52    parser.close()
53    parser.flush()
54    return pa.Table.from_pylist(parser.lines, schema=RECORDS_TEXT)
55
56
57def office_text(data, operation, parse):
58    with zipfile.ZipFile(io.BytesIO(data)) as source:
59        entries = source.infolist()
60        if len(entries) > 5000:
61            raise ValueError("Office package contains more than 5,000 entries")
62        if operation == "docx_parser":
63            parts = [entry for entry in entries if entry.filename == "word/document.xml"]
64            ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
65        else:
66            pattern = re.compile(r"ppt/slides/slide([1-9][0-9]{0,5})\.xml")
67            parts = [entry for entry in entries if pattern.fullmatch(entry.filename)]
68            parts.sort(key=lambda entry: int(pattern.fullmatch(entry.filename)[1]))
69            ns = "{http://schemas.openxmlformats.org/drawingml/2006/main}"
70        if not parts or len({entry.filename for entry in parts}) != len(parts):
71            raise ValueError("Office package must have unique matching document parts")
72        if sum(entry.file_size for entry in parts) > ARCHIVE_LIMIT:
73            raise ValueError("expanded document parts exceed 32 MiB")
74        rows = []
75        for index, entry in enumerate(parts, start=1):
76            root = parse(source.read(entry))
77            for paragraph in root.iter(ns + "p"):
78                text = "".join(node.text or "" for node in paragraph.iter(ns + "t"))
79                if text:
80                    row = {"text": text}
81                    if operation == "pptx_parser":
82                        row["slide"] = index
83                    rows.append(row)
84    schema = RECORDS_TEXT if operation == "docx_parser" else RECORDS_TEXT.append(pa.field("slide", pa.int64()))
85    return pa.Table.from_pylist(rows, schema=schema)
86
87
88def json_field(data, path, parse):
89    table = parse(data)
90    fields = path.split(".")
91    if any(not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_-]*", field) for field in fields):
92        raise ValueError("tag must be a dot-separated JSON field path")
93
94    def field_value(row):
95        for field in fields:
96            if not isinstance(row, dict):
97                return None
98            row = row.get(field)
99        return row
100
101    return pa.table({fields[-1]: [field_value(row) for row in table.to_pylist()]})
102
103
104def tag_path(root, path):
105    tags = path.strip("/").split("/")
106    if len(tags) > 32 or any(not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_.-]*", tag) for tag in tags):
107        raise ValueError("XML path must contain at most 32 plain element names")
108    if tags[0] != root.tag:
109        raise ValueError("XML path must begin with the root tag")
110    return root.findall("/".join(tags[1:])) if len(tags) > 1 else [root]
111
112
113def flatten_xml(root, params):
114    parent = tag_path(root, params["tagPath"].replace(",", "/"))
115    if len(parent) != 1:
116        raise ValueError("tagPath must select one parent element")
117    group = params["openTag"]
118    fields = [part.strip() for part in params["returnField"].split(",") if part.strip()]
119    target = parent[0]
120    shared = {node.tag: "".join(node.itertext()) for node in target if node.tag != group}
121    if fields:
122        shared = {key: shared.get(key) for key in fields}
123    if not group:
124        children = [shared]
125    else:
126        containers = [node for node in target if node.tag == group]
127        children = [
128            {**shared, **{f"{group}_{element.tag}": "".join(element.itertext()) for element in row}}
129            for container in containers for row in container
130        ]
131    names = dict.fromkeys(key for row in children for key in row)
132    return pa.table({key: pa.array([row.get(key) for row in children], type=pa.string()) for key in names})
133
134
135def xml_value(value, path, parse):
136    if value is None:
137        return None
138    return "".join("".join(element.itertext()) for element in tag_path(parse(value), path))
139
140
141def decompress(data):
142    with gzip.GzipFile(fileobj=io.BytesIO(data)) as source:
143        count = 0
144        while chunk := source.read(64 * 1024):
145            count += len(chunk)
146            if count > ARCHIVE_LIMIT:
147                raise ValueError("expanded gzip stream exceeds 32 MiB")
148            yield chunk