Source code

Download source archive ↓

src/piflow_portable.py

1"""Arrow stream adapters for portable PiFlow components."""
2
3import bisect
4import json
5import math
6import re
7import statistics
8from xml.etree import ElementTree
9
10import pyarrow as pa
11import piflow_documents as documents
12from xstream_operator import descriptor, preview_process, records_schema, result, run_operator
13
14MAX_INPUT = 8 * 1024 * 1024
15HALF_WIDTH = str.maketrans({**{c: c - 65248 for c in range(0xff01, 0xff5f)}, 0x3000: 32})
16PROVINCES = dict(zip(
17    "beijing tianjin shanghai chongqing anhui fujian gansu guangdong guizhou hainan hebei heilongjiang "
18    "henan hubei hunan jiangsu jiangxi jilin liaoning qinghai shaanxi shanxi shandong sichuan yunnan "
19    "zhejiang inner-mongolia ningxia guangxi xinjiang tibet".split(),
20    "北京 天津 上海 重庆 安徽 福建 甘肃 广东 贵州 海南 河北 黑龙江 河南 湖北 湖南 江苏 江西 吉林 辽宁 "
21    "青海 陕西 山西 山东 四川 云南 浙江 内蒙古自治区 宁夏回族自治区 广西壮族自治区 新疆维吾尔自治区 西藏自治区".split(),
22))
23CLEANERS = {"email_clean", "phone_number_clean", "identity_number_clean", "province_clean", "title_clean"}
24SCALERS = {"max_min_normalization", "scope_normalization", "z_score", "discretization"}
25
26
27def clean(value, operation):
28    if value is None:
29        return "" if operation == "province_clean" else None
30    if not isinstance(value, str):
31        raise ValueError("cleaning columns must contain strings")
32    value = value.translate(HALF_WIDTH)
33    if operation == "title_clean":
34        return value
35    if operation == "email_clean":
36        value = value.lower()
37        return value if re.fullmatch(r"[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z0-9-]+", value) else ""
38    if operation == "phone_number_clean":
39        value = re.sub("[^0-9]", "", value)
40        return value if re.fullmatch(r"1[34578][0-9]{9}", value) else None
41    if operation == "identity_number_clean":
42        value = re.sub("[^0-9X]", "", value.upper()).lstrip("0")
43        if re.fullmatch("[0-9]{15}", value):
44            return value
45        if re.fullmatch("[0-9]{17}[0-9X]", value):
46            weights = (7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
47            checksum = sum(int(digit) * weight for digit, weight in zip(value, weights))
48            if value[-1] == "10X98765432"[checksum % 11]:
49                return value
50        return None
51    normalized = value.strip().casefold().replace(" ", "-")
52    for pinyin, name in PROVINCES.items():
53        if normalized in {pinyin, name.casefold(), name + "省", name + "市"}:
54            return name
55        if name.endswith("自治区") and normalized in {"内蒙古", "宁夏", "广西", "新疆", "西藏"} and name.startswith(normalized):
56            return name
57    return value
58
59
60def names(text):
61    values = [value.strip() for value in text.split(",")]
62    if not all(values) or len(values) != len(set(values)):
63        raise ValueError("column names must be unique and nonempty")
64    return values
65
66
67def field_indices(schema, columns):
68    indices = [schema.get_field_index(name) for name in columns]
69    if -1 in indices:
70        raise ValueError("input includes an unknown or ambiguous column")
71    return indices
72
73
74def scale_columns(operation, params):
75    if operation == "z_score":
76        ins, outs = names(params["inputCols"]), names(params["outputCols"])
77        if len(ins) != len(outs):
78            raise ValueError("inputCols and outputCols must contain the same number of names")
79        return list(zip(ins, outs))
80    return [(params["inputCol"], params["outputCol"])]
81
82
83def scale(values, operation, params):
84    available = [float(value) for value in values if value is not None]
85    if not all(map(math.isfinite, available)):
86        raise ValueError("scaling requires finite numeric input")
87    if not available:
88        return values
89    if operation == "discretization":
90        count = params["numBins"]
91        if not 2 <= count <= 1000:
92            raise ValueError("numBins must be from 2 to 1000")
93        if params["method"] == "EqualWidth":
94            if any(value < 0 or value > count for value in available):
95                raise ValueError("EqualWidth accepts values from zero through numBins")
96            apply = lambda value: min(int(value), count - 1)
97        elif params["method"] == "EqualFrequency":
98            ordered = sorted(available)
99            cuts = sorted({ordered[min(len(ordered) - 1, math.ceil(len(ordered) * i / count) - 1)]
100                           for i in range(1, count)} - {ordered[0], ordered[-1]})
101            apply = lambda value: bisect.bisect_right(cuts, value)
102        else:
103            raise ValueError("method must be EqualWidth or EqualFrequency")
104    else:
105        if operation == "z_score":
106            origin = statistics.mean(available)
107            spread = statistics.stdev(available) if len(available) > 1 else 0
108        else:
109            origin = min(available)
110            spread = max(available) - origin
111        low, high = params["range"] if operation == "scope_normalization" else (0, 1)
112        if not math.isfinite(low) or not math.isfinite(high) or low >= high:
113            raise ValueError("range must have two increasing finite bounds")
114        apply = lambda value: (value - origin) / spread * (high - low) + low if spread else None
115    return [None if value is None else apply(value) for value in values]
116
117
118def replace_column(table, name, values):
119    index = table.schema.get_field_index(name)
120    return table.append_column(name, values) if index < 0 else table.set_column(index, name, values)
121
122
123def collect(chunks):
124    data = bytearray()
125    for chunk in chunks:
126        data.extend(chunk)
127        if len(data) > MAX_INPUT:
128            raise ValueError("input exceeds 8 MiB")
129    return bytes(data)
130
131
132def collect_records(port, schema):
133    batches, size, rows = [], 0, 0
134    for batch in port.records():
135        size += batch.nbytes
136        rows += batch.num_rows
137        if size > MAX_INPUT or rows > 100_000:
138            raise ValueError("batch operation accepts at most 8 MiB and 100,000 records")
139        batches.append(batch)
140    return pa.Table.from_batches(batches, schema=schema)
141
142
143def json_rows(data):
144    def invalid_number(value):
145        raise ValueError("non-finite JSON number: " + value)
146
147    root = json.loads(data, parse_constant=invalid_number)
148    rows = root if isinstance(root, list) else [root]
149    if not all(isinstance(row, dict) for row in rows):
150        raise ValueError("JSON input must be an object or an array of objects")
151    # Arrow's Python row inference reads its columns from the first row.
152    columns = dict.fromkeys(key for row in rows for key in row)
153    return pa.table({column: [row.get(column) for row in rows] for column in columns})
154
155
156class XmlBuilder(ElementTree.TreeBuilder):
157    def doctype(self, name, pubid, system):
158        raise ValueError("XML document types are unsupported")
159
160
161def xml_root(data):
162    if len(data) > MAX_INPUT:
163        raise ValueError("XML exceeds 8 MiB")
164    return ElementTree.fromstring(data, parser=ElementTree.XMLParser(target=XmlBuilder()))
165
166
167def xml_rows(root, operation, params):
168    if operation == "xml_parser":
169        selected = list(root.iter(params["rowTag"]))
170        columns = list(dict.fromkeys(child.tag for row in selected for child in row))
171    else:
172        tags = names(params["label"])
173        if root.tag != tags[0] or len(tags) < 2:
174            raise ValueError("label must start with the root and name a repeated child tag")
175        selected = root.findall("/".join(tags[1:]))
176        columns = names(params["schema"])
177    return pa.table({column: pa.array([row.findtext(column) or None for row in selected], type=pa.string())
178                     for column in columns})
179
180
181def xml_model(node, depth=0):
182    if depth >= 32:
183        raise ValueError("XML nesting exceeds 32 levels")
184    if not node.attrib and not len(node):
185        return node.text
186    model = {"@" + name: value for name, value in node.attrib.items()}
187    if node.text and node.text.strip():
188        model["#text"] = node.text
189    groups = {}
190    for child in node:
191        groups.setdefault(child.tag, []).append(xml_model(child, depth + 1))
192    model.update({key: values[0] if len(values) == 1 else values for key, values in groups.items()})
193    return model
194
195
196def parse_xml_column(value):
197    if value is None:
198        return None
199    root = xml_root(value)
200    return json.dumps({root.tag: xml_model(root)}, ensure_ascii=False)
201
202
203def put_xml_row(root, row):
204    target = ElementTree.SubElement(root, "ROW")
205    for name, value in row.items():
206        if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_.-]*", name):
207            raise ValueError("record column is not an XML name: " + name)
208        if isinstance(value, (list, dict)):
209            raise ValueError("XML serialization accepts scalar columns")
210        if value is not None:
211            ElementTree.SubElement(target, name).text = str(value)
212
213
214def emit(ctx, table):
215    out = ctx.records_output("result", table.schema)
216    for batch in table.to_batches(max_chunksize=1024):
217        out.write(batch)
218
219
220def inferred_schema(operation, schema, params):
221    if operation in {"docx_parser", "html_parser"}:
222        return documents.RECORDS_TEXT
223    if operation == "pptx_parser":
224        return documents.RECORDS_TEXT.append(pa.field("slide", pa.int64()))
225    if operation == "xml_parser_generate_new_field" and schema is not None:
226        field_indices(schema, [params["decoderField"]])
227        name = params["newField"]
228        if not name or name in schema.names:
229            raise ValueError("newField must be nonempty and absent from the input")
230        return schema.append(pa.field(name, pa.string()))
231    if operation in CLEANERS | {"xml_parser_columns"} and schema is not None:
232        columns = params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"]
233        indices = field_indices(schema, names(columns))
234        if any(not pa.types.is_string(schema.field(i).type) and not pa.types.is_large_string(schema.field(i).type)
235               for i in indices):
236            raise ValueError("selected columns must be strings")
237        return pa.schema([pa.field(field.name, pa.string()) if i in indices else field
238                          for i, field in enumerate(schema)], metadata=schema.metadata)
239    if operation in SCALERS and schema is not None:
240        for source, target in scale_columns(operation, params):
241            field = schema.field(field_indices(schema, [source])[0])
242            if not (pa.types.is_integer(field.type) or pa.types.is_floating(field.type)):
243                raise ValueError("scaling requires integer or floating-point columns")
244        for source, target in scale_columns(operation, params):
245            field = pa.field(target, pa.float64())
246            index = schema.get_field_index(target)
247            schema = schema.append(field) if index == -1 else schema.set(index, field)
248    if operation == "json_string_parser":
249        return json_rows(params["jsonString"]).schema
250    if operation == "xml_string_parser":
251        return pa.schema([(column, pa.string()) for column in names(params["schema"])])
252    return schema if operation in CLEANERS | SCALERS | {"xml_parser_columns"} else None
253
254
255def process(ctx, operation):
256    params = ctx.params
257    if operation in {"json_string_parser", "xml_string_parser"}:
258        if operation == "json_string_parser":
259            emit(ctx, json_rows(params["jsonString"]))
260        else:
261            emit(ctx, xml_rows(xml_root(params["XmlString"]), operation, params))
262        return
263    source = ctx.input("source")
264    if operation in {"docx_parser", "pptx_parser", "html_parser", "flatten_xml_parser", "evaluate_json_path", "decompress_gzip"}:
265        data = collect(source.bytes())
266        if operation == "decompress_gzip":
267            out = ctx.bytes_output("result")
268            for chunk in documents.decompress(data):
269                out.write(chunk)
270            return
271        if operation == "html_parser":
272            table = documents.html_text(data)
273        elif operation in {"docx_parser", "pptx_parser"}:
274            table = documents.office_text(data, operation, xml_root)
275        elif operation == "evaluate_json_path":
276            table = documents.json_field(data, params["tag"], json_rows)
277        else:
278            table = documents.flatten_xml(xml_root(data), params)
279        emit(ctx, table)
280    elif operation == "xml_parser_generate_new_field":
281        schema = inferred_schema(operation, source.schema, params)
282        source_index = schema.get_field_index(params["decoderField"])
283        out = ctx.records_output("result", schema)
284        for batch in source.records():
285            values = [documents.xml_value(value, params["tagXPath"], xml_root)
286                      for value in batch.column(source_index).to_pylist()]
287            out.write(pa.RecordBatch.from_arrays([*batch.columns, pa.array(values, type=pa.string())], schema=schema))
288    elif operation in {"json_parser", "xml_parser"}:
289        data = collect(source.bytes())
290        emit(ctx, json_rows(data) if operation == "json_parser" else xml_rows(xml_root(data), operation, params))
291    elif operation == "json_save":
292        out = ctx.bytes_output("result")
293        for batch in source.records():
294            for row in batch.to_pylist():
295                out.write((json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n").encode())
296    elif operation == "xml_save":
297        out = ctx.bytes_output("result")
298        out.write(b"<ROWS>")
299        for batch in source.records():
300            for row in batch.to_pylist():
301                temp = ElementTree.Element("ROWS")
302                put_xml_row(temp, row)
303                out.write(ElementTree.tostring(temp[0], encoding="utf-8"))
304        out.write(b"</ROWS>")
305    elif operation in SCALERS:
306        schema = source.schema
307        inferred_schema(operation, schema, params)
308        original = collect_records(source, schema)
309        table = original
310        for origin, target in scale_columns(operation, params):
311            values = scale(original[origin].to_pylist(), operation, params)
312            table = replace_column(table, target, pa.array(values, type=pa.float64()))
313        emit(ctx, table)
314    else:
315        schema = inferred_schema(operation, source.schema, params)
316        selected = names(params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"])
317        indices = field_indices(source.schema, selected)
318        out = ctx.records_output("result", schema)
319        for batch in source.records():
320            arrays = list(batch.columns)
321            for index in indices:
322                values = arrays[index].to_pylist()
323                cleaned = (parse_xml_column(value) if operation == "xml_parser_columns" else clean(value, operation)
324                           for value in values)
325                arrays[index] = pa.array(cleaned, type=pa.string())
326            out.write(pa.RecordBatch.from_arrays(arrays, schema=schema))
327
328
329def start(operation, output_view):
330    def execute(ctx):
331        process(ctx, operation)
332
333    def infer(inputs, params):
334        schema = None
335        if output_view == "records":
336            source = inputs.get("source")
337            schema = inferred_schema(operation, records_schema(source) if source else None, params)
338        formats = ["arrow"] if output_view == "records" else ["binary"]
339        return result({"result": descriptor([output_view], formats, schema,
340                                            certainty="known" if schema is not None or output_view == "bytes" else "unknown")})
341
342    def preview(inputs, params, limits):
343        return preview_process(execute, inputs, params, limits)
344
345    run_operator(execute, infer, preview_outputs=preview)