Source code

Download source archive ↓

src/piflow_portable.py

1"""Arrow stream adapters for portable PiFlow components."""
2
3import bisect
4from itertools import islice
5import json
6import math
7import re
8import statistics
9from xml.etree import ElementTree
10
11import pyarrow as pa
12import pyarrow.compute as pc
13import piflow_documents as documents
14import piflow_collections as collections
15import piflow_tabular as tabular
16import piflow_rdf as rdf
17import piflow_workbooks as workbooks
18from xstream_operator import descriptor, preview_process, records_schema, result, run_operator
19
20MAX_INPUT = 8 * 1024 * 1024
21HALF_WIDTH = str.maketrans({**{c: c - 65248 for c in range(0xff01, 0xff5f)}, 0x3000: 32})
22PROVINCES = dict(zip(
23    "beijing tianjin shanghai chongqing anhui fujian gansu guangdong guizhou hainan hebei heilongjiang "
24    "henan hubei hunan jiangsu jiangxi jilin liaoning qinghai shaanxi shanxi shandong sichuan yunnan "
25    "zhejiang inner-mongolia ningxia guangxi xinjiang tibet".split(),
26    "北京 天津 上海 重庆 安徽 福建 甘肃 广东 贵州 海南 河北 黑龙江 河南 湖北 湖南 江苏 江西 吉林 辽宁 "
27    "青海 陕西 山西 山东 四川 云南 浙江 内蒙古自治区 宁夏回族自治区 广西壮族自治区 新疆维吾尔自治区 西藏自治区".split(),
28))
29CLEANERS = {"email_clean", "phone_number_clean", "identity_number_clean", "province_clean", "title_clean"}
30SCALERS = {"max_min_normalization", "scope_normalization", "z_score", "discretization"}
31
32
33def clean(value, operation):
34    if value is None:
35        return "" if operation == "province_clean" else None
36    if not isinstance(value, str):
37        raise ValueError("cleaning columns must contain strings")
38    value = value.translate(HALF_WIDTH)
39    if operation == "title_clean":
40        return value
41    if operation == "email_clean":
42        value = value.lower()
43        return value if re.fullmatch(r"[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z0-9-]+", value) else ""
44    if operation == "phone_number_clean":
45        value = re.sub("[^0-9]", "", value)
46        return value if re.fullmatch(r"1[34578][0-9]{9}", value) else None
47    if operation == "identity_number_clean":
48        value = re.sub("[^0-9X]", "", value.upper()).lstrip("0")
49        if re.fullmatch("[0-9]{15}", value):
50            return value
51        if re.fullmatch("[0-9]{17}[0-9X]", value):
52            weights = (7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
53            checksum = sum(int(digit) * weight for digit, weight in zip(value, weights))
54            if value[-1] == "10X98765432"[checksum % 11]:
55                return value
56        return None
57    normalized = value.strip().casefold().replace(" ", "-")
58    for pinyin, name in PROVINCES.items():
59        if normalized in {pinyin, name.casefold(), name + "省", name + "市"}:
60            return name
61        if name.endswith("自治区") and normalized in {"内蒙古", "宁夏", "广西", "新疆", "西藏"} and name.startswith(normalized):
62            return name
63    return value
64
65
66def names(text):
67    values = [value.strip() for value in text.split(",")]
68    if not all(values) or len(values) != len(set(values)):
69        raise ValueError("column names must be unique and nonempty")
70    return values
71
72
73def field_indices(schema, columns):
74    indices = [schema.get_field_index(name) for name in columns]
75    if -1 in indices:
76        raise ValueError("input includes an unknown or ambiguous column")
77    return indices
78
79
80def scale_columns(operation, params):
81    if operation == "z_score":
82        ins, outs = names(params["inputCols"]), names(params["outputCols"])
83        if len(ins) != len(outs):
84            raise ValueError("inputCols and outputCols must contain the same number of names")
85        return list(zip(ins, outs))
86    return [(params["inputCol"], params["outputCol"])]
87
88
89def scale(values, operation, params):
90    available = [float(value) for value in values if value is not None]
91    if not all(map(math.isfinite, available)):
92        raise ValueError("scaling requires finite numeric input")
93    if not available:
94        return values
95    if operation == "discretization":
96        count = params["numBins"]
97        if not 2 <= count <= 1000:
98            raise ValueError("numBins must be from 2 to 1000")
99        if params["method"] == "EqualWidth":
100            if any(value < 0 or value > count for value in available):
101                raise ValueError("EqualWidth accepts values from zero through numBins")
102            apply = lambda value: min(int(value), count - 1)
103        elif params["method"] == "EqualFrequency":
104            ordered = sorted(available)
105            cuts = sorted({ordered[min(len(ordered) - 1, math.ceil(len(ordered) * i / count) - 1)]
106                           for i in range(1, count)} - {ordered[0], ordered[-1]})
107            apply = lambda value: bisect.bisect_right(cuts, value)
108        else:
109            raise ValueError("method must be EqualWidth or EqualFrequency")
110    else:
111        if operation == "z_score":
112            origin = statistics.mean(available)
113            spread = statistics.stdev(available) if len(available) > 1 else 0
114        else:
115            origin = min(available)
116            spread = max(available) - origin
117        low, high = params["range"] if operation == "scope_normalization" else (0, 1)
118        if not math.isfinite(low) or not math.isfinite(high) or low >= high:
119            raise ValueError("range must have two increasing finite bounds")
120        apply = lambda value: (value - origin) / spread * (high - low) + low if spread else None
121    return [None if value is None else apply(value) for value in values]
122
123
124def replace_column(table, name, values):
125    index = table.schema.get_field_index(name)
126    return table.append_column(name, values) if index < 0 else table.set_column(index, name, values)
127
128
129def collect(chunks):
130    data = bytearray()
131    for chunk in chunks:
132        data.extend(chunk)
133        if len(data) > MAX_INPUT:
134            raise ValueError("input exceeds 8 MiB")
135    return bytes(data)
136
137
138def collect_records(port, schema):
139    batches, size, rows = [], 0, 0
140    for batch in port.records():
141        size += batch.nbytes
142        rows += batch.num_rows
143        if size > MAX_INPUT or rows > 100_000:
144            raise ValueError("batch operation accepts at most 8 MiB and 100,000 records")
145        batches.append(batch)
146    return pa.Table.from_batches(batches, schema=schema)
147
148
149def json_rows(data):
150    def invalid_number(value):
151        raise ValueError("non-finite JSON number: " + value)
152
153    root = json.loads(data, parse_constant=invalid_number)
154    rows = root if isinstance(root, list) else [root]
155    if not all(isinstance(row, dict) for row in rows):
156        raise ValueError("JSON input must be an object or an array of objects")
157    # Arrow's Python row inference reads its columns from the first row.
158    columns = dict.fromkeys(key for row in rows for key in row)
159    return pa.table({column: [row.get(column) for row in rows] for column in columns})
160
161
162class XmlBuilder(ElementTree.TreeBuilder):
163    def doctype(self, name, pubid, system):
164        raise ValueError("XML document types are unsupported")
165
166
167def xml_root(data):
168    if len(data) > MAX_INPUT:
169        raise ValueError("XML exceeds 8 MiB")
170    return ElementTree.fromstring(data, parser=ElementTree.XMLParser(target=XmlBuilder()))
171
172
173def xml_rows(root, operation, params):
174    if operation == "xml_parser":
175        selected = list(root.iter(params["rowTag"]))
176        columns = list(dict.fromkeys(child.tag for row in selected for child in row))
177    else:
178        tags = names(params["label"])
179        if root.tag != tags[0] or len(tags) < 2:
180            raise ValueError("label must start with the root and name a repeated child tag")
181        selected = root.findall("/".join(tags[1:]))
182        columns = names(params["schema"])
183    return pa.table({column: pa.array([row.findtext(column) or None for row in selected], type=pa.string())
184                     for column in columns})
185
186
187def xml_model(node, depth=0):
188    if depth >= 32:
189        raise ValueError("XML nesting exceeds 32 levels")
190    if not node.attrib and not len(node):
191        return node.text
192    model = {"@" + name: value for name, value in node.attrib.items()}
193    if node.text and node.text.strip():
194        model["#text"] = node.text
195    groups = {}
196    for child in node:
197        groups.setdefault(child.tag, []).append(xml_model(child, depth + 1))
198    model.update({key: values[0] if len(values) == 1 else values for key, values in groups.items()})
199    return model
200
201
202def parse_xml_column(value):
203    if value is None:
204        return None
205    root = xml_root(value)
206    return json.dumps({root.tag: xml_model(root)}, ensure_ascii=False)
207
208
209def put_xml_row(root, row):
210    target = ElementTree.SubElement(root, "ROW")
211    for name, value in row.items():
212        if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_.-]*", name):
213            raise ValueError("record column is not an XML name: " + name)
214        if isinstance(value, (list, dict)):
215            raise ValueError("XML serialization accepts scalar columns")
216        if value is not None:
217            ElementTree.SubElement(target, name).text = str(value)
218
219
220def emit(ctx, table, port="result"):
221    out = ctx.records_output(port, table.schema)
222    for batch in table.to_batches(max_chunksize=1024):
223        out.write(batch)
224
225
226def emit_tables(ctx, tables):
227    outputs = {name: ctx.records_output(name, table.schema) for name, table in tables.items()}
228    for name, table in tables.items():
229        for batch in table.to_batches(max_chunksize=1024):
230            outputs[name].write(batch)
231
232
233def emit_graph(ctx, edges, vertices):
234    emit_tables(ctx, {
235        "edges": pa.Table.from_pylist(edges, schema=collections.EDGE_SCHEMA),
236        "vertex": pa.Table.from_pylist(vertices, schema=collections.VERTEX_SCHEMA),
237    })
238
239
240def inferred_schema(operation, schema, params):
241    if operation == "word_spliter":
242        return pa.schema([("words", pa.string())])
243    if operation == "average_temperature":
244        return tabular.SENSOR_SCHEMA
245    if operation in {"data_sort", "complement_by_memcache"}:
246        return schema
247    if operation == "field_type_conversion" and schema is not None:
248        return tabular.numeric_schema(schema, params, names)
249    if operation == "convert_schema_type" and schema is not None:
250        return tabular.convert_schema_type(schema, params, names)
251    if operation == "all_fields_clean_neo4j" and schema is not None:
252        return tabular.string_schema(schema)
253    if operation == "person_id_expansion" and schema is not None:
254        removed = [params["expandIDField"], params["expandIDType"]]
255        field_indices(schema, removed)
256        retained = [field for field in schema if field.name not in removed]
257        added = [*collections.ID_FIELDS, "source", "uuid", "id_hash"]
258        if len(set(removed)) != 2 or any(field.name in added for field in retained):
259            raise ValueError("input names conflict with generated identity fields")
260        return pa.schema(retained + [pa.field(name, pa.string()) for name in added], metadata=schema.metadata)
261    if operation == "folder_csv_parser":
262        return pa.schema([(name, pa.string()) for name in names(params["schema"])])
263    if operation == "select_files_by_name":
264        return pa.schema([("path", pa.string())])
265    if operation == "card_code_clean" and schema is not None:
266        field = schema.field(field_indices(schema, [params["cardCodeField"], params["cardTypeField"]])[0])
267        if not pa.types.is_string(field.type):
268            raise ValueError("cardCodeField must be a string column")
269        return schema
270    if operation == "regex_text_process" and schema is not None:
271        column = schema.field(field_indices(schema, [params["columnName"]])[0])
272        if not pa.types.is_string(column.type) or column.name + "_new" in schema.names:
273            raise ValueError("input must be a string column with an unused _new field name")
274        return schema.append(pa.field(column.name + "_new", pa.string()))
275    if operation == "pdf_parser":
276        return pa.schema([("page", pa.int64()), ("text", pa.string())])
277    if operation in {"docx_parser", "html_parser"}:
278        return documents.RECORDS_TEXT
279    if operation == "pptx_parser":
280        return documents.RECORDS_TEXT.append(pa.field("slide", pa.int64()))
281    if operation == "xml_parser_generate_new_field" and schema is not None:
282        field_indices(schema, [params["decoderField"]])
283        name = params["newField"]
284        if not name or name in schema.names:
285            raise ValueError("newField must be nonempty and absent from the input")
286        return schema.append(pa.field(name, pa.string()))
287    if operation in CLEANERS | {"xml_parser_columns"} and schema is not None:
288        columns = params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"]
289        indices = field_indices(schema, names(columns))
290        if any(not pa.types.is_string(schema.field(i).type) and not pa.types.is_large_string(schema.field(i).type)
291               for i in indices):
292            raise ValueError("selected columns must be strings")
293        return pa.schema([pa.field(field.name, pa.string()) if i in indices else field
294                          for i, field in enumerate(schema)], metadata=schema.metadata)
295    if operation in SCALERS and schema is not None:
296        for source, target in scale_columns(operation, params):
297            field = schema.field(field_indices(schema, [source])[0])
298            if not (pa.types.is_integer(field.type) or pa.types.is_floating(field.type)):
299                raise ValueError("scaling requires integer or floating-point columns")
300        for source, target in scale_columns(operation, params):
301            field = pa.field(target, pa.float64())
302            index = schema.get_field_index(target)
303            schema = schema.append(field) if index == -1 else schema.set(index, field)
304    if operation == "json_string_parser":
305        return json_rows(params["jsonString"]).schema
306    if operation == "xml_string_parser":
307        return pa.schema([(column, pa.string()) for column in names(params["schema"])])
308    return schema if operation in CLEANERS | SCALERS | {"xml_parser_columns"} else None
309
310
311def process(ctx, operation):
312    params = ctx.params
313    if operation in {"json_string_parser", "xml_string_parser"}:
314        if operation == "json_string_parser":
315            emit(ctx, json_rows(params["jsonString"]))
316        else:
317            emit(ctx, xml_rows(xml_root(params["XmlString"]), operation, params))
318        return
319    if operation == "excel_parser":
320        emit(ctx, workbooks.first_sheet_collection(ctx, collect))
321        return
322    if operation in collections.FOLDER_OPERATORS:
323        emit(ctx, collections.folder_tables(ctx, operation, params, collect, json_rows, xml_root, xml_rows, names))
324        return
325    if operation == "complement_by_memcache":
326        lookup = ctx.input("lookup")
327        materialized = collect_records(lookup, lookup.schema)
328        tabular.enrich_records(ctx, materialized, ctx.input("source").schema, names)
329        return
330    if operation == "excel_write_multiple_sheets":
331        collections.excel_workbook(ctx, collect_records)
332        return
333    if operation == "label_propagation":
334        edges = ctx.input("edges")
335        edge_rows = collect_records(edges, edges.schema).to_pylist()
336        vertex = ctx.input("vertex")
337        vertex_rows = collect_records(vertex, vertex.schema).to_pylist()
338        labels = collections.label_propagation(edge_rows, vertex_rows, params["maxIter"])
339        emit_graph(ctx, edge_rows, labels)
340        return
341    source = ctx.input("source")
342    if operation == "word_spliter":
343        import jieba
344        import unicodedata
345
346        value = collect(source.bytes()).decode("utf-8-sig")
347        text = "".join(char for char in value if not char.isspace()
348                       and not unicodedata.category(char).startswith(("P", "S")))
349        words = ({"words": word} for word in jieba.cut_for_search(text, HMM=False))
350        out = ctx.records_output("result", inferred_schema(operation, None, params))
351        for rows in iter(lambda: list(islice(words, 1024)), []):
352            out.write(pa.RecordBatch.from_pylist(rows, schema=inferred_schema(operation, None, params)))
353    elif operation == "rdf_to_df":
354        emit_tables(ctx, rdf.parse_rdf(collect(source.bytes()), params["syntax"]))
355    elif operation == "excel_write_sort":
356        table = collect_records(source, source.schema)
357        workbooks.write_sorted(ctx, tabular.sorted_table(table, params, names))
358    elif operation == "load_graph":
359        edges, vertices = collections.load_graph(collect(source.bytes()))
360        emit_graph(ctx, edges, vertices)
361    elif operation == "select_files_by_name":
362        schema = inferred_schema(operation, source.schema, params)
363        index = field_indices(source.schema, [params["pathColumn"]])[0]
364        pattern = params["selectionConditions"]
365        if not isinstance(pattern, str) or len(pattern) > 256:
366            raise ValueError("selectionConditions must be a glob up to 256 characters")
367        out = ctx.records_output("result", schema)
368        for batch in source.records():
369            matching = [value for value in batch.column(index).to_pylist() if collections.file_matches(value, pattern)]
370            out.write(pa.RecordBatch.from_arrays([pa.array(matching, type=pa.string())], schema=schema))
371    elif operation == "regex_text_process":
372        schema = inferred_schema(operation, source.schema, params)
373        index = field_indices(schema, [params["columnName"]])[0]
374        pattern, replacement = params["regex"], params["replaceStr"]
375        if len(pattern) > 1024 or len(replacement) > 2048:
376            raise ValueError("regular expression or replacement exceeds its size limit")
377        out = ctx.records_output("result", schema)
378        for batch in source.records():
379            # RE2 executes without backtracking. Limit input and expansion before calling Arrow.
380            column = batch.column(index)
381            widths = pc.binary_length(column).to_pylist()
382            max_bytes = sum((width + 1) * (len(replacement.encode()) + 1) for width in widths if width is not None)
383            if max_bytes > 32 * 1024 * 1024:
384                raise ValueError("potential regex expansion exceeds 32 MiB in one batch")
385            changed = pc.replace_substring_regex(column, pattern, replacement)
386            out.write(pa.RecordBatch.from_arrays([*batch.columns, changed], schema=schema))
387    elif operation in {"data_sort", "average_temperature"}:
388        table = collect_records(source, source.schema)
389        result = tabular.average_temperature(table.to_pylist()) if operation == "average_temperature" else tabular.sorted_table(table, params, names)
390        emit(ctx, result)
391    elif operation in {"field_type_conversion", "convert_schema_type", "all_fields_clean_neo4j"}:
392        schema = inferred_schema(operation, source.schema, params)
393        out = ctx.records_output("result", schema)
394        for batch in source.records():
395            columns = []
396            for index, field in enumerate(schema):
397                column = pc.cast(batch.column(index), field.type)
398                if operation == "all_fields_clean_neo4j":
399                    column = pc.replace_substring(column, '"', '')
400                columns.append(column)
401            out.write(pa.RecordBatch.from_arrays(columns, schema=schema))
402    elif operation in {"card_code_clean", "person_id_expansion"}:
403        schema = inferred_schema(operation, source.schema, params)
404        out = ctx.records_output("result", schema)
405        for batch in source.records():
406            rows = batch.to_pylist()
407            if operation == "card_code_clean":
408                for row in rows:
409                    if str(row[params["cardTypeField"]]) == "1":
410                        key = params["cardCodeField"]
411                        row[key] = clean(row[key], "identity_number_clean")
412            else:
413                rows = [collections.expand_id(row, params) for row in rows]
414            out.write(pa.RecordBatch.from_pylist(rows, schema=schema))
415    elif operation in {"docx_parser", "pptx_parser", "html_parser", "pdf_parser", "flatten_xml_parser",
416                       "evaluate_json_path", "decompress_gzip"}:
417        data = collect(source.bytes())
418        if operation == "decompress_gzip":
419            out = ctx.bytes_output("result")
420            for chunk in documents.decompress(data):
421                out.write(chunk)
422            return
423        if operation == "html_parser":
424            table = documents.html_text(data)
425        elif operation == "pdf_parser":
426            table = documents.pdf_text(data, inferred_schema(operation, None, params))
427        elif operation in {"docx_parser", "pptx_parser"}:
428            table = documents.office_text(data, operation, xml_root)
429        elif operation == "evaluate_json_path":
430            table = documents.json_field(data, params["tag"], json_rows)
431        else:
432            table = documents.flatten_xml(xml_root(data), params)
433        emit(ctx, table)
434    elif operation == "xml_parser_generate_new_field":
435        schema = inferred_schema(operation, source.schema, params)
436        source_index = schema.get_field_index(params["decoderField"])
437        out = ctx.records_output("result", schema)
438        for batch in source.records():
439            values = [documents.xml_value(value, params["tagXPath"], xml_root)
440                      for value in batch.column(source_index).to_pylist()]
441            out.write(pa.RecordBatch.from_arrays([*batch.columns, pa.array(values, type=pa.string())], schema=schema))
442    elif operation in {"json_parser", "xml_parser"}:
443        data = collect(source.bytes())
444        emit(ctx, json_rows(data) if operation == "json_parser" else xml_rows(xml_root(data), operation, params))
445    elif operation == "json_save":
446        out = ctx.bytes_output("result")
447        for batch in source.records():
448            for row in batch.to_pylist():
449                out.write((json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n").encode())
450    elif operation == "xml_save":
451        out = ctx.bytes_output("result")
452        out.write(b"<ROWS>")
453        for batch in source.records():
454            for row in batch.to_pylist():
455                temp = ElementTree.Element("ROWS")
456                put_xml_row(temp, row)
457                out.write(ElementTree.tostring(temp[0], encoding="utf-8"))
458        out.write(b"</ROWS>")
459    elif operation in SCALERS:
460        schema = source.schema
461        inferred_schema(operation, schema, params)
462        original = collect_records(source, schema)
463        table = original
464        for origin, target in scale_columns(operation, params):
465            values = scale(original[origin].to_pylist(), operation, params)
466            table = replace_column(table, target, pa.array(values, type=pa.float64()))
467        emit(ctx, table)
468    else:
469        schema = inferred_schema(operation, source.schema, params)
470        selected = names(params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"])
471        indices = field_indices(source.schema, selected)
472        out = ctx.records_output("result", schema)
473        for batch in source.records():
474            arrays = list(batch.columns)
475            for index in indices:
476                values = arrays[index].to_pylist()
477                cleaned = (parse_xml_column(value) if operation == "xml_parser_columns" else clean(value, operation)
478                           for value in values)
479                arrays[index] = pa.array(cleaned, type=pa.string())
480            out.write(pa.RecordBatch.from_arrays(arrays, schema=schema))
481
482
483def start(operation, output_view):
484    output_views = output_view if isinstance(output_view, dict) else {"result": output_view}
485    def execute(ctx):
486        process(ctx, operation)
487
488    def infer(inputs, params):
489        outputs = {}
490        for port, view in output_views.items():
491            schema = None
492            if view == "records":
493                if operation == "rdf_to_df":
494                    schema = rdf.ENTITY if port == "entityOut" else rdf.RELATION
495                elif operation in {"load_graph", "label_propagation"}:
496                    schema = collections.EDGE_SCHEMA if port == "edges" else collections.VERTEX_SCHEMA
497                else:
498                    source = inputs.get("source")
499                    schema = inferred_schema(operation, records_schema(source) if source else None, params)
500            formats = ["arrow"] if view == "records" else ["xlsx"] if operation in {"excel_write_multiple_sheets", "excel_write_sort"} else ["binary"]
501            outputs[port] = descriptor([view], formats, schema,
502                                       certainty="known" if schema is not None or view == "bytes" else "unknown")
503        return result(outputs)
504
505    def preview(inputs, params, limits):
506        if operation in collections.FOLDER_OPERATORS or operation in {"excel_parser", "excel_write_multiple_sheets", "excel_write_sort"}:
507            return {}
508        return preview_process(execute, inputs, params, limits)
509
510    run_operator(execute, infer, preview_outputs=preview)