src/piflow_portable.py
1"""Arrow stream adapters for portable PiFlow components."""
2
3import bisect
4import json
5import math
6import re
7import statistics
8from xml.etree import ElementTree
9
10import pyarrow as pa
11import pyarrow.compute as pc
12import piflow_documents as documents
13import piflow_collections as collections
14from xstream_operator import descriptor, preview_process, records_schema, result, run_operator
15
16MAX_INPUT = 8 * 1024 * 1024
17HALF_WIDTH = str.maketrans({**{c: c - 65248 for c in range(0xff01, 0xff5f)}, 0x3000: 32})
18PROVINCES = dict(zip(
19 "beijing tianjin shanghai chongqing anhui fujian gansu guangdong guizhou hainan hebei heilongjiang "
20 "henan hubei hunan jiangsu jiangxi jilin liaoning qinghai shaanxi shanxi shandong sichuan yunnan "
21 "zhejiang inner-mongolia ningxia guangxi xinjiang tibet".split(),
22 "北京 天津 上海 重庆 安徽 福建 甘肃 广东 贵州 海南 河北 黑龙江 河南 湖北 湖南 江苏 江西 吉林 辽宁 "
23 "青海 陕西 山西 山东 四川 云南 浙江 内蒙古自治区 宁夏回族自治区 广西壮族自治区 新疆维吾尔自治区 西藏自治区".split(),
24))
25CLEANERS = {"email_clean", "phone_number_clean", "identity_number_clean", "province_clean", "title_clean"}
26SCALERS = {"max_min_normalization", "scope_normalization", "z_score", "discretization"}
27
28
29def clean(value, operation):
30 if value is None:
31 return "" if operation == "province_clean" else None
32 if not isinstance(value, str):
33 raise ValueError("cleaning columns must contain strings")
34 value = value.translate(HALF_WIDTH)
35 if operation == "title_clean":
36 return value
37 if operation == "email_clean":
38 value = value.lower()
39 return value if re.fullmatch(r"[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z0-9-]+", value) else ""
40 if operation == "phone_number_clean":
41 value = re.sub("[^0-9]", "", value)
42 return value if re.fullmatch(r"1[34578][0-9]{9}", value) else None
43 if operation == "identity_number_clean":
44 value = re.sub("[^0-9X]", "", value.upper()).lstrip("0")
45 if re.fullmatch("[0-9]{15}", value):
46 return value
47 if re.fullmatch("[0-9]{17}[0-9X]", value):
48 weights = (7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
49 checksum = sum(int(digit) * weight for digit, weight in zip(value, weights))
50 if value[-1] == "10X98765432"[checksum % 11]:
51 return value
52 return None
53 normalized = value.strip().casefold().replace(" ", "-")
54 for pinyin, name in PROVINCES.items():
55 if normalized in {pinyin, name.casefold(), name + "省", name + "市"}:
56 return name
57 if name.endswith("自治区") and normalized in {"内蒙古", "宁夏", "广西", "新疆", "西藏"} and name.startswith(normalized):
58 return name
59 return value
60
61
62def names(text):
63 values = [value.strip() for value in text.split(",")]
64 if not all(values) or len(values) != len(set(values)):
65 raise ValueError("column names must be unique and nonempty")
66 return values
67
68
69def field_indices(schema, columns):
70 indices = [schema.get_field_index(name) for name in columns]
71 if -1 in indices:
72 raise ValueError("input includes an unknown or ambiguous column")
73 return indices
74
75
76def scale_columns(operation, params):
77 if operation == "z_score":
78 ins, outs = names(params["inputCols"]), names(params["outputCols"])
79 if len(ins) != len(outs):
80 raise ValueError("inputCols and outputCols must contain the same number of names")
81 return list(zip(ins, outs))
82 return [(params["inputCol"], params["outputCol"])]
83
84
85def scale(values, operation, params):
86 available = [float(value) for value in values if value is not None]
87 if not all(map(math.isfinite, available)):
88 raise ValueError("scaling requires finite numeric input")
89 if not available:
90 return values
91 if operation == "discretization":
92 count = params["numBins"]
93 if not 2 <= count <= 1000:
94 raise ValueError("numBins must be from 2 to 1000")
95 if params["method"] == "EqualWidth":
96 if any(value < 0 or value > count for value in available):
97 raise ValueError("EqualWidth accepts values from zero through numBins")
98 apply = lambda value: min(int(value), count - 1)
99 elif params["method"] == "EqualFrequency":
100 ordered = sorted(available)
101 cuts = sorted({ordered[min(len(ordered) - 1, math.ceil(len(ordered) * i / count) - 1)]
102 for i in range(1, count)} - {ordered[0], ordered[-1]})
103 apply = lambda value: bisect.bisect_right(cuts, value)
104 else:
105 raise ValueError("method must be EqualWidth or EqualFrequency")
106 else:
107 if operation == "z_score":
108 origin = statistics.mean(available)
109 spread = statistics.stdev(available) if len(available) > 1 else 0
110 else:
111 origin = min(available)
112 spread = max(available) - origin
113 low, high = params["range"] if operation == "scope_normalization" else (0, 1)
114 if not math.isfinite(low) or not math.isfinite(high) or low >= high:
115 raise ValueError("range must have two increasing finite bounds")
116 apply = lambda value: (value - origin) / spread * (high - low) + low if spread else None
117 return [None if value is None else apply(value) for value in values]
118
119
120def replace_column(table, name, values):
121 index = table.schema.get_field_index(name)
122 return table.append_column(name, values) if index < 0 else table.set_column(index, name, values)
123
124
125def collect(chunks):
126 data = bytearray()
127 for chunk in chunks:
128 data.extend(chunk)
129 if len(data) > MAX_INPUT:
130 raise ValueError("input exceeds 8 MiB")
131 return bytes(data)
132
133
134def collect_records(port, schema):
135 batches, size, rows = [], 0, 0
136 for batch in port.records():
137 size += batch.nbytes
138 rows += batch.num_rows
139 if size > MAX_INPUT or rows > 100_000:
140 raise ValueError("batch operation accepts at most 8 MiB and 100,000 records")
141 batches.append(batch)
142 return pa.Table.from_batches(batches, schema=schema)
143
144
145def json_rows(data):
146 def invalid_number(value):
147 raise ValueError("non-finite JSON number: " + value)
148
149 root = json.loads(data, parse_constant=invalid_number)
150 rows = root if isinstance(root, list) else [root]
151 if not all(isinstance(row, dict) for row in rows):
152 raise ValueError("JSON input must be an object or an array of objects")
153 # Arrow's Python row inference reads its columns from the first row.
154 columns = dict.fromkeys(key for row in rows for key in row)
155 return pa.table({column: [row.get(column) for row in rows] for column in columns})
156
157
158class XmlBuilder(ElementTree.TreeBuilder):
159 def doctype(self, name, pubid, system):
160 raise ValueError("XML document types are unsupported")
161
162
163def xml_root(data):
164 if len(data) > MAX_INPUT:
165 raise ValueError("XML exceeds 8 MiB")
166 return ElementTree.fromstring(data, parser=ElementTree.XMLParser(target=XmlBuilder()))
167
168
169def xml_rows(root, operation, params):
170 if operation == "xml_parser":
171 selected = list(root.iter(params["rowTag"]))
172 columns = list(dict.fromkeys(child.tag for row in selected for child in row))
173 else:
174 tags = names(params["label"])
175 if root.tag != tags[0] or len(tags) < 2:
176 raise ValueError("label must start with the root and name a repeated child tag")
177 selected = root.findall("/".join(tags[1:]))
178 columns = names(params["schema"])
179 return pa.table({column: pa.array([row.findtext(column) or None for row in selected], type=pa.string())
180 for column in columns})
181
182
183def xml_model(node, depth=0):
184 if depth >= 32:
185 raise ValueError("XML nesting exceeds 32 levels")
186 if not node.attrib and not len(node):
187 return node.text
188 model = {"@" + name: value for name, value in node.attrib.items()}
189 if node.text and node.text.strip():
190 model["#text"] = node.text
191 groups = {}
192 for child in node:
193 groups.setdefault(child.tag, []).append(xml_model(child, depth + 1))
194 model.update({key: values[0] if len(values) == 1 else values for key, values in groups.items()})
195 return model
196
197
198def parse_xml_column(value):
199 if value is None:
200 return None
201 root = xml_root(value)
202 return json.dumps({root.tag: xml_model(root)}, ensure_ascii=False)
203
204
205def put_xml_row(root, row):
206 target = ElementTree.SubElement(root, "ROW")
207 for name, value in row.items():
208 if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_.-]*", name):
209 raise ValueError("record column is not an XML name: " + name)
210 if isinstance(value, (list, dict)):
211 raise ValueError("XML serialization accepts scalar columns")
212 if value is not None:
213 ElementTree.SubElement(target, name).text = str(value)
214
215
216def emit(ctx, table, port="result"):
217 out = ctx.records_output(port, table.schema)
218 for batch in table.to_batches(max_chunksize=1024):
219 out.write(batch)
220
221
222def emit_graph(ctx, edges, vertices):
223 tables = {
224 "edges": pa.Table.from_pylist(edges, schema=collections.EDGE_SCHEMA),
225 "vertex": pa.Table.from_pylist(vertices, schema=collections.VERTEX_SCHEMA),
226 }
227 outputs = {name: ctx.records_output(name, table.schema) for name, table in tables.items()}
228 for name, table in tables.items():
229 for batch in table.to_batches(max_chunksize=1024):
230 outputs[name].write(batch)
231
232
233def inferred_schema(operation, schema, params):
234 if operation == "person_id_expansion" and schema is not None:
235 removed = [params["expandIDField"], params["expandIDType"]]
236 field_indices(schema, removed)
237 retained = [field for field in schema if field.name not in removed]
238 added = [*collections.ID_FIELDS, "source", "uuid", "id_hash"]
239 if len(set(removed)) != 2 or any(field.name in added for field in retained):
240 raise ValueError("input names conflict with generated identity fields")
241 return pa.schema(retained + [pa.field(name, pa.string()) for name in added], metadata=schema.metadata)
242 if operation == "folder_csv_parser":
243 return pa.schema([(name, pa.string()) for name in names(params["schema"])])
244 if operation == "select_files_by_name":
245 return pa.schema([("path", pa.string())])
246 if operation == "card_code_clean" and schema is not None:
247 field = schema.field(field_indices(schema, [params["cardCodeField"], params["cardTypeField"]])[0])
248 if not pa.types.is_string(field.type):
249 raise ValueError("cardCodeField must be a string column")
250 return schema
251 if operation == "regex_text_process" and schema is not None:
252 column = schema.field(field_indices(schema, [params["columnName"]])[0])
253 if not pa.types.is_string(column.type) or column.name + "_new" in schema.names:
254 raise ValueError("input must be a string column with an unused _new field name")
255 return schema.append(pa.field(column.name + "_new", pa.string()))
256 if operation == "pdf_parser":
257 return pa.schema([("page", pa.int64()), ("text", pa.string())])
258 if operation in {"docx_parser", "html_parser"}:
259 return documents.RECORDS_TEXT
260 if operation == "pptx_parser":
261 return documents.RECORDS_TEXT.append(pa.field("slide", pa.int64()))
262 if operation == "xml_parser_generate_new_field" and schema is not None:
263 field_indices(schema, [params["decoderField"]])
264 name = params["newField"]
265 if not name or name in schema.names:
266 raise ValueError("newField must be nonempty and absent from the input")
267 return schema.append(pa.field(name, pa.string()))
268 if operation in CLEANERS | {"xml_parser_columns"} and schema is not None:
269 columns = params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"]
270 indices = field_indices(schema, names(columns))
271 if any(not pa.types.is_string(schema.field(i).type) and not pa.types.is_large_string(schema.field(i).type)
272 for i in indices):
273 raise ValueError("selected columns must be strings")
274 return pa.schema([pa.field(field.name, pa.string()) if i in indices else field
275 for i, field in enumerate(schema)], metadata=schema.metadata)
276 if operation in SCALERS and schema is not None:
277 for source, target in scale_columns(operation, params):
278 field = schema.field(field_indices(schema, [source])[0])
279 if not (pa.types.is_integer(field.type) or pa.types.is_floating(field.type)):
280 raise ValueError("scaling requires integer or floating-point columns")
281 for source, target in scale_columns(operation, params):
282 field = pa.field(target, pa.float64())
283 index = schema.get_field_index(target)
284 schema = schema.append(field) if index == -1 else schema.set(index, field)
285 if operation == "json_string_parser":
286 return json_rows(params["jsonString"]).schema
287 if operation == "xml_string_parser":
288 return pa.schema([(column, pa.string()) for column in names(params["schema"])])
289 return schema if operation in CLEANERS | SCALERS | {"xml_parser_columns"} else None
290
291
292def process(ctx, operation):
293 params = ctx.params
294 if operation in {"json_string_parser", "xml_string_parser"}:
295 if operation == "json_string_parser":
296 emit(ctx, json_rows(params["jsonString"]))
297 else:
298 emit(ctx, xml_rows(xml_root(params["XmlString"]), operation, params))
299 return
300 if operation in collections.FOLDER_OPERATORS:
301 emit(ctx, collections.folder_tables(ctx, operation, params, collect, json_rows, xml_root, xml_rows, names))
302 return
303 if operation == "excel_write_multiple_sheets":
304 collections.excel_workbook(ctx, collect_records)
305 return
306 if operation == "label_propagation":
307 edges = ctx.input("edges")
308 edge_rows = collect_records(edges, edges.schema).to_pylist()
309 vertex = ctx.input("vertex")
310 vertex_rows = collect_records(vertex, vertex.schema).to_pylist()
311 labels = collections.label_propagation(edge_rows, vertex_rows, params["maxIter"])
312 emit_graph(ctx, edge_rows, labels)
313 return
314 source = ctx.input("source")
315 if operation == "load_graph":
316 edges, vertices = collections.load_graph(collect(source.bytes()))
317 emit_graph(ctx, edges, vertices)
318 elif operation == "select_files_by_name":
319 schema = inferred_schema(operation, source.schema, params)
320 index = field_indices(source.schema, [params["pathColumn"]])[0]
321 pattern = params["selectionConditions"]
322 if not isinstance(pattern, str) or len(pattern) > 256:
323 raise ValueError("selectionConditions must be a glob up to 256 characters")
324 out = ctx.records_output("result", schema)
325 for batch in source.records():
326 matching = [value for value in batch.column(index).to_pylist() if collections.file_matches(value, pattern)]
327 out.write(pa.RecordBatch.from_arrays([pa.array(matching, type=pa.string())], schema=schema))
328 elif operation == "regex_text_process":
329 schema = inferred_schema(operation, source.schema, params)
330 index = field_indices(schema, [params["columnName"]])[0]
331 pattern, replacement = params["regex"], params["replaceStr"]
332 if len(pattern) > 1024 or len(replacement) > 2048:
333 raise ValueError("regular expression or replacement exceeds its size limit")
334 out = ctx.records_output("result", schema)
335 for batch in source.records():
336 # RE2 executes without backtracking. Limit input and expansion before calling Arrow.
337 column = batch.column(index)
338 widths = pc.binary_length(column).to_pylist()
339 max_bytes = sum((width + 1) * (len(replacement.encode()) + 1) for width in widths if width is not None)
340 if max_bytes > 32 * 1024 * 1024:
341 raise ValueError("potential regex expansion exceeds 32 MiB in one batch")
342 changed = pc.replace_substring_regex(column, pattern, replacement)
343 out.write(pa.RecordBatch.from_arrays([*batch.columns, changed], schema=schema))
344 elif operation in {"card_code_clean", "person_id_expansion"}:
345 schema = inferred_schema(operation, source.schema, params)
346 out = ctx.records_output("result", schema)
347 for batch in source.records():
348 rows = batch.to_pylist()
349 if operation == "card_code_clean":
350 for row in rows:
351 if str(row[params["cardTypeField"]]) == "1":
352 key = params["cardCodeField"]
353 row[key] = clean(row[key], "identity_number_clean")
354 else:
355 rows = [collections.expand_id(row, params) for row in rows]
356 out.write(pa.RecordBatch.from_pylist(rows, schema=schema))
357 elif operation in {"docx_parser", "pptx_parser", "html_parser", "pdf_parser", "flatten_xml_parser",
358 "evaluate_json_path", "decompress_gzip"}:
359 data = collect(source.bytes())
360 if operation == "decompress_gzip":
361 out = ctx.bytes_output("result")
362 for chunk in documents.decompress(data):
363 out.write(chunk)
364 return
365 if operation == "html_parser":
366 table = documents.html_text(data)
367 elif operation == "pdf_parser":
368 table = documents.pdf_text(data, inferred_schema(operation, None, params))
369 elif operation in {"docx_parser", "pptx_parser"}:
370 table = documents.office_text(data, operation, xml_root)
371 elif operation == "evaluate_json_path":
372 table = documents.json_field(data, params["tag"], json_rows)
373 else:
374 table = documents.flatten_xml(xml_root(data), params)
375 emit(ctx, table)
376 elif operation == "xml_parser_generate_new_field":
377 schema = inferred_schema(operation, source.schema, params)
378 source_index = schema.get_field_index(params["decoderField"])
379 out = ctx.records_output("result", schema)
380 for batch in source.records():
381 values = [documents.xml_value(value, params["tagXPath"], xml_root)
382 for value in batch.column(source_index).to_pylist()]
383 out.write(pa.RecordBatch.from_arrays([*batch.columns, pa.array(values, type=pa.string())], schema=schema))
384 elif operation in {"json_parser", "xml_parser"}:
385 data = collect(source.bytes())
386 emit(ctx, json_rows(data) if operation == "json_parser" else xml_rows(xml_root(data), operation, params))
387 elif operation == "json_save":
388 out = ctx.bytes_output("result")
389 for batch in source.records():
390 for row in batch.to_pylist():
391 out.write((json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n").encode())
392 elif operation == "xml_save":
393 out = ctx.bytes_output("result")
394 out.write(b"<ROWS>")
395 for batch in source.records():
396 for row in batch.to_pylist():
397 temp = ElementTree.Element("ROWS")
398 put_xml_row(temp, row)
399 out.write(ElementTree.tostring(temp[0], encoding="utf-8"))
400 out.write(b"</ROWS>")
401 elif operation in SCALERS:
402 schema = source.schema
403 inferred_schema(operation, schema, params)
404 original = collect_records(source, schema)
405 table = original
406 for origin, target in scale_columns(operation, params):
407 values = scale(original[origin].to_pylist(), operation, params)
408 table = replace_column(table, target, pa.array(values, type=pa.float64()))
409 emit(ctx, table)
410 else:
411 schema = inferred_schema(operation, source.schema, params)
412 selected = names(params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"])
413 indices = field_indices(source.schema, selected)
414 out = ctx.records_output("result", schema)
415 for batch in source.records():
416 arrays = list(batch.columns)
417 for index in indices:
418 values = arrays[index].to_pylist()
419 cleaned = (parse_xml_column(value) if operation == "xml_parser_columns" else clean(value, operation)
420 for value in values)
421 arrays[index] = pa.array(cleaned, type=pa.string())
422 out.write(pa.RecordBatch.from_arrays(arrays, schema=schema))
423
424
425def start(operation, output_view):
426 output_views = output_view if isinstance(output_view, dict) else {"result": output_view}
427 def execute(ctx):
428 process(ctx, operation)
429
430 def infer(inputs, params):
431 outputs = {}
432 for port, view in output_views.items():
433 schema = None
434 if view == "records":
435 if operation in {"load_graph", "label_propagation"}:
436 schema = collections.EDGE_SCHEMA if port == "edges" else collections.VERTEX_SCHEMA
437 else:
438 source = inputs.get("source")
439 schema = inferred_schema(operation, records_schema(source) if source else None, params)
440 formats = ["arrow"] if view == "records" else ["xlsx"] if operation == "excel_write_multiple_sheets" else ["binary"]
441 outputs[port] = descriptor([view], formats, schema,
442 certainty="known" if schema is not None or view == "bytes" else "unknown")
443 return result(outputs)
444
445 def preview(inputs, params, limits):
446 if operation in collections.FOLDER_OPERATORS or operation == "excel_write_multiple_sheets":
447 return {}
448 return preview_process(execute, inputs, params, limits)
449
450 run_operator(execute, infer, preview_outputs=preview)