src/piflow_portable.py
1"""Arrow stream adapters for portable PiFlow components."""
2
3import bisect
4import json
5import math
6import re
7import statistics
8from xml.etree import ElementTree
9
10import pyarrow as pa
11from xstream_operator import descriptor, preview_process, records_schema, result, run_operator
12
13MAX_INPUT = 8 * 1024 * 1024
14HALF_WIDTH = str.maketrans({**{c: c - 65248 for c in range(0xff01, 0xff5f)}, 0x3000: 32})
15PROVINCES = dict(zip(
16 "beijing tianjin shanghai chongqing anhui fujian gansu guangdong guizhou hainan hebei heilongjiang "
17 "henan hubei hunan jiangsu jiangxi jilin liaoning qinghai shaanxi shanxi shandong sichuan yunnan "
18 "zhejiang inner-mongolia ningxia guangxi xinjiang tibet".split(),
19 "北京 天津 上海 重庆 安徽 福建 甘肃 广东 贵州 海南 河北 黑龙江 河南 湖北 湖南 江苏 江西 吉林 辽宁 "
20 "青海 陕西 山西 山东 四川 云南 浙江 内蒙古自治区 宁夏回族自治区 广西壮族自治区 新疆维吾尔自治区 西藏自治区".split(),
21))
22CLEANERS = {"email_clean", "phone_number_clean", "identity_number_clean", "province_clean", "title_clean"}
23SCALERS = {"max_min_normalization", "scope_normalization", "z_score", "discretization"}
24
25
26def clean(value, operation):
27 if value is None:
28 return "" if operation == "province_clean" else None
29 if not isinstance(value, str):
30 raise ValueError("cleaning columns must contain strings")
31 value = value.translate(HALF_WIDTH)
32 if operation == "title_clean":
33 return value
34 if operation == "email_clean":
35 value = value.lower()
36 return value if re.fullmatch(r"[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z0-9-]+", value) else ""
37 if operation == "phone_number_clean":
38 value = re.sub("[^0-9]", "", value)
39 return value if re.fullmatch(r"1[34578][0-9]{9}", value) else None
40 if operation == "identity_number_clean":
41 value = re.sub("[^0-9X]", "", value.upper()).lstrip("0")
42 if re.fullmatch("[0-9]{15}", value):
43 return value
44 if re.fullmatch("[0-9]{17}[0-9X]", value):
45 weights = (7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2)
46 checksum = sum(int(digit) * weight for digit, weight in zip(value, weights))
47 if value[-1] == "10X98765432"[checksum % 11]:
48 return value
49 return None
50 normalized = value.strip().casefold().replace(" ", "-")
51 for pinyin, name in PROVINCES.items():
52 if normalized in {pinyin, name.casefold(), name + "省", name + "市"}:
53 return name
54 if name.endswith("自治区") and normalized in {"内蒙古", "宁夏", "广西", "新疆", "西藏"} and name.startswith(normalized):
55 return name
56 return value
57
58
59def names(text):
60 values = [value.strip() for value in text.split(",")]
61 if not all(values) or len(values) != len(set(values)):
62 raise ValueError("column names must be unique and nonempty")
63 return values
64
65
66def field_indices(schema, columns):
67 indices = [schema.get_field_index(name) for name in columns]
68 if -1 in indices:
69 raise ValueError("input includes an unknown or ambiguous column")
70 return indices
71
72
73def scale_columns(operation, params):
74 if operation == "z_score":
75 ins, outs = names(params["inputCols"]), names(params["outputCols"])
76 if len(ins) != len(outs):
77 raise ValueError("inputCols and outputCols must contain the same number of names")
78 return list(zip(ins, outs))
79 return [(params["inputCol"], params["outputCol"])]
80
81
82def scale(values, operation, params):
83 available = [float(value) for value in values if value is not None]
84 if not all(map(math.isfinite, available)):
85 raise ValueError("scaling requires finite numeric input")
86 if not available:
87 return values
88 if operation == "discretization":
89 count = params["numBins"]
90 if not 2 <= count <= 1000:
91 raise ValueError("numBins must be from 2 to 1000")
92 if params["method"] == "EqualWidth":
93 if any(value < 0 or value > count for value in available):
94 raise ValueError("EqualWidth accepts values from zero through numBins")
95 apply = lambda value: min(int(value), count - 1)
96 elif params["method"] == "EqualFrequency":
97 ordered = sorted(available)
98 cuts = sorted({ordered[min(len(ordered) - 1, math.ceil(len(ordered) * i / count) - 1)]
99 for i in range(1, count)} - {ordered[0], ordered[-1]})
100 apply = lambda value: bisect.bisect_right(cuts, value)
101 else:
102 raise ValueError("method must be EqualWidth or EqualFrequency")
103 else:
104 if operation == "z_score":
105 origin = statistics.mean(available)
106 spread = statistics.stdev(available) if len(available) > 1 else 0
107 else:
108 origin = min(available)
109 spread = max(available) - origin
110 low, high = params["range"] if operation == "scope_normalization" else (0, 1)
111 if not math.isfinite(low) or not math.isfinite(high) or low >= high:
112 raise ValueError("range must have two increasing finite bounds")
113 apply = lambda value: (value - origin) / spread * (high - low) + low if spread else None
114 return [None if value is None else apply(value) for value in values]
115
116
117def replace_column(table, name, values):
118 index = table.schema.get_field_index(name)
119 return table.append_column(name, values) if index < 0 else table.set_column(index, name, values)
120
121
122def collect(chunks):
123 data = bytearray()
124 for chunk in chunks:
125 data.extend(chunk)
126 if len(data) > MAX_INPUT:
127 raise ValueError("input exceeds 8 MiB")
128 return bytes(data)
129
130
131def collect_records(port, schema):
132 batches, size, rows = [], 0, 0
133 for batch in port.records():
134 size += batch.nbytes
135 rows += batch.num_rows
136 if size > MAX_INPUT or rows > 100_000:
137 raise ValueError("batch operation accepts at most 8 MiB and 100,000 records")
138 batches.append(batch)
139 return pa.Table.from_batches(batches, schema=schema)
140
141
142def json_rows(data):
143 def invalid_number(value):
144 raise ValueError("non-finite JSON number: " + value)
145
146 root = json.loads(data, parse_constant=invalid_number)
147 rows = root if isinstance(root, list) else [root]
148 if not all(isinstance(row, dict) for row in rows):
149 raise ValueError("JSON input must be an object or an array of objects")
150 # Arrow's Python row inference reads its columns from the first row.
151 columns = dict.fromkeys(key for row in rows for key in row)
152 return pa.table({column: [row.get(column) for row in rows] for column in columns})
153
154
155class XmlBuilder(ElementTree.TreeBuilder):
156 def doctype(self, name, pubid, system):
157 raise ValueError("XML document types are unsupported")
158
159
160def xml_root(data):
161 if len(data) > MAX_INPUT:
162 raise ValueError("XML exceeds 8 MiB")
163 return ElementTree.fromstring(data, parser=ElementTree.XMLParser(target=XmlBuilder()))
164
165
166def xml_rows(root, operation, params):
167 if operation == "xml_parser":
168 selected = list(root.iter(params["rowTag"]))
169 columns = list(dict.fromkeys(child.tag for row in selected for child in row))
170 else:
171 tags = names(params["label"])
172 if root.tag != tags[0] or len(tags) < 2:
173 raise ValueError("label must start with the root and name a repeated child tag")
174 selected = root.findall("/".join(tags[1:]))
175 columns = names(params["schema"])
176 return pa.table({column: pa.array([row.findtext(column) or None for row in selected], type=pa.string())
177 for column in columns})
178
179
180def xml_model(node, depth=0):
181 if depth >= 32:
182 raise ValueError("XML nesting exceeds 32 levels")
183 if not node.attrib and not len(node):
184 return node.text
185 model = {"@" + name: value for name, value in node.attrib.items()}
186 if node.text and node.text.strip():
187 model["#text"] = node.text
188 groups = {}
189 for child in node:
190 groups.setdefault(child.tag, []).append(xml_model(child, depth + 1))
191 model.update({key: values[0] if len(values) == 1 else values for key, values in groups.items()})
192 return model
193
194
195def parse_xml_column(value):
196 if value is None:
197 return None
198 root = xml_root(value)
199 return json.dumps({root.tag: xml_model(root)}, ensure_ascii=False)
200
201
202def put_xml_row(root, row):
203 target = ElementTree.SubElement(root, "ROW")
204 for name, value in row.items():
205 if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_.-]*", name):
206 raise ValueError("record column is not an XML name: " + name)
207 if isinstance(value, (list, dict)):
208 raise ValueError("XML serialization accepts scalar columns")
209 if value is not None:
210 ElementTree.SubElement(target, name).text = str(value)
211
212
213def emit(ctx, table):
214 out = ctx.records_output("result", table.schema)
215 for batch in table.to_batches(max_chunksize=1024):
216 out.write(batch)
217
218
219def inferred_schema(operation, schema, params):
220 if operation in CLEANERS | {"xml_parser_columns"} and schema is not None:
221 columns = params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"]
222 indices = field_indices(schema, names(columns))
223 if any(not pa.types.is_string(schema.field(i).type) and not pa.types.is_large_string(schema.field(i).type)
224 for i in indices):
225 raise ValueError("selected columns must be strings")
226 return pa.schema([pa.field(field.name, pa.string()) if i in indices else field
227 for i, field in enumerate(schema)], metadata=schema.metadata)
228 if operation in SCALERS and schema is not None:
229 for source, target in scale_columns(operation, params):
230 field = schema.field(field_indices(schema, [source])[0])
231 if not (pa.types.is_integer(field.type) or pa.types.is_floating(field.type)):
232 raise ValueError("scaling requires integer or floating-point columns")
233 for source, target in scale_columns(operation, params):
234 field = pa.field(target, pa.float64())
235 index = schema.get_field_index(target)
236 schema = schema.append(field) if index == -1 else schema.set(index, field)
237 if operation == "json_string_parser":
238 return json_rows(params["jsonString"]).schema
239 if operation == "xml_string_parser":
240 return pa.schema([(column, pa.string()) for column in names(params["schema"])])
241 return schema if operation in CLEANERS | SCALERS | {"xml_parser_columns"} else None
242
243
244def process(ctx, operation):
245 params = ctx.params
246 if operation in {"json_string_parser", "xml_string_parser"}:
247 if operation == "json_string_parser":
248 emit(ctx, json_rows(params["jsonString"]))
249 else:
250 emit(ctx, xml_rows(xml_root(params["XmlString"]), operation, params))
251 return
252 source = ctx.input("source")
253 if operation in {"json_parser", "xml_parser"}:
254 data = collect(source.bytes())
255 emit(ctx, json_rows(data) if operation == "json_parser" else xml_rows(xml_root(data), operation, params))
256 elif operation == "json_save":
257 out = ctx.bytes_output("result")
258 for batch in source.records():
259 for row in batch.to_pylist():
260 out.write((json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n").encode())
261 elif operation == "xml_save":
262 out = ctx.bytes_output("result")
263 out.write(b"<ROWS>")
264 for batch in source.records():
265 for row in batch.to_pylist():
266 temp = ElementTree.Element("ROWS")
267 put_xml_row(temp, row)
268 out.write(ElementTree.tostring(temp[0], encoding="utf-8"))
269 out.write(b"</ROWS>")
270 elif operation in SCALERS:
271 schema = source.schema
272 inferred_schema(operation, schema, params)
273 original = collect_records(source, schema)
274 table = original
275 for origin, target in scale_columns(operation, params):
276 values = scale(original[origin].to_pylist(), operation, params)
277 table = replace_column(table, target, pa.array(values, type=pa.float64()))
278 emit(ctx, table)
279 else:
280 schema = inferred_schema(operation, source.schema, params)
281 selected = names(params["xmlColumns"] if operation == "xml_parser_columns" else params["columnName"])
282 indices = field_indices(source.schema, selected)
283 out = ctx.records_output("result", schema)
284 for batch in source.records():
285 arrays = list(batch.columns)
286 for index in indices:
287 values = arrays[index].to_pylist()
288 cleaned = (parse_xml_column(value) if operation == "xml_parser_columns" else clean(value, operation)
289 for value in values)
290 arrays[index] = pa.array(cleaned, type=pa.string())
291 out.write(pa.RecordBatch.from_arrays(arrays, schema=schema))
292
293
294def start(operation, output_view):
295 def execute(ctx):
296 process(ctx, operation)
297
298 def infer(inputs, params):
299 schema = None
300 if output_view == "records":
301 source = inputs.get("source")
302 schema = inferred_schema(operation, records_schema(source) if source else None, params)
303 formats = ["arrow"] if output_view == "records" else ["binary"]
304 return result({"result": descriptor([output_view], formats, schema,
305 certainty="known" if schema is not None or output_view == "bytes" else "unknown")})
306
307 def preview(inputs, params, limits):
308 return preview_process(execute, inputs, params, limits)
309
310 run_operator(execute, infer, preview_outputs=preview)