src/piflow_collections.py
1"""Graph and collections adapters shared by several PiFlow ports."""
2
3from collections import Counter
4import csv
5from fnmatch import fnmatchcase
6import hashlib
7import io
8import json
9from pathlib import PurePosixPath
10import uuid
11
12import pyarrow as pa
13
14EDGE_SCHEMA = pa.schema([("srcId", pa.int64()), ("dstId", pa.int64()), ("attr", pa.int64())])
15VERTEX_SCHEMA = pa.schema([("id", pa.int64()), ("attr", pa.int64())])
16GRAPH_LIMIT = 100_000
17ID_FIELDS = ("0", "identity_card", "military_id", "passport", "4", "home_return_permit",
18 "mainland_travel_permit_for_taiwan_residents")
19FOLDER_OPERATORS = {"folder_csv_parser", "folder_json_parser", "multi_folder_json_parser", "xml_parser_folder"}
20
21
22def load_graph(data):
23 edges = []
24 vertices = set()
25 for line in data.decode("utf-8").splitlines():
26 if not line.strip() or line.lstrip().startswith("#"):
27 continue
28 fields = line.split()
29 if len(fields) != 2:
30 raise ValueError("each graph edge must contain exactly two integer vertex identifiers")
31 source, target = map(int, fields)
32 edges.append(dict(srcId=source, dstId=target, attr=1))
33 vertices.update((source, target))
34 if len(edges) > GRAPH_LIMIT:
35 raise ValueError("edge limit exceeded")
36 return edges, [dict(id=vertex, attr=1) for vertex in sorted(vertices)]
37
38
39def label_propagation(edges, vertices, iterations):
40 if not 1 <= iterations <= 50:
41 raise ValueError("maxIter must be from 1 to 50")
42 labels = {row["id"]: row["id"] for row in vertices}
43 if len(labels) != len(vertices) or len(labels) > GRAPH_LIMIT or len(edges) > GRAPH_LIMIT:
44 raise ValueError("vertex IDs must be unique and both streams support up to 100,000 rows")
45 neighbors = {key: [] for key in labels}
46 for edge in edges:
47 source, target = edge["srcId"], edge["dstId"]
48 if source not in neighbors or target not in neighbors:
49 raise ValueError("each edge must reference known vertices")
50 neighbors[source].append(target)
51 neighbors[target].append(source)
52 for _ in range(iterations):
53 new_labels = {}
54 for vertex, adjacent in neighbors.items():
55 votes = Counter(labels[key] for key in adjacent)
56 new_labels[vertex] = min(votes, key=lambda label: (-votes[label], label)) if votes else labels[vertex]
57 if new_labels == labels:
58 break
59 labels = new_labels
60 return [dict(id=key, attr=value) for key, value in sorted(labels.items())]
61
62
63def folder_tables(ctx, operation, params, collect, json_rows, xml_root, xml_rows, names):
64 extension = ".csv" if operation == "folder_csv_parser" else ".xml" if operation == "xml_parser_folder" else ".json"
65 tables, total_bytes, count = [], 0, 0
66 if extension == ".csv":
67 columns = names(params["schema"])
68 schema = pa.schema([(name, pa.string()) for name in columns])
69 delimiter = params["delimiter"]
70 if not isinstance(delimiter, str) or len(delimiter) != 1:
71 raise ValueError("delimiter must be one character")
72 for port in ctx.input_names:
73 for item in ctx.input(port).items():
74 if item["kind"] != "data" or not item["name"].lower().endswith(extension):
75 continue
76 count += 1
77 if count > 128:
78 raise ValueError("at most 128 matching input items are supported")
79 with ctx.open_item(item, "bytes") as stream:
80 data = collect(stream.bytes())
81 total_bytes += len(data)
82 if total_bytes > 24 * 1024 * 1024:
83 raise ValueError("matched item contents exceed 24 MiB")
84 if extension == ".csv":
85 reader = csv.reader(io.StringIO(data.decode("utf-8-sig"), newline=""), delimiter=delimiter, strict=True)
86 rows = []
87 for index, row in enumerate(reader):
88 if index == 0 and row == columns:
89 continue
90 if len(row) != len(columns):
91 raise ValueError("CSV line width differs from the declared schema")
92 rows.append(dict(zip(columns, row)))
93 table = pa.Table.from_pylist(rows, schema=schema)
94 elif extension == ".xml":
95 table = xml_rows(xml_root(data), "xml_parser", params)
96 else:
97 table = json_rows(data)
98 if params["tag"]:
99 table = table.select(names(params["tag"]))
100 tables.append(table)
101 if not tables:
102 if extension == ".csv":
103 return pa.Table.from_batches([], schema=schema)
104 raise ValueError("no matching items provide an output schema")
105 return pa.concat_tables(tables, promote_options="default")
106
107
108def excel_workbook(ctx, collect_records):
109 from openpyxl import Workbook
110 from openpyxl.cell import WriteOnlyCell
111
112 book = Workbook(write_only=True)
113 try:
114 for port in ctx.input_names:
115 source = ctx.input(port)
116 schema = source.schema
117 table = collect_records(source, schema)
118 sheet = book.create_sheet(port)
119
120 def append_row(values):
121 cells = []
122 for value in values:
123 if isinstance(value, (list, dict, bytes)):
124 raise ValueError("worksheet input accepts scalar values only")
125 cell = WriteOnlyCell(sheet, value=value)
126 if isinstance(value, str):
127 cell.data_type = "s"
128 cells.append(cell)
129 sheet.append(cells)
130
131 if ctx.params["header"] == "yes":
132 append_row(schema.names)
133 for batch in table.to_batches(max_chunksize=1024):
134 for row in batch.to_pylist():
135 append_row([row[field] for field in schema.names])
136 data = io.BytesIO()
137 book.save(data)
138 output = ctx.bytes_output("result", "xlsx")
139 data.seek(0)
140 while chunk := data.read(64 * 1024):
141 output.write(chunk)
142 finally:
143 book.close()
144
145
146def expand_id(row, params):
147 code = row.pop(params["expandIDField"])
148 group = str(row.pop(params["expandIDType"]))
149 origin = params["sourceLabel"]
150 row.update(dict.fromkeys(ID_FIELDS))
151 if group in tuple(str(index) for index in range(len(ID_FIELDS))) and code is not None:
152 row[ID_FIELDS[int(group)]] = str(code)
153 digest = hashlib.sha256(json.dumps([group, code], ensure_ascii=False).encode()).hexdigest()
154 row.update(source=origin, id_hash=digest, uuid=str(uuid.uuid5(uuid.NAMESPACE_URL, origin + ":" + digest)))
155 return row
156
157
158def file_matches(value, pattern):
159 return isinstance(value, str) and fnmatchcase(PurePosixPath(value).name, pattern)