Source code

Download source archive ↓

src/piflow_collections.py

1"""Graph and collections adapters shared by several PiFlow ports."""
2
3from collections import Counter
4import csv
5from fnmatch import fnmatchcase
6import hashlib
7import io
8import json
9from pathlib import PurePosixPath
10import uuid
11
12import pyarrow as pa
13
14EDGE_SCHEMA = pa.schema([("srcId", pa.int64()), ("dstId", pa.int64()), ("attr", pa.int64())])
15VERTEX_SCHEMA = pa.schema([("id", pa.int64()), ("attr", pa.int64())])
16GRAPH_LIMIT = 100_000
17ID_FIELDS = ("0", "identity_card", "military_id", "passport", "4", "home_return_permit",
18             "mainland_travel_permit_for_taiwan_residents")
19FOLDER_OPERATORS = {"folder_csv_parser", "folder_json_parser", "multi_folder_json_parser", "xml_parser_folder"}
20
21
22def load_graph(data):
23    edges = []
24    vertices = set()
25    for line in data.decode("utf-8").splitlines():
26        if not line.strip() or line.lstrip().startswith("#"):
27            continue
28        fields = line.split()
29        if len(fields) != 2:
30            raise ValueError("each graph edge must contain exactly two integer vertex identifiers")
31        source, target = map(int, fields)
32        edges.append(dict(srcId=source, dstId=target, attr=1))
33        vertices.update((source, target))
34        if len(edges) > GRAPH_LIMIT:
35            raise ValueError("edge limit exceeded")
36    return edges, [dict(id=vertex, attr=1) for vertex in sorted(vertices)]
37
38
39def label_propagation(edges, vertices, iterations):
40    if not 1 <= iterations <= 50:
41        raise ValueError("maxIter must be from 1 to 50")
42    labels = {row["id"]: row["id"] for row in vertices}
43    if len(labels) != len(vertices) or len(labels) > GRAPH_LIMIT or len(edges) > GRAPH_LIMIT:
44        raise ValueError("vertex IDs must be unique and both streams support up to 100,000 rows")
45    neighbors = {key: [] for key in labels}
46    for edge in edges:
47        source, target = edge["srcId"], edge["dstId"]
48        if source not in neighbors or target not in neighbors:
49            raise ValueError("each edge must reference known vertices")
50        neighbors[source].append(target)
51        neighbors[target].append(source)
52    for _ in range(iterations):
53        new_labels = {}
54        for vertex, adjacent in neighbors.items():
55            votes = Counter(labels[key] for key in adjacent)
56            new_labels[vertex] = min(votes, key=lambda label: (-votes[label], label)) if votes else labels[vertex]
57        if new_labels == labels:
58            break
59        labels = new_labels
60    return [dict(id=key, attr=value) for key, value in sorted(labels.items())]
61
62
63def folder_tables(ctx, operation, params, collect, json_rows, xml_root, xml_rows, names):
64    extension = ".csv" if operation == "folder_csv_parser" else ".xml" if operation == "xml_parser_folder" else ".json"
65    tables, total_bytes, count = [], 0, 0
66    if extension == ".csv":
67        columns = names(params["schema"])
68        schema = pa.schema([(name, pa.string()) for name in columns])
69        delimiter = params["delimiter"]
70        if not isinstance(delimiter, str) or len(delimiter) != 1:
71            raise ValueError("delimiter must be one character")
72    for port in ctx.input_names:
73        for item in ctx.input(port).items():
74            if item["kind"] != "data" or not item["name"].lower().endswith(extension):
75                continue
76            count += 1
77            if count > 128:
78                raise ValueError("at most 128 matching input items are supported")
79            with ctx.open_item(item, "bytes") as stream:
80                data = collect(stream.bytes())
81            total_bytes += len(data)
82            if total_bytes > 24 * 1024 * 1024:
83                raise ValueError("matched item contents exceed 24 MiB")
84            if extension == ".csv":
85                reader = csv.reader(io.StringIO(data.decode("utf-8-sig"), newline=""), delimiter=delimiter, strict=True)
86                rows = []
87                for index, row in enumerate(reader):
88                    if index == 0 and row == columns:
89                        continue
90                    if len(row) != len(columns):
91                        raise ValueError("CSV line width differs from the declared schema")
92                    rows.append(dict(zip(columns, row)))
93                table = pa.Table.from_pylist(rows, schema=schema)
94            elif extension == ".xml":
95                table = xml_rows(xml_root(data), "xml_parser", params)
96            else:
97                table = json_rows(data)
98                if params["tag"]:
99                    table = table.select(names(params["tag"]))
100            tables.append(table)
101    if not tables:
102        if extension == ".csv":
103            return pa.Table.from_batches([], schema=schema)
104        raise ValueError("no matching items provide an output schema")
105    return pa.concat_tables(tables, promote_options="default")
106
107
108def excel_workbook(ctx, collect_records):
109    from openpyxl import Workbook
110    from openpyxl.cell import WriteOnlyCell
111
112    book = Workbook(write_only=True)
113    try:
114        for port in ctx.input_names:
115            source = ctx.input(port)
116            schema = source.schema
117            table = collect_records(source, schema)
118            sheet = book.create_sheet(port)
119
120            def append_row(values):
121                cells = []
122                for value in values:
123                    if isinstance(value, (list, dict, bytes)):
124                        raise ValueError("worksheet input accepts scalar values only")
125                    cell = WriteOnlyCell(sheet, value=value)
126                    if isinstance(value, str):
127                        cell.data_type = "s"
128                    cells.append(cell)
129                sheet.append(cells)
130
131            if ctx.params["header"] == "yes":
132                append_row(schema.names)
133            for batch in table.to_batches(max_chunksize=1024):
134                for row in batch.to_pylist():
135                    append_row([row[field] for field in schema.names])
136        data = io.BytesIO()
137        book.save(data)
138        output = ctx.bytes_output("result", "xlsx")
139        data.seek(0)
140        while chunk := data.read(64 * 1024):
141            output.write(chunk)
142    finally:
143        book.close()
144
145
146def expand_id(row, params):
147    code = row.pop(params["expandIDField"])
148    group = str(row.pop(params["expandIDType"]))
149    origin = params["sourceLabel"]
150    row.update(dict.fromkeys(ID_FIELDS))
151    if group in tuple(str(index) for index in range(len(ID_FIELDS))) and code is not None:
152        row[ID_FIELDS[int(group)]] = str(code)
153    digest = hashlib.sha256(json.dumps([group, code], ensure_ascii=False).encode()).hexdigest()
154    row.update(source=origin, id_hash=digest, uuid=str(uuid.uuid5(uuid.NAMESPACE_URL, origin + ":" + digest)))
155    return row
156
157
158def file_matches(value, pattern):
159    return isinstance(value, str) and fnmatchcase(PurePosixPath(value).name, pattern)