Source code

Download source archive ↓

src/xstream_piflow_consistency_check.py

1"""PiFlow ConsistencyCheck: left join reference keys and prepend a 0/1 flag."""
2import pyarrow as pa
3from dacp_piflow_support import params, columns, indexes, materialize, write_table
4
5
6def process(ctx):
7    p = params(ctx, ['flagField', 'correlationColumn'])
8    names = columns(p['correlationColumn'])
9    flag = p['flagField']
10    left = materialize(ctx.input('left'))
11    right = materialize(ctx.input('right'))
12    indexes(left.schema, names)
13    indexes(right.schema, names)
14    if not isinstance(flag, str) or not flag or flag in left.column_names:
15        raise ValueError('flagField must name a new column')
16    import pickle
17    counts = {}
18    for row in right.select(names).to_pylist():
19        if any(v is None for v in row.values()):
20            continue
21        key = pickle.dumps(row)
22        counts[key] = counts.get(key, 0) + 1
23    schema = pa.schema([pa.field(flag, pa.int32(), nullable=False), *left.schema])
24    out = ctx.records_output('result', schema)
25    rows = []
26    for row in left.to_pylist():
27        keys = {n: row[n] for n in names}
28        count = 0 if any(v is None for v in keys.values()) else counts.get(pickle.dumps(keys), 0)
29        # Preserve left-join multiplicity when the reference has duplicate keys.
30        for _ in range(max(count, 1)):
31            rows.append({flag: 0 if count else 1, **row})
32            if len(rows) == 256:
33                out.write(pa.RecordBatch.from_pylist(rows, schema=schema))
34                rows.clear()
35    if rows:
36        out.write(pa.RecordBatch.from_pylist(rows, schema=schema))
37
38
39def infer_outputs(inputs, params):
40    import pyarrow as pa
41    from xstream_operator import descriptor, result, records_schema
42    expected = {'right', 'left'}
43    if set(inputs) != expected:
44        raise ValueError('unexpected input ports')
45    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
46    schema = records_schema(inputs['source']) if 'source' in inputs else None
47    if schema is not None:
48        outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
49    return result(outputs)
50
51
52def preview_outputs(inputs, params, limits):
53    from xstream_operator import preview_process
54    return preview_process(process, inputs, params, limits)
55
56
57if __name__ == '__main__':
58    from xstream_operator import run_operator
59    run_operator(process, infer_outputs, preview_outputs=preview_outputs)