src/xstream_piflow_consistency_check.py
1"""PiFlow ConsistencyCheck: left join reference keys and prepend a 0/1 flag."""
2import pyarrow as pa
3from dacp_piflow_support import params, columns, indexes, materialize, write_table
4
5
6def process(ctx):
7 p = params(ctx, ['flagField', 'correlationColumn'])
8 names = columns(p['correlationColumn'])
9 flag = p['flagField']
10 left = materialize(ctx.input('left'))
11 right = materialize(ctx.input('right'))
12 indexes(left.schema, names)
13 indexes(right.schema, names)
14 if not isinstance(flag, str) or not flag or flag in left.column_names:
15 raise ValueError('flagField must name a new column')
16 import pickle
17 counts = {}
18 for row in right.select(names).to_pylist():
19 if any(v is None for v in row.values()):
20 continue
21 key = pickle.dumps(row)
22 counts[key] = counts.get(key, 0) + 1
23 schema = pa.schema([pa.field(flag, pa.int32(), nullable=False), *left.schema])
24 out = ctx.records_output('result', schema)
25 rows = []
26 for row in left.to_pylist():
27 keys = {n: row[n] for n in names}
28 count = 0 if any(v is None for v in keys.values()) else counts.get(pickle.dumps(keys), 0)
29 # Preserve left-join multiplicity when the reference has duplicate keys.
30 for _ in range(max(count, 1)):
31 rows.append({flag: 0 if count else 1, **row})
32 if len(rows) == 256:
33 out.write(pa.RecordBatch.from_pylist(rows, schema=schema))
34 rows.clear()
35 if rows:
36 out.write(pa.RecordBatch.from_pylist(rows, schema=schema))
37
38
39def infer_outputs(inputs, params):
40 import pyarrow as pa
41 from xstream_operator import descriptor, result, records_schema
42 expected = {'right', 'left'}
43 if set(inputs) != expected:
44 raise ValueError('unexpected input ports')
45 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
46 schema = records_schema(inputs['source']) if 'source' in inputs else None
47 if schema is not None:
48 outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
49 return result(outputs)
50
51
52def preview_outputs(inputs, params, limits):
53 from xstream_operator import preview_process
54 return preview_process(process, inputs, params, limits)
55
56
57if __name__ == '__main__':
58 from xstream_operator import run_operator
59 run_operator(process, infer_outputs, preview_outputs=preview_outputs)