src/xstream_piflow_subtract.py
1"""PiFlow Subtract: set difference over entire records (EXCEPT DISTINCT)."""
2import pickle
3import pyarrow as pa
4from dacp_piflow_support import params, materialize, write_table
5
6
7def process(ctx):
8 params(ctx)
9 left = materialize(ctx.input('left'))
10 right = materialize(ctx.input('right'))
11 if left.schema != right.schema:
12 raise ValueError('subtract requires identical ordered schemas')
13 excluded = {pickle.dumps(row) for row in right.to_pylist()}
14 seen, keep = set(), []
15 for i, row in enumerate(left.to_pylist()):
16 key = pickle.dumps(row)
17 if key not in excluded and key not in seen:
18 seen.add(key)
19 keep.append(i)
20 write_table(ctx, left.take(pa.array(keep, type=pa.int64())))
21
22
23def infer_outputs(inputs, params):
24 import pyarrow as pa
25 from xstream_operator import descriptor, result, records_schema
26 expected = {'right', 'left'}
27 if set(inputs) != expected:
28 raise ValueError('unexpected input ports')
29 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
30 schema = records_schema(inputs['source']) if 'source' in inputs else None
31 if schema is not None:
32 outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
33 return result(outputs)
34
35
36def preview_outputs(inputs, params, limits):
37 from xstream_operator import preview_process
38 return preview_process(process, inputs, params, limits)
39
40
41if __name__ == '__main__':
42 from xstream_operator import run_operator
43 run_operator(process, infer_outputs, preview_outputs=preview_outputs)