Source code

Download source archive ↓

src/xstream_piflow_subtract.py

1"""PiFlow Subtract: set difference over entire records (EXCEPT DISTINCT)."""
2import pickle
3import pyarrow as pa
4from dacp_piflow_support import params, materialize, write_table
5
6
7def process(ctx):
8    params(ctx)
9    left = materialize(ctx.input('left'))
10    right = materialize(ctx.input('right'))
11    if left.schema != right.schema:
12        raise ValueError('subtract requires identical ordered schemas')
13    excluded = {pickle.dumps(row) for row in right.to_pylist()}
14    seen, keep = set(), []
15    for i, row in enumerate(left.to_pylist()):
16        key = pickle.dumps(row)
17        if key not in excluded and key not in seen:
18            seen.add(key)
19            keep.append(i)
20    write_table(ctx, left.take(pa.array(keep, type=pa.int64())))
21
22
23def infer_outputs(inputs, params):
24    import pyarrow as pa
25    from xstream_operator import descriptor, result, records_schema
26    expected = {'right', 'left'}
27    if set(inputs) != expected:
28        raise ValueError('unexpected input ports')
29    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
30    schema = records_schema(inputs['source']) if 'source' in inputs else None
31    if schema is not None:
32        outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
33    return result(outputs)
34
35
36def preview_outputs(inputs, params, limits):
37    from xstream_operator import preview_process
38    return preview_process(process, inputs, params, limits)
39
40
41if __name__ == '__main__':
42    from xstream_operator import run_operator
43    run_operator(process, infer_outputs, preview_outputs=preview_outputs)