Source code

Download source archive ↓

src/dacp_piflow_merge.py

1"""PiFlow Merge adapted to two explicit input ports and a streaming union."""
2from dacp_piflow_support import params
3
4
5def process(ctx):
6    params(ctx)
7    left = ctx.input('left')
8    schema = left.schema
9    out = ctx.records_output('result', schema)
10    for batch in left.records():
11        out.write(batch)
12    right = ctx.input('right')
13    if right.schema != schema:
14        raise ValueError('merge requires identical ordered schemas')
15    for batch in right.records():
16        out.write(batch)
17
18
19def infer_outputs(inputs, params):
20    import pyarrow as pa
21    from dacp_operator import descriptor, result, records_schema
22    expected = {'right', 'left'}
23    if set(inputs) != expected:
24        raise ValueError('unexpected input ports')
25    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
26    return result(outputs)
27
28
29def preview_outputs(inputs, params, limits):
30    from dacp_operator import preview_process
31    return preview_process(process, inputs, params, limits)
32
33
34if __name__ == '__main__':
35    from dacp_operator import run_operator
36    run_operator(process, infer_outputs, preview_outputs=preview_outputs)