Source code

Download source archive ↓

src/xstream_piflow_distinct.py

1"""PiFlow Distinct: keep the first row per key across input batches."""
2import pickle
3from dacp_piflow_support import params, columns, indexes, MAX_BYTES
4
5
6def process(ctx):
7    p = params(ctx, optional=['columnNames'])
8    source = ctx.input('source')
9    names = columns(p.get('columnNames', ''), allow_empty=True) or source.schema.names
10    indexes(source.schema, names)
11    out = ctx.records_output('result', source.schema)
12    seen, size = set(), 0
13    for batch in source.records():
14        keep = []
15        for i, row in enumerate(batch.select(names).to_pylist()):
16            key = pickle.dumps(row, protocol=4)
17            if key not in seen:
18                size += len(key) + 128
19                if size > MAX_BYTES:
20                    raise ValueError('distinct key state exceeds 128 MiB')
21                seen.add(key)
22                keep.append(i)
23        if keep:
24            out.write(batch.take(keep))
25
26
27def infer_outputs(inputs, params):
28    import pyarrow as pa
29    from xstream_operator import descriptor, result, records_schema
30    expected = {'source'}
31    if set(inputs) != expected:
32        raise ValueError('unexpected input ports')
33    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
34    schema = records_schema(inputs['source']) if 'source' in inputs else None
35    if schema is not None:
36        outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
37    return result(outputs)
38
39
40def preview_outputs(inputs, params, limits):
41    from xstream_operator import preview_process
42    return preview_process(process, inputs, params, limits)
43
44
45if __name__ == '__main__':
46    from xstream_operator import run_operator
47    run_operator(process, infer_outputs, preview_outputs=preview_outputs)