src/xstream_piflow_distinct.py
1"""PiFlow Distinct: keep the first row per key across input batches."""
2import pickle
3from dacp_piflow_support import params, columns, indexes, MAX_BYTES
4
5
6def process(ctx):
7 p = params(ctx, optional=['columnNames'])
8 source = ctx.input('source')
9 names = columns(p.get('columnNames', ''), allow_empty=True) or source.schema.names
10 indexes(source.schema, names)
11 out = ctx.records_output('result', source.schema)
12 seen, size = set(), 0
13 for batch in source.records():
14 keep = []
15 for i, row in enumerate(batch.select(names).to_pylist()):
16 key = pickle.dumps(row, protocol=4)
17 if key not in seen:
18 size += len(key) + 128
19 if size > MAX_BYTES:
20 raise ValueError('distinct key state exceeds 128 MiB')
21 seen.add(key)
22 keep.append(i)
23 if keep:
24 out.write(batch.take(keep))
25
26
27def infer_outputs(inputs, params):
28 import pyarrow as pa
29 from xstream_operator import descriptor, result, records_schema
30 expected = {'source'}
31 if set(inputs) != expected:
32 raise ValueError('unexpected input ports')
33 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
34 schema = records_schema(inputs['source']) if 'source' in inputs else None
35 if schema is not None:
36 outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
37 return result(outputs)
38
39
40def preview_outputs(inputs, params, limits):
41 from xstream_operator import preview_process
42 return preview_process(process, inputs, params, limits)
43
44
45if __name__ == '__main__':
46 from xstream_operator import run_operator
47 run_operator(process, infer_outputs, preview_outputs=preview_outputs)