src/xstream_piflow_select_field.py
1"""PiFlow SelectField: project columns in the requested order, batch by batch."""
2def process(ctx):
3 if set(ctx.params) != {'columnNames'} or not isinstance(ctx.params['columnNames'], str):
4 raise ValueError('columnNames must be a comma-separated string')
5 names = [s.strip() for s in ctx.params['columnNames'].split(',')]
6 if not all(names) or len(names) != len(set(names)):
7 raise ValueError('columnNames must contain unique, nonempty names')
8 source = ctx.input('source')
9 indexes = [source.schema.get_field_index(n) for n in names]
10 if min(indexes) < 0:
11 raise ValueError('unknown or ambiguous column')
12 import pyarrow as pa
13 schema = pa.schema([source.schema.field(i) for i in indexes], metadata=source.schema.metadata)
14 out = ctx.records_output('result', schema)
15 for batch in source.records():
16 out.write(batch.select(indexes))
17
18
19def infer_outputs(inputs, params):
20 import pyarrow as pa
21 from xstream_operator import descriptor, result, records_schema
22 expected = {'source'}
23 if set(inputs) != expected:
24 raise ValueError('unexpected input ports')
25 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
26 schema = records_schema(inputs['source']) if 'source' in inputs else None
27 names = params.get('columnNames')
28 if not isinstance(names, str) or not names or len(set(names.split(','))) != len(names.split(',')):
29 raise ValueError('columnNames must contain unique names')
30 names = [n.strip() for n in names.split(',')]
31 if schema is not None:
32 if any(schema.get_field_index(n) < 0 for n in names):
33 raise ValueError('unknown or ambiguous column')
34 schema = pa.schema([schema.field(n) for n in names], metadata=schema.metadata)
35 if schema is not None:
36 outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
37 return result(outputs)
38
39
40def preview_outputs(inputs, params, limits):
41 from xstream_operator import preview_process
42 return preview_process(process, inputs, params, limits)
43
44
45if __name__ == '__main__':
46 from xstream_operator import run_operator
47 run_operator(process, infer_outputs, preview_outputs=preview_outputs)