Source code

Download source archive ↓

src/xstream_piflow_select_field.py

1"""PiFlow SelectField: project columns in the requested order, batch by batch."""
2def process(ctx):
3    if set(ctx.params) != {'columnNames'} or not isinstance(ctx.params['columnNames'], str):
4        raise ValueError('columnNames must be a comma-separated string')
5    names = [s.strip() for s in ctx.params['columnNames'].split(',')]
6    if not all(names) or len(names) != len(set(names)):
7        raise ValueError('columnNames must contain unique, nonempty names')
8    source = ctx.input('source')
9    indexes = [source.schema.get_field_index(n) for n in names]
10    if min(indexes) < 0:
11        raise ValueError('unknown or ambiguous column')
12    import pyarrow as pa
13    schema = pa.schema([source.schema.field(i) for i in indexes], metadata=source.schema.metadata)
14    out = ctx.records_output('result', schema)
15    for batch in source.records():
16        out.write(batch.select(indexes))
17
18
19def infer_outputs(inputs, params):
20    import pyarrow as pa
21    from xstream_operator import descriptor, result, records_schema
22    expected = {'source'}
23    if set(inputs) != expected:
24        raise ValueError('unexpected input ports')
25    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
26    schema = records_schema(inputs['source']) if 'source' in inputs else None
27    names = params.get('columnNames')
28    if not isinstance(names, str) or not names or len(set(names.split(','))) != len(names.split(',')):
29        raise ValueError('columnNames must contain unique names')
30    names = [n.strip() for n in names.split(',')]
31    if schema is not None:
32        if any(schema.get_field_index(n) < 0 for n in names):
33            raise ValueError('unknown or ambiguous column')
34        schema = pa.schema([schema.field(n) for n in names], metadata=schema.metadata)
35    if schema is not None:
36        outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
37    return result(outputs)
38
39
40def preview_outputs(inputs, params, limits):
41    from xstream_operator import preview_process
42    return preview_process(process, inputs, params, limits)
43
44
45if __name__ == '__main__':
46    from xstream_operator import run_operator
47    run_operator(process, infer_outputs, preview_outputs=preview_outputs)