Source code

Download source archive ↓

src/xstream_piflow_convert_schema.py

1"""PiFlow ConvertSchema renames columns; it does not cast their types."""
2import pyarrow as pa
3from dacp_piflow_support import params, columns
4
5
6def process(ctx):
7    p = params(ctx, ['schema'])
8    pairs = columns(p['schema'])
9    mapping = []
10    for pair in pairs:
11        split = [s.strip() for s in pair.split('->')]
12        if len(split) != 2 or not all(split):
13            raise ValueError('schema must be old->new mappings separated by commas')
14        mapping.append(split)
15    source = ctx.input('source')
16    schema = source.schema
17    names = schema.names
18    for old, new in mapping:
19        names = [new if n == old else n for n in names]
20    if len(set(names)) != len(names):
21        raise ValueError('renaming creates duplicate columns')
22    out_schema = pa.schema([field.with_name(n) for field, n in zip(schema, names)], metadata=schema.metadata)
23    out = ctx.records_output('result', out_schema)
24    for batch in source.records():
25        out.write(pa.RecordBatch.from_arrays(batch.columns, schema=out_schema))
26
27
28def infer_outputs(inputs, params):
29    import pyarrow as pa
30    from xstream_operator import descriptor, result, records_schema
31    expected = {'source'}
32    if set(inputs) != expected:
33        raise ValueError('unexpected input ports')
34    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
35    return result(outputs)
36
37
38def preview_outputs(inputs, params, limits):
39    from xstream_operator import preview_process
40    return preview_process(process, inputs, params, limits)
41
42
43if __name__ == '__main__':
44    from xstream_operator import run_operator
45    run_operator(process, infer_outputs, preview_outputs=preview_outputs)