src/xstream_piflow_convert_schema.py
1"""PiFlow ConvertSchema renames columns; it does not cast their types."""
2import pyarrow as pa
3from dacp_piflow_support import params, columns
4
5
6def process(ctx):
7 p = params(ctx, ['schema'])
8 pairs = columns(p['schema'])
9 mapping = []
10 for pair in pairs:
11 split = [s.strip() for s in pair.split('->')]
12 if len(split) != 2 or not all(split):
13 raise ValueError('schema must be old->new mappings separated by commas')
14 mapping.append(split)
15 source = ctx.input('source')
16 schema = source.schema
17 names = schema.names
18 for old, new in mapping:
19 names = [new if n == old else n for n in names]
20 if len(set(names)) != len(names):
21 raise ValueError('renaming creates duplicate columns')
22 out_schema = pa.schema([field.with_name(n) for field, n in zip(schema, names)], metadata=schema.metadata)
23 out = ctx.records_output('result', out_schema)
24 for batch in source.records():
25 out.write(pa.RecordBatch.from_arrays(batch.columns, schema=out_schema))
26
27
28def infer_outputs(inputs, params):
29 import pyarrow as pa
30 from xstream_operator import descriptor, result, records_schema
31 expected = {'source'}
32 if set(inputs) != expected:
33 raise ValueError('unexpected input ports')
34 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
35 return result(outputs)
36
37
38def preview_outputs(inputs, params, limits):
39 from xstream_operator import preview_process
40 return preview_process(process, inputs, params, limits)
41
42
43if __name__ == '__main__':
44 from xstream_operator import run_operator
45 run_operator(process, infer_outputs, preview_outputs=preview_outputs)