Source code

Download source archive ↓

src/xstream_piflow_drop_field.py

1"""PiFlow DropField: remove named columns from each Arrow batch."""
2from dacp_piflow_support import params, columns
3import pyarrow as pa
4
5
6def process(ctx):
7    p = params(ctx, ['columnNames'])
8    names = columns(p['columnNames'])
9    source = ctx.input('source')
10    # Spark drop ignores absent names; retain that behavior.
11    keep = [n for n in source.schema.names if n not in names]
12    if not keep:
13        raise ValueError('at least one output column is required')
14    schema = source.schema
15    out = ctx.records_output('result', pa.schema([schema.field(n) for n in keep], metadata=schema.metadata))
16    for batch in source.records():
17        out.write(batch.select(keep))
18
19
20def infer_outputs(inputs, params):
21    import pyarrow as pa
22    from xstream_operator import descriptor, result, records_schema
23    expected = {'source'}
24    if set(inputs) != expected:
25        raise ValueError('unexpected input ports')
26    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
27    schema = records_schema(inputs['source']) if 'source' in inputs else None
28    names = params.get('columnNames')
29    if not isinstance(names, str) or not names or len(set(names.split(','))) != len(names.split(',')):
30        raise ValueError('columnNames must contain unique names')
31    names = [n.strip() for n in names.split(',')]
32    if schema is not None:
33        fields = [f for f in schema if f.name not in names]
34        if not fields:
35            raise ValueError('at least one output column is required')
36        schema = pa.schema(fields, metadata=schema.metadata)
37    if schema is not None:
38        outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
39    return result(outputs)
40
41
42def preview_outputs(inputs, params, limits):
43    from xstream_operator import preview_process
44    return preview_process(process, inputs, params, limits)
45
46
47if __name__ == '__main__':
48    from xstream_operator import run_operator
49    run_operator(process, infer_outputs, preview_outputs=preview_outputs)