src/xstream_piflow_drop_field.py
1"""PiFlow DropField: remove named columns from each Arrow batch."""
2from dacp_piflow_support import params, columns
3import pyarrow as pa
4
5
6def process(ctx):
7 p = params(ctx, ['columnNames'])
8 names = columns(p['columnNames'])
9 source = ctx.input('source')
10 # Spark drop ignores absent names; retain that behavior.
11 keep = [n for n in source.schema.names if n not in names]
12 if not keep:
13 raise ValueError('at least one output column is required')
14 schema = source.schema
15 out = ctx.records_output('result', pa.schema([schema.field(n) for n in keep], metadata=schema.metadata))
16 for batch in source.records():
17 out.write(batch.select(keep))
18
19
20def infer_outputs(inputs, params):
21 import pyarrow as pa
22 from xstream_operator import descriptor, result, records_schema
23 expected = {'source'}
24 if set(inputs) != expected:
25 raise ValueError('unexpected input ports')
26 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
27 schema = records_schema(inputs['source']) if 'source' in inputs else None
28 names = params.get('columnNames')
29 if not isinstance(names, str) or not names or len(set(names.split(','))) != len(names.split(',')):
30 raise ValueError('columnNames must contain unique names')
31 names = [n.strip() for n in names.split(',')]
32 if schema is not None:
33 fields = [f for f in schema if f.name not in names]
34 if not fields:
35 raise ValueError('at least one output column is required')
36 schema = pa.schema(fields, metadata=schema.metadata)
37 if schema is not None:
38 outputs = {p: descriptor(['records'], ['arrow'], schema, certainty='known') for p in outputs}
39 return result(outputs)
40
41
42def preview_outputs(inputs, params, limits):
43 from xstream_operator import preview_process
44 return preview_process(process, inputs, params, limits)
45
46
47if __name__ == '__main__':
48 from xstream_operator import run_operator
49 run_operator(process, infer_outputs, preview_outputs=preview_outputs)