src/dacp_piflow_merge.py
1"""Merge two record streams by field name, filling absent fields with nulls."""
2from dacp_piflow_support import params
3
4
5def merged_schema(left, right):
6 import pyarrow as pa
7 for schema in (left, right):
8 if len(schema.names) != len(set(schema.names)):
9 raise ValueError('merge input schema contains duplicate field names')
10 fields = []
11 for name in dict.fromkeys(left.names + right.names):
12 a = left.field(name) if name in left.names else None
13 b = right.field(name) if name in right.names else None
14 if a is not None and b is not None:
15 if a.type != b.type and not (pa.types.is_null(a.type) or pa.types.is_null(b.type)):
16 raise ValueError('merge has incompatible types for field: ' + name)
17 base = b if pa.types.is_null(a.type) else a
18 fields.append(pa.field(name, base.type, nullable=a.nullable or b.nullable
19 or pa.types.is_null(a.type) or pa.types.is_null(b.type),
20 metadata=base.metadata))
21 else:
22 base = a if a is not None else b
23 fields.append(pa.field(name, base.type, nullable=True, metadata=base.metadata))
24 return pa.schema(fields, metadata=left.metadata)
25
26
27def process(ctx):
28 import pyarrow as pa
29 params(ctx)
30 left, right = ctx.input('left'), ctx.input('right')
31 schema = merged_schema(left.schema, right.schema)
32 out = ctx.records_output('result', schema)
33 for stream in (left, right):
34 for batch in stream.records():
35 arrays = [batch.column(batch.schema.get_field_index(field.name)).cast(field.type)
36 if field.name in batch.schema.names else pa.nulls(batch.num_rows, field.type)
37 for field in schema]
38 out.write(pa.RecordBatch.from_arrays(arrays, schema=schema))
39
40
41def infer_outputs(inputs, params):
42 from dacp_operator import descriptor, result, records_schema
43 if set(inputs) != {'left', 'right'}:
44 raise ValueError('unexpected input ports')
45 left, right = records_schema(inputs['left']), records_schema(inputs['right'])
46 schema = merged_schema(left, right) if left is not None and right is not None else None
47 return result({'result': descriptor(['records'], ['arrow'], schema,
48 'known' if schema is not None else 'unknown')})
49
50
51def preview_outputs(inputs, params, limits):
52 from dacp_operator import preview_process
53 return preview_process(process, inputs, params, limits)
54
55
56if __name__ == '__main__':
57 from dacp_operator import run_operator
58 run_operator(process, infer_outputs, preview_outputs=preview_outputs)