Source code

Download source archive ↓

src/xstream_piflow_merge.py

1"""Merge two record streams by field name, filling absent fields with nulls."""
2from dacp_piflow_support import params
3
4
5def merged_schema(left, right):
6    import pyarrow as pa
7    for schema in (left, right):
8        if len(schema.names) != len(set(schema.names)):
9            raise ValueError('merge input schema contains duplicate field names')
10    fields = []
11    for name in dict.fromkeys(left.names + right.names):
12        a = left.field(name) if name in left.names else None
13        b = right.field(name) if name in right.names else None
14        if a is not None and b is not None:
15            if a.type != b.type and not (pa.types.is_null(a.type) or pa.types.is_null(b.type)):
16                raise ValueError('merge has incompatible types for field: ' + name)
17            base = b if pa.types.is_null(a.type) else a
18            fields.append(pa.field(name, base.type, nullable=a.nullable or b.nullable
19                                   or pa.types.is_null(a.type) or pa.types.is_null(b.type),
20                                   metadata=base.metadata))
21        else:
22            base = a if a is not None else b
23            fields.append(pa.field(name, base.type, nullable=True, metadata=base.metadata))
24    return pa.schema(fields, metadata=left.metadata)
25
26
27def process(ctx):
28    import pyarrow as pa
29    params(ctx)
30    left, right = ctx.input('left'), ctx.input('right')
31    schema = merged_schema(left.schema, right.schema)
32    out = ctx.records_output('result', schema)
33    for stream in (left, right):
34        for batch in stream.records():
35            arrays = [batch.column(batch.schema.get_field_index(field.name)).cast(field.type)
36                      if field.name in batch.schema.names else pa.nulls(batch.num_rows, field.type)
37                      for field in schema]
38            out.write(pa.RecordBatch.from_arrays(arrays, schema=schema))
39
40
41def infer_outputs(inputs, params):
42    from xstream_operator import descriptor, result, records_schema
43    if set(inputs) != {'left', 'right'}:
44        raise ValueError('unexpected input ports')
45    left, right = records_schema(inputs['left']), records_schema(inputs['right'])
46    schema = merged_schema(left, right) if left is not None and right is not None else None
47    return result({'result': descriptor(['records'], ['arrow'], schema,
48                                       'known' if schema is not None else 'unknown')})
49
50
51def preview_outputs(inputs, params, limits):
52    from xstream_operator import preview_process
53    return preview_process(process, inputs, params, limits)
54
55
56if __name__ == '__main__':
57    from xstream_operator import run_operator
58    run_operator(process, infer_outputs, preview_outputs=preview_outputs)