Source code

Download source archive ↓

src/piflow_tabular.py

1"""Arrow implementations of PiFlow's data sorting and numeric-field operations."""
2
3import math
4import re
5
6import pyarrow as pa
7import pyarrow.compute as pc
8
9SENSOR_SCHEMA = pa.schema([("id", pa.string()), ("timestamp", pa.int64()), ("temperature", pa.float64())])
10
11
12def sorted_table(table, params, names):
13    keys = names(params["sortField"])
14    if any(table.schema.get_field_index(key) < 0 for key in keys):
15        raise ValueError("each sort field must identify one input column")
16    order = params["sortOrder"]
17    if order not in {"asc", "desc"}:
18        raise ValueError("sortOrder must be asc or desc")
19    direction = "ascending" if order == "asc" else "descending"
20    nulls = "at_start" if order == "asc" else "at_end"
21    return table.take(pc.sort_indices(table, sort_keys=[(key, direction) for key in keys], null_placement=nulls))
22
23
24def numeric_schema(schema, params, names):
25    size = re.fullmatch(r"(?i)decimal\(\s*([0-9]{1,2})\s*,\s*([0-9]{1,2})\s*\)", params["decimalTypePrecision"])
26    if not size:
27        raise ValueError("decimalTypePrecision must have the form DECIMAL(precision, scale)")
28    precision, scale = map(int, size.groups())
29    if not 1 <= precision <= 38 or not 0 <= scale <= precision:
30        raise ValueError("decimal precision must be 1..38 and scale from zero to precision")
31    changes = {}
32    for name, dtype in (("doubleTypeField", pa.float64()), ("intTypeField", pa.int32()),
33                        ("decimalTypeField", pa.decimal128(precision, scale))):
34        selection = params[name]
35        for column in names(selection) if selection else []:
36            index = schema.get_field_index(column)
37            if index < 0 or index in changes:
38                raise ValueError("cast columns must be unique and present in the input")
39            changes[index] = dtype
40    return pa.schema([field.with_type(changes[i]) if i in changes else field
41                      for i, field in enumerate(schema)], metadata=schema.metadata)
42
43
44def string_schema(schema):
45    return pa.schema([field.with_type(pa.string()) for field in schema], metadata=schema.metadata)
46
47
48def convert_schema_type(schema, params, names):
49    if params["all"]:
50        return string_schema(schema)
51    changes = {}
52    for setting, dtype in (("stringType", pa.string()), ("integerType", pa.int32())):
53        selection = params[setting]
54        for name in names(selection) if selection else []:
55            index = schema.get_field_index(name)
56            if index < 0 or index in changes:
57                raise ValueError("cast columns must be unique and present in the input")
58            changes[index] = dtype
59    return pa.schema([field.with_type(changes[i]) if i in changes else field
60                      for i, field in enumerate(schema)], metadata=schema.metadata)
61
62
63def average_temperature(rows):
64    totals = {}
65    for row in rows:
66        sensor, millis, fahrenheit = (row[field] for field in ("id", "timestamp", "temperature"))
67        if not isinstance(sensor, str) or type(millis) is not int or fahrenheit is None or not math.isfinite(fahrenheit):
68            raise ValueError("each reading requires a sensor ID, integer epoch milliseconds and finite Fahrenheit")
69        end = (millis // 1000 + 1) * 1000
70        total, count = totals.get((end, sensor), (0.0, 0))
71        totals[end, sensor] = (total + (fahrenheit - 32.0) * (5.0 / 9.0), count + 1)
72    rows = [dict(id=sensor, timestamp=end, temperature=total / count)
73            for (end, sensor), (total, count) in sorted(totals.items())]
74    return pa.Table.from_pylist(rows, schema=SENSOR_SCHEMA)
75
76
77def enrich_records(ctx, lookup, schema, names):
78    key = ctx.params["keyFile"]
79    fields = names(ctx.params["replaceField"])
80    source = ctx.input("source")
81    original = source.schema
82    if original.get_field_index(key) < 0 or lookup.schema.get_field_index(key) < 0:
83        raise ValueError("both streams must provide the key column")
84    if original.field(key).type != lookup.schema.field(key).type:
85        raise ValueError("lookup and input keys must have the same type")
86    if any(original.get_field_index(field) < 0 or lookup.schema.get_field_index(field) < 0 for field in fields):
87        raise ValueError("each replacement field must occur in both streams")
88    if key in fields or any(original.field(field).type != lookup.schema.field(field).type for field in fields):
89        raise ValueError("replacement fields must have matching types and cannot include the key")
90    by_key = {}
91    for row in lookup.to_pylist():
92        value = row[key]
93        if value is None:
94            continue
95        if value in by_key:
96            raise ValueError("duplicate lookup key")
97        by_key[value] = {field: row[field] for field in fields}
98    result = ctx.records_output("result", schema)
99    for batch in source.records():
100        rows = batch.to_pylist()
101        for row in rows:
102            row.update(by_key.get(row[key], {}))
103        result.write(pa.RecordBatch.from_pylist(rows, schema=schema))