Source code

Download source archive ↓

src/piflow_tabular.py

1"""Arrow implementations of PiFlow's data sorting and numeric-field operations."""
2
3import math
4import re
5
6import pyarrow as pa
7import pyarrow.compute as pc
8
9SENSOR_SCHEMA = pa.schema([("id", pa.string()), ("timestamp", pa.int64()), ("temperature", pa.float64())])
10
11
12def sorted_table(table, params, names):
13    keys = names(params["sortField"])
14    if any(table.schema.get_field_index(key) < 0 for key in keys):
15        raise ValueError("each sort field must identify one input column")
16    order = params["sortOrder"]
17    if order not in {"asc", "desc"}:
18        raise ValueError("sortOrder must be asc or desc")
19    direction = "ascending" if order == "asc" else "descending"
20    nulls = "at_start" if order == "asc" else "at_end"
21    return table.take(pc.sort_indices(table, sort_keys=[(key, direction) for key in keys], null_placement=nulls))
22
23
24def numeric_schema(schema, params, names):
25    size = re.fullmatch(r"(?i)decimal\(\s*([0-9]{1,2})\s*,\s*([0-9]{1,2})\s*\)", params["decimalTypePrecision"])
26    if not size:
27        raise ValueError("decimalTypePrecision must have the form DECIMAL(precision, scale)")
28    precision, scale = map(int, size.groups())
29    if not 1 <= precision <= 38 or not 0 <= scale <= precision:
30        raise ValueError("decimal precision must be 1..38 and scale from zero to precision")
31    changes = {}
32    for name, dtype in (("doubleTypeField", pa.float64()), ("intTypeField", pa.int32()),
33                        ("decimalTypeField", pa.decimal128(precision, scale))):
34        selection = params[name]
35        for column in names(selection) if selection else []:
36            index = schema.get_field_index(column)
37            if index < 0 or index in changes:
38                raise ValueError("cast columns must be unique and present in the input")
39            changes[index] = dtype
40    return pa.schema([field.with_type(changes[i]) if i in changes else field
41                      for i, field in enumerate(schema)], metadata=schema.metadata)
42
43
44def string_schema(schema):
45    return pa.schema([field.with_type(pa.string()) for field in schema], metadata=schema.metadata)
46
47
48def average_temperature(rows):
49    totals = {}
50    for row in rows:
51        sensor, millis, fahrenheit = (row[field] for field in ("id", "timestamp", "temperature"))
52        if not isinstance(sensor, str) or type(millis) is not int or fahrenheit is None or not math.isfinite(fahrenheit):
53            raise ValueError("each reading requires a sensor ID, integer epoch milliseconds and finite Fahrenheit")
54        end = (millis // 1000 + 1) * 1000
55        total, count = totals.get((end, sensor), (0.0, 0))
56        totals[end, sensor] = (total + (fahrenheit - 32.0) * (5.0 / 9.0), count + 1)
57    rows = [dict(id=sensor, timestamp=end, temperature=total / count)
58            for (end, sensor), (total, count) in sorted(totals.items())]
59    return pa.Table.from_pylist(rows, schema=SENSOR_SCHEMA)
60
61
62def enrich_records(ctx, lookup, schema, names):
63    key = ctx.params["keyFile"]
64    fields = names(ctx.params["replaceField"])
65    source = ctx.input("source")
66    original = source.schema
67    if original.get_field_index(key) < 0 or lookup.schema.get_field_index(key) < 0:
68        raise ValueError("both streams must provide the key column")
69    if original.field(key).type != lookup.schema.field(key).type:
70        raise ValueError("lookup and input keys must have the same type")
71    if any(original.get_field_index(field) < 0 or lookup.schema.get_field_index(field) < 0 for field in fields):
72        raise ValueError("each replacement field must occur in both streams")
73    if key in fields or any(original.field(field).type != lookup.schema.field(field).type for field in fields):
74        raise ValueError("replacement fields must have matching types and cannot include the key")
75    by_key = {}
76    for row in lookup.to_pylist():
77        value = row[key]
78        if value is None:
79            continue
80        if value in by_key:
81            raise ValueError("duplicate lookup key")
82        by_key[value] = {field: row[field] for field in fields}
83    result = ctx.records_output("result", schema)
84    for batch in source.records():
85        rows = batch.to_pylist()
86        for row in rows:
87            row.update(by_key.get(row[key], {}))
88        result.write(pa.RecordBatch.from_pylist(rows, schema=schema))