src/piflow_tabular.py
1"""Arrow implementations of PiFlow's data sorting and numeric-field operations."""
2
3import math
4import re
5
6import pyarrow as pa
7import pyarrow.compute as pc
8
9SENSOR_SCHEMA = pa.schema([("id", pa.string()), ("timestamp", pa.int64()), ("temperature", pa.float64())])
10
11
12def sorted_table(table, params, names):
13 keys = names(params["sortField"])
14 if any(table.schema.get_field_index(key) < 0 for key in keys):
15 raise ValueError("each sort field must identify one input column")
16 order = params["sortOrder"]
17 if order not in {"asc", "desc"}:
18 raise ValueError("sortOrder must be asc or desc")
19 direction = "ascending" if order == "asc" else "descending"
20 nulls = "at_start" if order == "asc" else "at_end"
21 return table.take(pc.sort_indices(table, sort_keys=[(key, direction) for key in keys], null_placement=nulls))
22
23
24def numeric_schema(schema, params, names):
25 size = re.fullmatch(r"(?i)decimal\(\s*([0-9]{1,2})\s*,\s*([0-9]{1,2})\s*\)", params["decimalTypePrecision"])
26 if not size:
27 raise ValueError("decimalTypePrecision must have the form DECIMAL(precision, scale)")
28 precision, scale = map(int, size.groups())
29 if not 1 <= precision <= 38 or not 0 <= scale <= precision:
30 raise ValueError("decimal precision must be 1..38 and scale from zero to precision")
31 changes = {}
32 for name, dtype in (("doubleTypeField", pa.float64()), ("intTypeField", pa.int32()),
33 ("decimalTypeField", pa.decimal128(precision, scale))):
34 selection = params[name]
35 for column in names(selection) if selection else []:
36 index = schema.get_field_index(column)
37 if index < 0 or index in changes:
38 raise ValueError("cast columns must be unique and present in the input")
39 changes[index] = dtype
40 return pa.schema([field.with_type(changes[i]) if i in changes else field
41 for i, field in enumerate(schema)], metadata=schema.metadata)
42
43
44def string_schema(schema):
45 return pa.schema([field.with_type(pa.string()) for field in schema], metadata=schema.metadata)
46
47
48def average_temperature(rows):
49 totals = {}
50 for row in rows:
51 sensor, millis, fahrenheit = (row[field] for field in ("id", "timestamp", "temperature"))
52 if not isinstance(sensor, str) or type(millis) is not int or fahrenheit is None or not math.isfinite(fahrenheit):
53 raise ValueError("each reading requires a sensor ID, integer epoch milliseconds and finite Fahrenheit")
54 end = (millis // 1000 + 1) * 1000
55 total, count = totals.get((end, sensor), (0.0, 0))
56 totals[end, sensor] = (total + (fahrenheit - 32.0) * (5.0 / 9.0), count + 1)
57 rows = [dict(id=sensor, timestamp=end, temperature=total / count)
58 for (end, sensor), (total, count) in sorted(totals.items())]
59 return pa.Table.from_pylist(rows, schema=SENSOR_SCHEMA)
60
61
62def enrich_records(ctx, lookup, schema, names):
63 key = ctx.params["keyFile"]
64 fields = names(ctx.params["replaceField"])
65 source = ctx.input("source")
66 original = source.schema
67 if original.get_field_index(key) < 0 or lookup.schema.get_field_index(key) < 0:
68 raise ValueError("both streams must provide the key column")
69 if original.field(key).type != lookup.schema.field(key).type:
70 raise ValueError("lookup and input keys must have the same type")
71 if any(original.get_field_index(field) < 0 or lookup.schema.get_field_index(field) < 0 for field in fields):
72 raise ValueError("each replacement field must occur in both streams")
73 if key in fields or any(original.field(field).type != lookup.schema.field(field).type for field in fields):
74 raise ValueError("replacement fields must have matching types and cannot include the key")
75 by_key = {}
76 for row in lookup.to_pylist():
77 value = row[key]
78 if value is None:
79 continue
80 if value in by_key:
81 raise ValueError("duplicate lookup key")
82 by_key[value] = {field: row[field] for field in fields}
83 result = ctx.records_output("result", schema)
84 for batch in source.records():
85 rows = batch.to_pylist()
86 for row in rows:
87 row.update(by_key.get(row[key], {}))
88 result.write(pa.RecordBatch.from_pylist(rows, schema=schema))