src/piflow_tabular.py
1"""Arrow implementations of PiFlow's data sorting and numeric-field operations."""
2
3import math
4import re
5
6import pyarrow as pa
7import pyarrow.compute as pc
8
9SENSOR_SCHEMA = pa.schema([("id", pa.string()), ("timestamp", pa.int64()), ("temperature", pa.float64())])
10
11
12def sorted_table(table, params, names):
13 keys = names(params["sortField"])
14 if any(table.schema.get_field_index(key) < 0 for key in keys):
15 raise ValueError("each sort field must identify one input column")
16 order = params["sortOrder"]
17 if order not in {"asc", "desc"}:
18 raise ValueError("sortOrder must be asc or desc")
19 direction = "ascending" if order == "asc" else "descending"
20 nulls = "at_start" if order == "asc" else "at_end"
21 return table.take(pc.sort_indices(table, sort_keys=[(key, direction) for key in keys], null_placement=nulls))
22
23
24def numeric_schema(schema, params, names):
25 size = re.fullmatch(r"(?i)decimal\(\s*([0-9]{1,2})\s*,\s*([0-9]{1,2})\s*\)", params["decimalTypePrecision"])
26 if not size:
27 raise ValueError("decimalTypePrecision must have the form DECIMAL(precision, scale)")
28 precision, scale = map(int, size.groups())
29 if not 1 <= precision <= 38 or not 0 <= scale <= precision:
30 raise ValueError("decimal precision must be 1..38 and scale from zero to precision")
31 changes = {}
32 for name, dtype in (("doubleTypeField", pa.float64()), ("intTypeField", pa.int32()),
33 ("decimalTypeField", pa.decimal128(precision, scale))):
34 selection = params[name]
35 for column in names(selection) if selection else []:
36 index = schema.get_field_index(column)
37 if index < 0 or index in changes:
38 raise ValueError("cast columns must be unique and present in the input")
39 changes[index] = dtype
40 return pa.schema([field.with_type(changes[i]) if i in changes else field
41 for i, field in enumerate(schema)], metadata=schema.metadata)
42
43
44def string_schema(schema):
45 return pa.schema([field.with_type(pa.string()) for field in schema], metadata=schema.metadata)
46
47
48def convert_schema_type(schema, params, names):
49 if params["all"]:
50 return string_schema(schema)
51 changes = {}
52 for setting, dtype in (("stringType", pa.string()), ("integerType", pa.int32())):
53 selection = params[setting]
54 for name in names(selection) if selection else []:
55 index = schema.get_field_index(name)
56 if index < 0 or index in changes:
57 raise ValueError("cast columns must be unique and present in the input")
58 changes[index] = dtype
59 return pa.schema([field.with_type(changes[i]) if i in changes else field
60 for i, field in enumerate(schema)], metadata=schema.metadata)
61
62
63def average_temperature(rows):
64 totals = {}
65 for row in rows:
66 sensor, millis, fahrenheit = (row[field] for field in ("id", "timestamp", "temperature"))
67 if not isinstance(sensor, str) or type(millis) is not int or fahrenheit is None or not math.isfinite(fahrenheit):
68 raise ValueError("each reading requires a sensor ID, integer epoch milliseconds and finite Fahrenheit")
69 end = (millis // 1000 + 1) * 1000
70 total, count = totals.get((end, sensor), (0.0, 0))
71 totals[end, sensor] = (total + (fahrenheit - 32.0) * (5.0 / 9.0), count + 1)
72 rows = [dict(id=sensor, timestamp=end, temperature=total / count)
73 for (end, sensor), (total, count) in sorted(totals.items())]
74 return pa.Table.from_pylist(rows, schema=SENSOR_SCHEMA)
75
76
77def enrich_records(ctx, lookup, schema, names):
78 key = ctx.params["keyFile"]
79 fields = names(ctx.params["replaceField"])
80 source = ctx.input("source")
81 original = source.schema
82 if original.get_field_index(key) < 0 or lookup.schema.get_field_index(key) < 0:
83 raise ValueError("both streams must provide the key column")
84 if original.field(key).type != lookup.schema.field(key).type:
85 raise ValueError("lookup and input keys must have the same type")
86 if any(original.get_field_index(field) < 0 or lookup.schema.get_field_index(field) < 0 for field in fields):
87 raise ValueError("each replacement field must occur in both streams")
88 if key in fields or any(original.field(field).type != lookup.schema.field(field).type for field in fields):
89 raise ValueError("replacement fields must have matching types and cannot include the key")
90 by_key = {}
91 for row in lookup.to_pylist():
92 value = row[key]
93 if value is None:
94 continue
95 if value in by_key:
96 raise ValueError("duplicate lookup key")
97 by_key[value] = {field: row[field] for field in fields}
98 result = ctx.records_output("result", schema)
99 for batch in source.records():
100 rows = batch.to_pylist()
101 for row in rows:
102 row.update(by_key.get(row[key], {}))
103 result.write(pa.RecordBatch.from_pylist(rows, schema=schema))