Source code

Download source archive ↓

src/xstream_piflow_reichstein_check.py

1"""Bounded Reichstein screening adapted from EcoFlow's six-group/twenty-bin algorithm."""
2import math
3import statistics
4import pyarrow as pa
5from dacp_piflow_support import params, materialize, write_table, indexes
6
7
8def process(ctx):
9    p=params(ctx,['soilTemperatureField','frictionWindSpeedField','FC','flagValue','percentFlux'])
10    soil,wind,flux=p['soilTemperatureField'],p['frictionWindSpeedField'],p['FC']
11    flag,percent=p['flagValue'],p['percentFlux']
12    if any(not isinstance(n,str) or not n for n in (soil,wind,flux)) or len({soil,wind,flux})!=3:
13        raise ValueError('three distinct column names required')
14    if type(flag) is not int or not -2**31<=flag<2**31 or type(percent) not in (int,float) or not 0<percent<=1:
15        raise ValueError('flagValue must be int32; percentFlux must be numeric in (0,1]')
16    table=materialize(ctx.input('source'))
17    indexes(table.schema,[soil,wind,flux,'day_status'])
18    for name in (soil,wind,flux):
19        typ=table.schema.field(name).type
20        if not (pa.types.is_integer(typ) or pa.types.is_floating(typ)):
21            raise ValueError('temperature, wind and flux columns must be numeric')
22    rows=table.to_pylist()
23    valid=[r for r in rows if all(r[n] is not None and math.isfinite(r[n]) and r[n]!=flag for n in (soil,wind,flux))]
24    if len(valid)<120:
25        raise ValueError('at least120 valid observations required for six temperature groups')
26    valid.sort(key=lambda r:r[soil])
27    size=len(valid)//6
28    thresholds=[]
29    for i in range(6):
30        group=valid[i*size:(i+1)*size if i<5 else len(valid)]
31        xs=[float(r[soil]) for r in group];ys=[float(r[wind]) for r in group]
32        mx,my=statistics.mean(xs),statistics.mean(ys)
33        xx=sum((x-mx)**2 for x in xs);yy=sum((y-my)**2 for y in ys)
34        if xx==0 or yy==0:
35            continue
36        correlation=sum((x-mx)*(y-my) for x,y in zip(xs,ys))/math.sqrt(xx*yy)
37        if correlation>=.4:
38            continue
39        low,high=min(ys),max(ys)
40        width=(high-low)/20
41        ordered=sorted(group,key=lambda r:r[wind])
42        for b in range(1,21):
43            remaining=[r[flux] for r in group if r[wind]>=low+b*width]
44            if not remaining:
45                continue
46            average=statistics.mean(remaining)
47            candidates=[r for r in ordered if low+(b-1)*width<=r[wind]<(low+b*width) and r[flux]>average*percent]
48            if candidates:
49                thresholds.append(float(candidates[0][wind]))
50                break
51    if not thresholds:
52        raise ValueError('no valid u-star critical threshold found')
53    threshold=statistics.median(thresholds)
54    values=[flag if r['day_status']=='night' and r[wind] is not None and r[wind]<threshold else r[flux] for r in rows]
55    index=table.schema.get_field_index(flux)
56    write_table(ctx,table.set_column(index,table.schema.field(flux),pa.array(values,type=table.schema.field(flux).type)))
57
58
59def infer_outputs(inputs, params):
60    import pyarrow as pa
61    from xstream_operator import descriptor, result, records_schema
62    expected = {'source'}
63    if set(inputs) != expected:
64        raise ValueError('unexpected input ports')
65    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
66    return result(outputs)
67
68
69def preview_outputs(inputs, params, limits):
70    from xstream_operator import preview_process
71    return preview_process(process, inputs, params, limits)
72
73
74if __name__ == '__main__':
75    from xstream_operator import run_operator
76    run_operator(process, infer_outputs, preview_outputs=preview_outputs)