src/xstream_piflow_reichstein_check.py
1"""Bounded Reichstein screening adapted from EcoFlow's six-group/twenty-bin algorithm."""
2import math
3import statistics
4import pyarrow as pa
5from dacp_piflow_support import params, materialize, write_table, indexes
6
7
8def process(ctx):
9 p=params(ctx,['soilTemperatureField','frictionWindSpeedField','FC','flagValue','percentFlux'])
10 soil,wind,flux=p['soilTemperatureField'],p['frictionWindSpeedField'],p['FC']
11 flag,percent=p['flagValue'],p['percentFlux']
12 if any(not isinstance(n,str) or not n for n in (soil,wind,flux)) or len({soil,wind,flux})!=3:
13 raise ValueError('three distinct column names required')
14 if type(flag) is not int or not -2**31<=flag<2**31 or type(percent) not in (int,float) or not 0<percent<=1:
15 raise ValueError('flagValue must be int32; percentFlux must be numeric in (0,1]')
16 table=materialize(ctx.input('source'))
17 indexes(table.schema,[soil,wind,flux,'day_status'])
18 for name in (soil,wind,flux):
19 typ=table.schema.field(name).type
20 if not (pa.types.is_integer(typ) or pa.types.is_floating(typ)):
21 raise ValueError('temperature, wind and flux columns must be numeric')
22 rows=table.to_pylist()
23 valid=[r for r in rows if all(r[n] is not None and math.isfinite(r[n]) and r[n]!=flag for n in (soil,wind,flux))]
24 if len(valid)<120:
25 raise ValueError('at least120 valid observations required for six temperature groups')
26 valid.sort(key=lambda r:r[soil])
27 size=len(valid)//6
28 thresholds=[]
29 for i in range(6):
30 group=valid[i*size:(i+1)*size if i<5 else len(valid)]
31 xs=[float(r[soil]) for r in group];ys=[float(r[wind]) for r in group]
32 mx,my=statistics.mean(xs),statistics.mean(ys)
33 xx=sum((x-mx)**2 for x in xs);yy=sum((y-my)**2 for y in ys)
34 if xx==0 or yy==0:
35 continue
36 correlation=sum((x-mx)*(y-my) for x,y in zip(xs,ys))/math.sqrt(xx*yy)
37 if correlation>=.4:
38 continue
39 low,high=min(ys),max(ys)
40 width=(high-low)/20
41 ordered=sorted(group,key=lambda r:r[wind])
42 for b in range(1,21):
43 remaining=[r[flux] for r in group if r[wind]>=low+b*width]
44 if not remaining:
45 continue
46 average=statistics.mean(remaining)
47 candidates=[r for r in ordered if low+(b-1)*width<=r[wind]<(low+b*width) and r[flux]>average*percent]
48 if candidates:
49 thresholds.append(float(candidates[0][wind]))
50 break
51 if not thresholds:
52 raise ValueError('no valid u-star critical threshold found')
53 threshold=statistics.median(thresholds)
54 values=[flag if r['day_status']=='night' and r[wind] is not None and r[wind]<threshold else r[flux] for r in rows]
55 index=table.schema.get_field_index(flux)
56 write_table(ctx,table.set_column(index,table.schema.field(flux),pa.array(values,type=table.schema.field(flux).type)))
57
58
59def infer_outputs(inputs, params):
60 import pyarrow as pa
61 from xstream_operator import descriptor, result, records_schema
62 expected = {'source'}
63 if set(inputs) != expected:
64 raise ValueError('unexpected input ports')
65 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
66 return result(outputs)
67
68
69def preview_outputs(inputs, params, limits):
70 from xstream_operator import preview_process
71 return preview_process(process, inputs, params, limits)
72
73
74if __name__ == '__main__':
75 from xstream_operator import run_operator
76 run_operator(process, infer_outputs, preview_outputs=preview_outputs)