src/analysis.py
1"""Bounded, sequential tile processing over Arrow byte streams."""
2import csv
3import hashlib
4import io
5import json
6import math
7from pathlib import Path
8import tarfile
9import zipfile
10import numpy as np
11from rasterio.io import MemoryFile
12from pyproj import Transformer,CRS
13from shapely.geometry import Point,shape,mapping
14from shapely.ops import transform as transform_geometry
15from common import region,project_region,metric_crs,render
16
17MAX_MEMBER=48*1024*1024
18MAX_CELLS=3_000_000
19
20
21class ChunkReader(io.RawIOBase):
22 def __init__(self,chunks):self.chunks=iter(chunks);self.pending=memoryview(b'');self.closed_input=False
23 def readable(self):return True
24 def readinto(self,target):
25 while not self.pending:
26 try:self.pending=memoryview(next(self.chunks))
27 except StopIteration:return 0
28 count=min(len(target),len(self.pending));target[:count]=self.pending[:count];self.pending=self.pending[count:];return count
29
30
31def bundle(chunks):
32 with io.BufferedReader(ChunkReader(chunks),buffer_size=65536) as stream:
33 with tarfile.open(fileobj=stream,mode='r|') as archive:
34 for member in archive:
35 if not member.isfile() or member.size>MAX_MEMBER or '..' in Path(member.name).parts:raise ValueError('invalid bundle member')
36 yield member.name,archive.extractfile(member).read()
37 # Consume the Arrow end frame even when tar reaches its end marker first.
38 while stream.read(65536):pass
39
40
41def read_json(ctx,name):
42 data=bytearray()
43 for block in ctx.input(name).bytes():
44 data.extend(block)
45 if len(data)>8*1024*1024:raise ValueError('intermediate JSON exceeds limit')
46 return json.loads(data)
47
48
49def gully_slop(dem,feature):
50 line=shape(feature['geometry']);first,last=list(line.coords)[0],list(line.coords)[-1]
51 values=[float(v[0]) for v in dem.sample([first,last],masked=True)]
52 if not all(math.isfinite(v) and v!=dem.nodata for v in values) or line.length<=0:return None
53 return (values[0]-values[1])/line.length
54
55
56def geo_entropy(elevations,labels,basin_id,bins=16):
57 values=elevations[(labels==basin_id)&np.isfinite(elevations)&(elevations!=-9999)]
58 if not len(values):return None
59 if np.ptp(values)==0:return 0.0
60 counts=np.histogram(values,bins=bins)[0];p=counts[counts>0]/len(values)
61 return float(-np.sum(p*np.log(p))/np.log(bins))
62
63
64def terrain(chunks,params):
65 region(params);metadata=None;dem_data=None;features=[];tiles=0
66 for name,data in bundle(chunks):
67 if name.endswith('/metadata.json'):metadata=json.loads(data);dem_data=None
68 elif name.endswith('/dem.tif'):dem_data=data
69 elif name.endswith('/catchments.tif'):
70 if not metadata or dem_data is None:raise ValueError('invalid terrain member order')
71 with MemoryFile(dem_data) as dm,MemoryFile(data) as cm,dm.open() as dem,cm.open() as catch:
72 if dem.crs!=catch.crs or dem.transform!=catch.transform or dem.shape!=catch.shape or dem.width*dem.height>MAX_CELLS:raise ValueError('inconsistent terrain grids')
73 metric_crs(dem.crs);z=dem.read(1);labels=catch.read(1);entropies={b:geo_entropy(z,labels,b,params.get('entropy_bins',16)) for b in [1,2]}
74 for f in metadata['features']:
75 props=dict(f['properties']);slope=gully_slop(dem,f);entropy=entropies[props['basin_id']]
76 props.update(slope_ratio=slope,slope_percent=None if slope is None else slope*100,entropy=entropy)
77 features.append(dict(f,properties=props))
78 tiles+=1
79 dem_data=None
80 return {'synthetic':True,'crs':'EPSG:32649','features':features,'tiles_processed':tiles,'region_bbox':params['region_bbox'],'region':params['region'],
81 'entropy_definition':'H = -sum(p_i ln(p_i))/ln(16), equally spaced elevation bins over each complete synthetic tile basin'}
82
83
84def susceptibility(metrics,params):
85 by_id={f['properties']['HydroID']:f for f in metrics['features']};selected={};rejected=[]
86 for identifier,f in by_id.items():
87 p=f['properties'];s=p['slope_ratio'];h=p['entropy']
88 if s is None or h is None or not params.get('min_slope_ratio',.08)<=s<=params.get('max_slope_ratio',.50) or h>=params.get('max_entropy',.2):
89 rejected.append({'HydroID':identifier,'reason':'seed_threshold','slope_ratio':s,'entropy':h});continue
90 current=identifier;seen=set()
91 for hop in range(params.get('max_downstream_hops',32)+1):
92 if current is None or current not in by_id:break
93 if current in seen:raise ValueError('cycle in channel topology')
94 seen.add(current);selected.setdefault(current,dict(seed_HydroID=identifier,downstream_hops=hop))
95 current=by_id[current]['properties']['NextDownID']
96 # Preserve complete topology for analysis; clip geometries only for rendering.
97 selection=project_region(params,metrics['crs']);visible=[]
98 for identifier,p in selected.items():
99 f=by_id[identifier];geom=shape(f['geometry']).intersection(selection)
100 if not geom.is_empty:visible.append(dict(f,geometry=mapping(geom),properties=dict(f['properties'],**p)))
101 return {'synthetic':True,'crs':metrics['crs'],'region_bbox':params['region_bbox'],'region':params['region'],'HydroIDs':sorted(selected),
102 'features':visible,'channel_rejections':rejected,'metrics':metrics['features'],'tiles_processed':metrics['tiles_processed']}
103
104
105def model_path():
106 baked=Path('/opt/models/check-dam.pt')
107 if baked.is_file():return baked
108 source=Path(__file__)
109 if source.exists():return source.with_name('model.pt')
110 # Python entrypoint runs directly from a ZIP; the model is an immutable
111 # package asset, extracted into tmpfs rather than an intermediate dataset.
112 archive=next((p for p in source.parents if p.suffix=='.zip'),None)
113 if archive is None:raise ValueError('model package not found')
114 target=Path('/tmp/check-dam-model.pt')
115 with zipfile.ZipFile(archive) as package:target.write_bytes(package.read('model.pt'))
116 return target
117
118
119def detect(chunks,params):
120 import torch
121 torch.set_num_threads(2)
122 import contextlib,sys
123 with contextlib.redirect_stdout(sys.stderr):
124 from ultralytics import YOLO
125 model=YOLO(str(model_path()));selection=region(params);metadata=None;image_data=None;output=[];windows=0;tiles=0
126 for name,data in bundle(chunks):
127 if name.endswith('/metadata.json'):metadata=json.loads(data);image_data=None
128 elif name.endswith('/imagery.tif'):image_data=data
129 elif name.endswith('/coordinates.csv'):
130 if not metadata or image_data is None:raise ValueError('invalid imagery member order')
131 with MemoryFile(image_data) as mem,mem.open() as image:
132 if image.width*image.height>MAX_CELLS or image.count!=3:raise ValueError('invalid imagery')
133 for row in csv.DictReader(io.StringIO(data.decode())):
134 x,y=image.xy(int(row['row']),int(row['col']))
135 if CRS.from_user_input(row['crs'])!=image.crs or math.hypot(x-float(row['x']),y-float(row['y']))>.01:raise ValueError('coordinate control mismatch')
136 if not selection.intersects(__import__('shapely').geometry.box(*metadata['bbox_wgs84'])):continue
137 tile_selection=project_region(params,image.crs);to_geo=Transformer.from_crs(image.crs,'EPSG:4326',always_xy=True)
138 items=[];offsets=[]
139 def run_batch():
140 nonlocal windows
141 if not items:return
142 predictions=model.predict(items,imgsz=128,conf=params.get('min_confidence',.35),iou=.5,device='cpu',verbose=False)
143 windows+=len(items)
144 for prediction,(r0,c0) in zip(predictions,offsets):
145 for coords,conf in zip(prediction.boxes.xyxy.tolist(),prediction.boxes.conf.tolist()):
146 x0,y0,x1,y1=coords;col=c0+(x0+x1)/2;row=r0+(y0+y1)/2
147 x,y=image.transform*(col+.5,row+.5);lon,lat=to_geo.transform(x,y)
148 if not selection.covers(Point(lon,lat)):continue
149 output.append({'point_id':f"{metadata['tile']}-M{len(output)+1:04}",'tile':metadata['tile'],'x':x,'y':y,
150 'longitude':lon,'latitude':lat,'confidence':round(conf,5),'pixel_box':[c0+x0,r0+y0,c0+x1,r0+y1]})
151 items.clear();offsets.clear()
152 from rasterio.windows import Window,bounds
153 for row in range(0,image.height,96):
154 for col in range(0,image.width,96):
155 window=Window(col,row,min(128,image.width-col),min(128,image.height-row))
156 if not tile_selection.intersects(__import__('shapely').geometry.box(*bounds(window,image.transform))):continue
157 rgb=np.moveaxis(np.clip(image.read(window=window)*255,0,255).astype('uint8'),0,2)
158 padded=np.full((128,128,3),150,dtype='uint8');padded[:rgb.shape[0],:rgb.shape[1]]=rgb
159 # Ultralytics expects BGR arrays.
160 items.append(padded[:,:,::-1].copy());offsets.append((row,col))
161 if len(items)==16:run_batch()
162 run_batch();tiles+=1
163 image_data=None
164 # Overlapping windows can detect the same object more than once.
165 unique=[];duplicates=[]
166 for p in sorted(output,key=lambda p:-p['confidence']):
167 if any(math.hypot(p['x']-q['x'],p['y']-q['y'])<params.get('duplicate_distance_m',60) for q in unique):
168 duplicates.append({'point_id':p['point_id'],'reason':'duplicate_detection'});continue
169 unique.append(p)
170 return {'synthetic':True,'crs':'EPSG:32649','region_bbox':params['region_bbox'],'region':params['region'],'points':unique,'rejections':duplicates,
171 'windows_inferred':windows,'tiles_inferred':tiles,'model':'YOLO11n-seg trained only on synthetic motifs'}
172
173
174def overlap(hazards,detected,chunks,params):
175 if hazards['region']!=detected['region'] or hazards['region']!=params['region']:raise ValueError('selection mismatch')
176 selection=region(params);selected=set(hazards['HydroIDs']);accepted=[];rejected=list(detected['rejections']);processed=set();metadata=None
177 by_tile={}
178 for p in detected['points']:
179 if not selection.covers(Point(p['longitude'],p['latitude'])):
180 rejected.append({'point_id':p['point_id'],'reason':'outside_region'});processed.add(p['point_id']);continue
181 by_tile.setdefault(p['tile'],[]).append(p)
182 bias=params.get('bias',3)
183 if not isinstance(bias,int) or not 0<=bias<=16:raise ValueError('invalid pixel window bias')
184 for name,data in bundle(chunks):
185 if name.endswith('/metadata.json'):metadata=json.loads(data)
186 elif name.endswith('/hydroid.tif'):
187 for p in by_tile.get(metadata['tile'],[]):
188 with MemoryFile(data) as mem,mem.open() as grid:
189 row,col=grid.index(p['x'],p['y']);choices=[]
190 from rasterio.windows import Window
191 r0,c0=max(0,row-bias),max(0,col-bias);r1,c1=min(grid.height,row+bias+1),min(grid.width,col+bias+1)
192 if r1>r0 and c1>c0:
193 values=grid.read(1,window=Window(c0,r0,c1-c0,r1-r0))
194 for rr,cc in zip(*np.nonzero(values)):
195 x,y=grid.xy(r0+rr,c0+cc);choices.append((math.hypot(p['x']-x,p['y']-y),int(values[rr,cc])))
196 processed.add(p['point_id']);choices.sort(key=lambda v:(v[0],v[1]));ids=sorted(set(v[1] for v in choices))
197 if not choices or choices[0][1] not in selected:
198 rejected.append(dict(point_id=p['point_id'],reason='no_channel_in_window' if not choices else 'non_susceptible_channel',matched_HydroIDs=ids));continue
199 props=dict(p,HydroID=choices[0][1],distance_to_channel_m=round(choices[0][0],2),ambiguous=len(ids)>1,matched_HydroIDs=ids)
200 accepted.append({'type':'Feature','geometry':{'type':'Point','coordinates':[p['x'],p['y']]},'properties':props})
201 for p in detected['points']:
202 if p['point_id'] not in processed:rejected.append({'point_id':p['point_id'],'reason':'missing_channel_tile'})
203 geojson={'type':'FeatureCollection','features':[dict(f,geometry={'type':'Point','coordinates':[f['properties']['longitude'],f['properties']['latitude']]}) for f in accepted]}
204 return {'synthetic':True,'crs':hazards['crs'],'region_bbox':params['region_bbox'],'region':params['region'],'features':accepted,'geojson':geojson,
205 'susceptible_channels':hazards['features'],'candidate_count':len(detected['points']),'rejections':rejected,
206 'channel_rejections':hazards['channel_rejections'],'parameters':params,'model':detected['model'],
207 'HydroIDs':hazards['HydroIDs'],'windows_inferred':detected['windows_inferred'],
208 'method':'Synthetic YOLO detection, normalized elevation entropy and topology, nearest HydroID in 7x7 window.'}
209
210
211def process(ctx):
212 params=dict(ctx.params);region(params);operation=params['operation']
213 if operation=='terrain':value=terrain(ctx.input('source').bytes(),params)
214 elif operation=='susceptibility':value=susceptibility(read_json(ctx,'source'),params)
215 elif operation=='detect':value=detect(ctx.input('source').bytes(),params)
216 elif operation=='overlap':value=overlap(read_json(ctx,'a_hazards'),read_json(ctx,'b_candidates'),ctx.input('c_grids').bytes(),params)
217 elif operation=='map':
218 out=ctx.bytes_output('result','svg');out.write(render(read_json(ctx,'source')));return
219 else:raise ValueError('unknown operation')
220 out=ctx.bytes_output('result','json');out.write(json.dumps(value,allow_nan=False).encode())
221
222
223def infer_outputs(inputs,params):
224 from xstream_operator import descriptor,result
225 return result({'result':descriptor(['bytes'],['svg' if params['operation']=='map' else 'json'],None,certainty='known')})
226
227
228def preview_outputs(inputs,params,limits):
229 from xstream_operator import preview_process
230 return preview_process(process,inputs,params,limits)
231
232if __name__=='__main__':
233 from xstream_operator import run_operator
234 run_operator(process,infer_outputs,preview_outputs=preview_outputs)