Source code

Download source archive ↓

src/analysis.py

1"""Bounded, sequential tile processing over Arrow byte streams."""
2import csv
3import hashlib
4import io
5import json
6import math
7from pathlib import Path
8import tarfile
9import zipfile
10import numpy as np
11from rasterio.io import MemoryFile
12from pyproj import Transformer,CRS
13from shapely.geometry import Point,shape,mapping
14from shapely.ops import transform as transform_geometry
15from common import region,project_region,metric_crs,render
16
17MAX_MEMBER=48*1024*1024
18MAX_CELLS=3_000_000
19
20
21class ChunkReader(io.RawIOBase):
22    def __init__(self,chunks):self.chunks=iter(chunks);self.pending=memoryview(b'');self.closed_input=False
23    def readable(self):return True
24    def readinto(self,target):
25        while not self.pending:
26            try:self.pending=memoryview(next(self.chunks))
27            except StopIteration:return 0
28        count=min(len(target),len(self.pending));target[:count]=self.pending[:count];self.pending=self.pending[count:];return count
29
30
31def bundle(chunks):
32    with io.BufferedReader(ChunkReader(chunks),buffer_size=65536) as stream:
33        with tarfile.open(fileobj=stream,mode='r|') as archive:
34            for member in archive:
35                if not member.isfile() or member.size>MAX_MEMBER or '..' in Path(member.name).parts:raise ValueError('invalid bundle member')
36                yield member.name,archive.extractfile(member).read()
37        # Consume the Arrow end frame even when tar reaches its end marker first.
38        while stream.read(65536):pass
39
40
41def read_json(ctx,name):
42    data=bytearray()
43    for block in ctx.input(name).bytes():
44        data.extend(block)
45        if len(data)>8*1024*1024:raise ValueError('intermediate JSON exceeds limit')
46    return json.loads(data)
47
48
49def gully_slop(dem,feature):
50    line=shape(feature['geometry']);first,last=list(line.coords)[0],list(line.coords)[-1]
51    values=[float(v[0]) for v in dem.sample([first,last],masked=True)]
52    if not all(math.isfinite(v) and v!=dem.nodata for v in values) or line.length<=0:return None
53    return (values[0]-values[1])/line.length
54
55
56def geo_entropy(elevations,labels,basin_id,bins=16):
57    values=elevations[(labels==basin_id)&np.isfinite(elevations)&(elevations!=-9999)]
58    if not len(values):return None
59    if np.ptp(values)==0:return 0.0
60    counts=np.histogram(values,bins=bins)[0];p=counts[counts>0]/len(values)
61    return float(-np.sum(p*np.log(p))/np.log(bins))
62
63
64def terrain(chunks,params):
65    region(params);metadata=None;dem_data=None;features=[];tiles=0
66    for name,data in bundle(chunks):
67        if name.endswith('/metadata.json'):metadata=json.loads(data);dem_data=None
68        elif name.endswith('/dem.tif'):dem_data=data
69        elif name.endswith('/catchments.tif'):
70            if not metadata or dem_data is None:raise ValueError('invalid terrain member order')
71            with MemoryFile(dem_data) as dm,MemoryFile(data) as cm,dm.open() as dem,cm.open() as catch:
72                if dem.crs!=catch.crs or dem.transform!=catch.transform or dem.shape!=catch.shape or dem.width*dem.height>MAX_CELLS:raise ValueError('inconsistent terrain grids')
73                metric_crs(dem.crs);z=dem.read(1);labels=catch.read(1);entropies={b:geo_entropy(z,labels,b,params.get('entropy_bins',16)) for b in [1,2]}
74                for f in metadata['features']:
75                    props=dict(f['properties']);slope=gully_slop(dem,f);entropy=entropies[props['basin_id']]
76                    props.update(slope_ratio=slope,slope_percent=None if slope is None else slope*100,entropy=entropy)
77                    features.append(dict(f,properties=props))
78                tiles+=1
79            dem_data=None
80    return {'synthetic':True,'crs':'EPSG:32649','features':features,'tiles_processed':tiles,'region_bbox':params['region_bbox'],'region':params['region'],
81            'entropy_definition':'H = -sum(p_i ln(p_i))/ln(16), equally spaced elevation bins over each complete synthetic tile basin'}
82
83
84def susceptibility(metrics,params):
85    by_id={f['properties']['HydroID']:f for f in metrics['features']};selected={};rejected=[]
86    for identifier,f in by_id.items():
87        p=f['properties'];s=p['slope_ratio'];h=p['entropy']
88        if s is None or h is None or not params.get('min_slope_ratio',.08)<=s<=params.get('max_slope_ratio',.50) or h>=params.get('max_entropy',.2):
89            rejected.append({'HydroID':identifier,'reason':'seed_threshold','slope_ratio':s,'entropy':h});continue
90        current=identifier;seen=set()
91        for hop in range(params.get('max_downstream_hops',32)+1):
92            if current is None or current not in by_id:break
93            if current in seen:raise ValueError('cycle in channel topology')
94            seen.add(current);selected.setdefault(current,dict(seed_HydroID=identifier,downstream_hops=hop))
95            current=by_id[current]['properties']['NextDownID']
96    # Preserve complete topology for analysis; clip geometries only for rendering.
97    selection=project_region(params,metrics['crs']);visible=[]
98    for identifier,p in selected.items():
99        f=by_id[identifier];geom=shape(f['geometry']).intersection(selection)
100        if not geom.is_empty:visible.append(dict(f,geometry=mapping(geom),properties=dict(f['properties'],**p)))
101    return {'synthetic':True,'crs':metrics['crs'],'region_bbox':params['region_bbox'],'region':params['region'],'HydroIDs':sorted(selected),
102            'features':visible,'channel_rejections':rejected,'metrics':metrics['features'],'tiles_processed':metrics['tiles_processed']}
103
104
105def model_path():
106    baked=Path('/opt/models/check-dam.pt')
107    if baked.is_file():return baked
108    source=Path(__file__)
109    if source.exists():return source.with_name('model.pt')
110    # Python entrypoint runs directly from a ZIP; the model is an immutable
111    # package asset, extracted into tmpfs rather than an intermediate dataset.
112    archive=next((p for p in source.parents if p.suffix=='.zip'),None)
113    if archive is None:raise ValueError('model package not found')
114    target=Path('/tmp/check-dam-model.pt')
115    with zipfile.ZipFile(archive) as package:target.write_bytes(package.read('model.pt'))
116    return target
117
118
119def detect(chunks,params):
120    import torch
121    torch.set_num_threads(2)
122    import contextlib,sys
123    with contextlib.redirect_stdout(sys.stderr):
124        from ultralytics import YOLO
125    model=YOLO(str(model_path()));selection=region(params);metadata=None;image_data=None;output=[];windows=0;tiles=0
126    for name,data in bundle(chunks):
127        if name.endswith('/metadata.json'):metadata=json.loads(data);image_data=None
128        elif name.endswith('/imagery.tif'):image_data=data
129        elif name.endswith('/coordinates.csv'):
130            if not metadata or image_data is None:raise ValueError('invalid imagery member order')
131            with MemoryFile(image_data) as mem,mem.open() as image:
132                if image.width*image.height>MAX_CELLS or image.count!=3:raise ValueError('invalid imagery')
133                for row in csv.DictReader(io.StringIO(data.decode())):
134                    x,y=image.xy(int(row['row']),int(row['col']))
135                    if CRS.from_user_input(row['crs'])!=image.crs or math.hypot(x-float(row['x']),y-float(row['y']))>.01:raise ValueError('coordinate control mismatch')
136                if not selection.intersects(__import__('shapely').geometry.box(*metadata['bbox_wgs84'])):continue
137                tile_selection=project_region(params,image.crs);to_geo=Transformer.from_crs(image.crs,'EPSG:4326',always_xy=True)
138                items=[];offsets=[]
139                def run_batch():
140                    nonlocal windows
141                    if not items:return
142                    predictions=model.predict(items,imgsz=128,conf=params.get('min_confidence',.35),iou=.5,device='cpu',verbose=False)
143                    windows+=len(items)
144                    for prediction,(r0,c0) in zip(predictions,offsets):
145                        for coords,conf in zip(prediction.boxes.xyxy.tolist(),prediction.boxes.conf.tolist()):
146                            x0,y0,x1,y1=coords;col=c0+(x0+x1)/2;row=r0+(y0+y1)/2
147                            x,y=image.transform*(col+.5,row+.5);lon,lat=to_geo.transform(x,y)
148                            if not selection.covers(Point(lon,lat)):continue
149                            output.append({'point_id':f"{metadata['tile']}-M{len(output)+1:04}",'tile':metadata['tile'],'x':x,'y':y,
150                                           'longitude':lon,'latitude':lat,'confidence':round(conf,5),'pixel_box':[c0+x0,r0+y0,c0+x1,r0+y1]})
151                    items.clear();offsets.clear()
152                from rasterio.windows import Window,bounds
153                for row in range(0,image.height,96):
154                    for col in range(0,image.width,96):
155                        window=Window(col,row,min(128,image.width-col),min(128,image.height-row))
156                        if not tile_selection.intersects(__import__('shapely').geometry.box(*bounds(window,image.transform))):continue
157                        rgb=np.moveaxis(np.clip(image.read(window=window)*255,0,255).astype('uint8'),0,2)
158                        padded=np.full((128,128,3),150,dtype='uint8');padded[:rgb.shape[0],:rgb.shape[1]]=rgb
159                        # Ultralytics expects BGR arrays.
160                        items.append(padded[:,:,::-1].copy());offsets.append((row,col))
161                        if len(items)==16:run_batch()
162                run_batch();tiles+=1
163            image_data=None
164    # Overlapping windows can detect the same object more than once.
165    unique=[];duplicates=[]
166    for p in sorted(output,key=lambda p:-p['confidence']):
167        if any(math.hypot(p['x']-q['x'],p['y']-q['y'])<params.get('duplicate_distance_m',60) for q in unique):
168            duplicates.append({'point_id':p['point_id'],'reason':'duplicate_detection'});continue
169        unique.append(p)
170    return {'synthetic':True,'crs':'EPSG:32649','region_bbox':params['region_bbox'],'region':params['region'],'points':unique,'rejections':duplicates,
171            'windows_inferred':windows,'tiles_inferred':tiles,'model':'YOLO11n-seg trained only on synthetic motifs'}
172
173
174def overlap(hazards,detected,chunks,params):
175    if hazards['region']!=detected['region'] or hazards['region']!=params['region']:raise ValueError('selection mismatch')
176    selection=region(params);selected=set(hazards['HydroIDs']);accepted=[];rejected=list(detected['rejections']);processed=set();metadata=None
177    by_tile={}
178    for p in detected['points']:
179        if not selection.covers(Point(p['longitude'],p['latitude'])):
180            rejected.append({'point_id':p['point_id'],'reason':'outside_region'});processed.add(p['point_id']);continue
181        by_tile.setdefault(p['tile'],[]).append(p)
182    bias=params.get('bias',3)
183    if not isinstance(bias,int) or not 0<=bias<=16:raise ValueError('invalid pixel window bias')
184    for name,data in bundle(chunks):
185        if name.endswith('/metadata.json'):metadata=json.loads(data)
186        elif name.endswith('/hydroid.tif'):
187            for p in by_tile.get(metadata['tile'],[]):
188                with MemoryFile(data) as mem,mem.open() as grid:
189                    row,col=grid.index(p['x'],p['y']);choices=[]
190                    from rasterio.windows import Window
191                    r0,c0=max(0,row-bias),max(0,col-bias);r1,c1=min(grid.height,row+bias+1),min(grid.width,col+bias+1)
192                    if r1>r0 and c1>c0:
193                        values=grid.read(1,window=Window(c0,r0,c1-c0,r1-r0))
194                        for rr,cc in zip(*np.nonzero(values)):
195                            x,y=grid.xy(r0+rr,c0+cc);choices.append((math.hypot(p['x']-x,p['y']-y),int(values[rr,cc])))
196                processed.add(p['point_id']);choices.sort(key=lambda v:(v[0],v[1]));ids=sorted(set(v[1] for v in choices))
197                if not choices or choices[0][1] not in selected:
198                    rejected.append(dict(point_id=p['point_id'],reason='no_channel_in_window' if not choices else 'non_susceptible_channel',matched_HydroIDs=ids));continue
199                props=dict(p,HydroID=choices[0][1],distance_to_channel_m=round(choices[0][0],2),ambiguous=len(ids)>1,matched_HydroIDs=ids)
200                accepted.append({'type':'Feature','geometry':{'type':'Point','coordinates':[p['x'],p['y']]},'properties':props})
201    for p in detected['points']:
202        if p['point_id'] not in processed:rejected.append({'point_id':p['point_id'],'reason':'missing_channel_tile'})
203    geojson={'type':'FeatureCollection','features':[dict(f,geometry={'type':'Point','coordinates':[f['properties']['longitude'],f['properties']['latitude']]}) for f in accepted]}
204    return {'synthetic':True,'crs':hazards['crs'],'region_bbox':params['region_bbox'],'region':params['region'],'features':accepted,'geojson':geojson,
205       'susceptible_channels':hazards['features'],'candidate_count':len(detected['points']),'rejections':rejected,
206       'channel_rejections':hazards['channel_rejections'],'parameters':params,'model':detected['model'],
207       'HydroIDs':hazards['HydroIDs'],'windows_inferred':detected['windows_inferred'],
208       'method':'Synthetic YOLO detection, normalized elevation entropy and topology, nearest HydroID in 7x7 window.'}
209
210
211def process(ctx):
212    params=dict(ctx.params);region(params);operation=params['operation']
213    if operation=='terrain':value=terrain(ctx.input('source').bytes(),params)
214    elif operation=='susceptibility':value=susceptibility(read_json(ctx,'source'),params)
215    elif operation=='detect':value=detect(ctx.input('source').bytes(),params)
216    elif operation=='overlap':value=overlap(read_json(ctx,'a_hazards'),read_json(ctx,'b_candidates'),ctx.input('c_grids').bytes(),params)
217    elif operation=='map':
218        out=ctx.bytes_output('result','svg');out.write(render(read_json(ctx,'source')));return
219    else:raise ValueError('unknown operation')
220    out=ctx.bytes_output('result','json');out.write(json.dumps(value,allow_nan=False).encode())
221
222
223def infer_outputs(inputs,params):
224    from xstream_operator import descriptor,result
225    return result({'result':descriptor(['bytes'],['svg' if params['operation']=='map' else 'json'],None,certainty='known')})
226
227
228def preview_outputs(inputs,params,limits):
229    from xstream_operator import preview_process
230    return preview_process(process,inputs,params,limits)
231
232if __name__=='__main__':
233    from xstream_operator import run_operator
234    run_operator(process,infer_outputs,preview_outputs=preview_outputs)