src/dacp_operator/inference.py
1"""Structure inference and bounded, operator-supplied output previews."""
2import base64
3from contextlib import redirect_stdout
4import json
5from pathlib import Path
6import re
7import sys
8import pyarrow as pa
9
10MAX_BYTES = 1024 * 1024
11
12
13def descriptor(representations=(), formats=(), schema=None, certainty='unknown', structure=None, sample=None):
14 value = dict(representations=list(representations), formats=list(formats), certainty=certainty)
15 if schema is not None:
16 value['schema'] = base64.b64encode(schema.serialize().to_pybytes()).decode('ascii')
17 if structure is not None:
18 value['structure'] = structure
19 if sample is not None:
20 value['sample'] = sample
21 _descriptor(value)
22 return value
23
24
25def records_schema(value):
26 raw = value.get('schema')
27 return pa.ipc.read_schema(pa.BufferReader(base64.b64decode(raw, validate=True))) if raw is not None else None
28
29
30def result(outputs, diagnostics=()):
31 value = dict(outputs=outputs, diagnostics=list(diagnostics))
32 validate_result(value)
33 return value
34
35
36def _descriptor(value):
37 if not isinstance(value, dict) or not {'representations', 'formats', 'certainty'} <= value.keys() or value.keys() - {'representations', 'formats', 'certainty', 'schema', 'structure', 'sample'}:
38 raise ValueError('invalid stream descriptor')
39 for key in ('representations', 'formats'):
40 xs = value[key]
41 if not isinstance(xs, list) or not all(isinstance(x, str) for x in xs) or len(set(xs)) != len(xs):
42 raise ValueError('descriptor values must be unique strings')
43 if any(not re.fullmatch(r'[a-z][a-z0-9_-]*', x) for x in value['representations']):
44 raise ValueError('invalid representation')
45 if value['certainty'] not in ('known', 'unknown') or (value['certainty'] == 'known' and not value['representations']):
46 raise ValueError('invalid inference certainty')
47 if value.get('schema') is not None:
48 if value['representations'] != ['records']:
49 raise ValueError('Arrow schema requires records representation')
50 records_schema(value)
51 if value['certainty'] == 'known' and value['representations'] == ['records'] and value.get('schema') is None:
52 raise ValueError('known records need a schema')
53 if value.get('structure') is not None and not isinstance(value['structure'], dict):
54 raise ValueError('structure must be an object')
55 if value.get('sample') is not None:
56 _sample(value['sample'])
57 if value['sample']['representation'] not in value['representations']:
58 raise ValueError('sample representation must match output')
59
60
61def _ports(values):
62 if not isinstance(values, dict) or len(values) > 128:
63 raise ValueError('invalid inference ports')
64 for name, value in values.items():
65 if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]{0,127}', name):
66 raise ValueError('invalid inference port')
67 _descriptor(value)
68
69
70def validate_result(value):
71 if not isinstance(value, dict) or value.keys() != {'outputs', 'diagnostics'}:
72 raise ValueError('inference result requires outputs and diagnostics')
73 _ports(value['outputs'])
74 ds = value['diagnostics']
75 if not isinstance(ds, list) or len(ds) > 128:
76 raise ValueError('invalid inference diagnostics')
77 for d in ds:
78 if not isinstance(d, dict) or d.keys() != {'severity', 'code', 'message'} or d['severity'] not in ('error', 'warning') or not all(isinstance(d[k], str) and d[k] for k in ('code', 'message')):
79 raise ValueError('invalid inference diagnostic')
80
81
82def preview_limits(value):
83 if not isinstance(value, dict) or value.keys() != {'max_rows', 'max_bytes'}:
84 raise ValueError('invalid preview limits')
85 if type(value['max_rows']) is not int or not 1 <= value['max_rows'] <= 100:
86 raise ValueError('preview max_rows must be between 1 and 100')
87 if type(value['max_bytes']) is not int or not 256 <= value['max_bytes'] <= 65536:
88 raise ValueError('preview max_bytes must be between 256 and 65536')
89 return value
90
91
92def sample_records(rows, *, truncated=False):
93 return dict(representation='records', format='json', rows=list(rows), truncated=truncated)
94
95
96def sample_bytes(data, *, format='binary', truncated=False):
97 return dict(representation='bytes', format=format,
98 data_base64=base64.b64encode(data).decode('ascii'), truncated=truncated)
99
100
101def _sample(value, max_bytes=65536):
102 if not isinstance(value, dict):
103 raise ValueError('invalid output sample')
104 key = 'rows' if value.get('representation') == 'records' else 'data_base64'
105 if value.keys() != {'representation', 'format', 'truncated', key} or not isinstance(value['format'], str) or not 1 <= len(value['format']) <= 128 or type(value['truncated']) is not bool:
106 raise ValueError('invalid output sample fields')
107 if key == 'rows':
108 if not isinstance(value['rows'], list) or len(value['rows']) > 100 or not all(isinstance(row, dict) for row in value['rows']):
109 raise ValueError('invalid sample rows')
110 elif value['representation'] == 'bytes' and isinstance(value['data_base64'], str):
111 base64.b64decode(value['data_base64'], validate=True)
112 else:
113 raise ValueError('invalid sample representation')
114 if len(json.dumps(value, allow_nan=False).encode('utf-8')) > max_bytes:
115 raise ValueError('output sample exceeds byte budget')
116
117
118def bounded_sample(value, limits):
119 preview_limits(limits)
120 value = dict(value)
121 if value.get('representation') == 'records':
122 rows = value.get('rows', [])
123 value['rows'] = rows[:limits['max_rows']]
124 value['truncated'] = bool(value.get('truncated')) or len(rows) > limits['max_rows']
125 while len(json.dumps(value, allow_nan=False).encode('utf-8')) > limits['max_bytes'] and value['rows']:
126 value['rows'].pop()
127 value['truncated'] = True
128 elif value.get('representation') == 'bytes':
129 data = base64.b64decode(value['data_base64'], validate=True)
130 cropped = data[:limits['max_bytes']]
131 value['truncated'] = bool(value.get('truncated')) or len(cropped) < len(data)
132 while True:
133 value['data_base64'] = base64.b64encode(cropped).decode('ascii')
134 excess = len(json.dumps(value, allow_nan=False).encode('utf-8')) - limits['max_bytes']
135 if excess <= 0 or not cropped:
136 break
137 cropped = cropped[:max(0, len(cropped) - excess)]
138 value['truncated'] = True
139 _sample(value, limits['max_bytes'])
140 return value
141
142
143def _attach_samples(value, samples, limits):
144 if not isinstance(samples, dict) or not samples.keys() <= value['outputs'].keys():
145 raise ValueError('sample refers to an undeclared output port')
146 value = {**value, 'outputs': {k: {a: b for a, b in d.items() if a != 'sample'} for k, d in value['outputs'].items()}}
147 budget = 65536
148 for port, sample in sorted(samples.items()):
149 if budget < 256:
150 break
151 bounded = bounded_sample(sample, {**limits, 'max_bytes': min(limits['max_bytes'], budget)})
152 value['outputs'][port]['sample'] = bounded
153 budget -= len(json.dumps(bounded, allow_nan=False).encode('utf-8'))
154 return value
155
156
157def evaluate_inference(method, raw, preview=None):
158 if not callable(preview):
159 raise TypeError('every operator must implement preview_outputs and register it with preview=')
160 if len(raw) > MAX_BYTES:
161 raise ValueError('inference request too large')
162 request = json.loads(raw)
163 if not isinstance(request, dict) or not {'inputs', 'params'} <= request.keys() or request.keys() - {'inputs', 'params', 'preview'} or not isinstance(request['params'], dict):
164 raise ValueError('invalid inference request')
165 _ports(request['inputs'])
166 with redirect_stdout(sys.stderr):
167 value = method(request['inputs'], request['params'])
168 if 'preview' in request:
169 limits = preview_limits(request['preview'])
170 try:
171 samples = preview(request['inputs'], request['params'], limits)
172 value = _attach_samples(value, samples, limits)
173 except Exception as error:
174 value = {**value, 'diagnostics': list(value.get('diagnostics', [])) + [
175 {'severity': 'warning', 'code': 'sample_unavailable', 'message': str(error) or 'Sample unavailable'}]}
176 validate_result(value)
177 raw = json.dumps(value, allow_nan=False).encode()
178 if len(raw) > MAX_BYTES:
179 raise ValueError('inference result too large')
180 return raw
181
182
183def serve_inference(method, args=None, input_stream=None, output=None, preview=None):
184 """A worker processes requests serially; errors terminate it instead of reusing state."""
185 if not callable(preview):
186 raise TypeError('every operator must implement preview_outputs')
187 import struct
188 from contextlib import ExitStack
189 args = sys.argv[1:] if args is None else args
190 if len(args) != 3 or args[0] != 'infer-worker':
191 raise ValueError('expected infer-worker REQUEST_FIFO RESPONSE_FIFO')
192 with ExitStack() as stack:
193 source = input_stream if input_stream is not None else stack.enter_context(open(args[1], 'rb', buffering=0))
194 target = output if output is not None else stack.enter_context(open(args[2], 'wb', buffering=0))
195 def read_exact(size):
196 data = bytearray()
197 while len(data) < size:
198 chunk = source.read(size - len(data))
199 if not chunk:
200 if not data:
201 return None
202 raise ValueError('truncated inference frame')
203 data.extend(chunk)
204 return bytes(data)
205 def write_all(raw):
206 view = memoryview(raw)
207 while view:
208 n = target.write(view)
209 if not n:
210 raise IOError('inference pipe closed')
211 view = view[n:]
212 write_all(struct.pack('>I', 0)); target.flush() # Ready; no result is cached.
213 while True:
214 header = read_exact(4)
215 if header is None:
216 return
217 size, = struct.unpack('>I', header)
218 if not 0 < size <= MAX_BYTES:
219 raise ValueError('inference frame exceeds limits')
220 raw = read_exact(size)
221 if raw is None:
222 raise ValueError('truncated inference request')
223 response = evaluate_inference(method, raw, preview)
224 write_all(struct.pack('>I', len(response))); write_all(response); target.flush()
225
226
227def run_inference(method, args=None, output=None, preview=None):
228 args = sys.argv[1:] if args is None else args
229 if args and args[0] == 'infer-worker':
230 return serve_inference(method, args, output=output, preview=preview)
231 if len(args) != 2 or args[0] != 'infer':
232 raise ValueError('expected infer REQUEST_JSON')
233 path = Path(args[1])
234 if path.stat().st_size > MAX_BYTES:
235 raise ValueError('inference request too large')
236 (sys.stdout.buffer if output is None else output).write(evaluate_inference(method, path.read_bytes(), preview))
237
238
239def run_operator(process, infer_outputs, args=None, preview_outputs=None):
240 """Entrypoint for function-style operators. Both methods are required."""
241 from . import OperatorContext
242 if not callable(preview_outputs):
243 raise TypeError('every operator must implement preview_outputs')
244 args = sys.argv[1:] if args is None else args
245 if args and args[0] in ('infer', 'infer-worker'):
246 run_inference(infer_outputs, args, preview=preview_outputs)
247 else:
248 with OperatorContext(args) as ctx:
249 process(ctx)
250
251
252def preview_process(process, inputs, params, limits):
253 """Run explicitly opted-in pure transforms on bounded in-memory samples only.
254
255 No filesystem paths or remote streams are exposed. This is not a sandbox for
256 arbitrary user code; use it only for side-effect-free processing methods.
257 """
258 from types import SimpleNamespace
259 import datetime
260 import decimal
261 limits = preview_limits(limits)
262 outputs = {}
263 class EnoughSamples(BaseException):
264 pass
265 def json_value(v):
266 if isinstance(v, (datetime.date, datetime.time, datetime.datetime)):
267 return v.isoformat()
268 if isinstance(v, decimal.Decimal):
269 return str(v)
270 if isinstance(v, bytes):
271 return base64.b64encode(v).decode('ascii')
272 if isinstance(v, dict):
273 return {k: json_value(x) for k, x in v.items()}
274 if isinstance(v, (list, tuple)):
275 return [json_value(x) for x in v]
276 return v
277 class Source:
278 def __init__(self, name):
279 self.description = inputs[name]
280 sample = self.description.get('sample')
281 if sample is None:
282 raise ValueError('Upstream sample unavailable for input: ' + name)
283 self.sample = bounded_sample(sample, limits)
284 self.schema = records_schema(self.description)
285 if self.schema is None and sample['representation'] == 'records':
286 # A sample-only schema is not advertised as the inferred output schema.
287 self.schema = pa.Table.from_pylist(self.sample['rows']).schema
288 self.representation = sample['representation']
289 self.format = sample['format']
290 def bytes(self):
291 if self.representation != 'bytes':
292 raise ValueError('bytes sample required')
293 yield base64.b64decode(self.sample['data_base64'], validate=True)
294 def records(self):
295 if self.representation != 'records' or self.schema is None:
296 raise ValueError('record sample and schema required')
297 yield pa.RecordBatch.from_pylist(self.sample['rows'], schema=self.schema)
298 def records_output(port, schema):
299 sample = sample_records([])
300 outputs[port] = sample
301 def write(batch):
302 for row in batch.to_pylist():
303 if len(sample['rows']) >= limits['max_rows']:
304 sample['truncated'] = True
305 raise EnoughSamples()
306 candidate = bounded_sample({**sample, 'rows': sample['rows'] + [json_value(row)]}, limits)
307 if len(candidate['rows']) <= len(sample['rows']):
308 sample['truncated'] = True
309 raise EnoughSamples()
310 sample.update(candidate)
311 return SimpleNamespace(write=write)
312 def bytes_output(port, format='binary'):
313 outputs[port] = sample_bytes(b'', format=format)
314 data = bytearray()
315 def write(chunk):
316 data.extend(bytes(chunk)[:max(0, limits['max_bytes'] + 1 - len(data))])
317 outputs[port] = bounded_sample(sample_bytes(bytes(data), format=format), limits)
318 if outputs[port]['truncated']:
319 raise EnoughSamples()
320 return SimpleNamespace(write=write)
321 context = SimpleNamespace(params=params, input=Source, input_names=list(inputs),
322 records_output=records_output, bytes_output=bytes_output)
323 try:
324 process(context)
325 except EnoughSamples:
326 pass
327 if any(d.get('sample', {}).get('truncated') for d in inputs.values()):
328 for sample in outputs.values():
329 sample['truncated'] = True
330 return {port: bounded_sample(sample, limits) for port, sample in outputs.items()}