Source code

Download source archive ↓

src/dacp_operator/inference.py

1"""Structure inference and bounded, operator-supplied output previews."""
2import base64
3from contextlib import redirect_stdout
4import json
5from pathlib import Path
6import re
7import sys
8import pyarrow as pa
9
10MAX_BYTES = 1024 * 1024
11
12
13def descriptor(representations=(), formats=(), schema=None, certainty='unknown', structure=None, sample=None):
14    value = dict(representations=list(representations), formats=list(formats), certainty=certainty)
15    if schema is not None:
16        value['schema'] = base64.b64encode(schema.serialize().to_pybytes()).decode('ascii')
17    if structure is not None:
18        value['structure'] = structure
19    if sample is not None:
20        value['sample'] = sample
21    _descriptor(value)
22    return value
23
24
25def records_schema(value):
26    raw = value.get('schema')
27    return pa.ipc.read_schema(pa.BufferReader(base64.b64decode(raw, validate=True))) if raw is not None else None
28
29
30def result(outputs, diagnostics=()):
31    value = dict(outputs=outputs, diagnostics=list(diagnostics))
32    validate_result(value)
33    return value
34
35
36def _descriptor(value):
37    if not isinstance(value, dict) or not {'representations', 'formats', 'certainty'} <= value.keys() or value.keys() - {'representations', 'formats', 'certainty', 'schema', 'structure', 'sample'}:
38        raise ValueError('invalid stream descriptor')
39    for key in ('representations', 'formats'):
40        xs = value[key]
41        if not isinstance(xs, list) or not all(isinstance(x, str) for x in xs) or len(set(xs)) != len(xs):
42            raise ValueError('descriptor values must be unique strings')
43    if any(not re.fullmatch(r'[a-z][a-z0-9_-]*', x) for x in value['representations']):
44        raise ValueError('invalid representation')
45    if value['certainty'] not in ('known', 'unknown') or (value['certainty'] == 'known' and not value['representations']):
46        raise ValueError('invalid inference certainty')
47    if value.get('schema') is not None:
48        if value['representations'] != ['records']:
49            raise ValueError('Arrow schema requires records representation')
50        records_schema(value)
51    if value['certainty'] == 'known' and value['representations'] == ['records'] and value.get('schema') is None:
52        raise ValueError('known records need a schema')
53    if value.get('structure') is not None and not isinstance(value['structure'], dict):
54        raise ValueError('structure must be an object')
55    if value.get('sample') is not None:
56        _sample(value['sample'])
57        if value['sample']['representation'] not in value['representations']:
58            raise ValueError('sample representation must match output')
59
60
61def _ports(values):
62    if not isinstance(values, dict) or len(values) > 128:
63        raise ValueError('invalid inference ports')
64    for name, value in values.items():
65        if not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]{0,127}', name):
66            raise ValueError('invalid inference port')
67        _descriptor(value)
68
69
70def validate_result(value):
71    if not isinstance(value, dict) or value.keys() != {'outputs', 'diagnostics'}:
72        raise ValueError('inference result requires outputs and diagnostics')
73    _ports(value['outputs'])
74    ds = value['diagnostics']
75    if not isinstance(ds, list) or len(ds) > 128:
76        raise ValueError('invalid inference diagnostics')
77    for d in ds:
78        if not isinstance(d, dict) or d.keys() != {'severity', 'code', 'message'} or d['severity'] not in ('error', 'warning') or not all(isinstance(d[k], str) and d[k] for k in ('code', 'message')):
79            raise ValueError('invalid inference diagnostic')
80
81
82def preview_limits(value):
83    if not isinstance(value, dict) or value.keys() != {'max_rows', 'max_bytes'}:
84        raise ValueError('invalid preview limits')
85    if type(value['max_rows']) is not int or not 1 <= value['max_rows'] <= 100:
86        raise ValueError('preview max_rows must be between 1 and 100')
87    if type(value['max_bytes']) is not int or not 256 <= value['max_bytes'] <= 65536:
88        raise ValueError('preview max_bytes must be between 256 and 65536')
89    return value
90
91
92def sample_records(rows, *, truncated=False):
93    return dict(representation='records', format='json', rows=list(rows), truncated=truncated)
94
95
96def sample_bytes(data, *, format='binary', truncated=False):
97    return dict(representation='bytes', format=format,
98                data_base64=base64.b64encode(data).decode('ascii'), truncated=truncated)
99
100
101def _sample(value, max_bytes=65536):
102    if not isinstance(value, dict):
103        raise ValueError('invalid output sample')
104    key = 'rows' if value.get('representation') == 'records' else 'data_base64'
105    if value.keys() != {'representation', 'format', 'truncated', key} or not isinstance(value['format'], str) or not 1 <= len(value['format']) <= 128 or type(value['truncated']) is not bool:
106        raise ValueError('invalid output sample fields')
107    if key == 'rows':
108        if not isinstance(value['rows'], list) or len(value['rows']) > 100 or not all(isinstance(row, dict) for row in value['rows']):
109            raise ValueError('invalid sample rows')
110    elif value['representation'] == 'bytes' and isinstance(value['data_base64'], str):
111        base64.b64decode(value['data_base64'], validate=True)
112    else:
113        raise ValueError('invalid sample representation')
114    if len(json.dumps(value, allow_nan=False).encode('utf-8')) > max_bytes:
115        raise ValueError('output sample exceeds byte budget')
116
117
118def bounded_sample(value, limits):
119    preview_limits(limits)
120    value = dict(value)
121    if value.get('representation') == 'records':
122        rows = value.get('rows', [])
123        value['rows'] = rows[:limits['max_rows']]
124        value['truncated'] = bool(value.get('truncated')) or len(rows) > limits['max_rows']
125        while len(json.dumps(value, allow_nan=False).encode('utf-8')) > limits['max_bytes'] and value['rows']:
126            value['rows'].pop()
127            value['truncated'] = True
128    elif value.get('representation') == 'bytes':
129        data = base64.b64decode(value['data_base64'], validate=True)
130        cropped = data[:limits['max_bytes']]
131        value['truncated'] = bool(value.get('truncated')) or len(cropped) < len(data)
132        while True:
133            value['data_base64'] = base64.b64encode(cropped).decode('ascii')
134            excess = len(json.dumps(value, allow_nan=False).encode('utf-8')) - limits['max_bytes']
135            if excess <= 0 or not cropped:
136                break
137            cropped = cropped[:max(0, len(cropped) - excess)]
138            value['truncated'] = True
139    _sample(value, limits['max_bytes'])
140    return value
141
142
143def _attach_samples(value, samples, limits):
144    if not isinstance(samples, dict) or not samples.keys() <= value['outputs'].keys():
145        raise ValueError('sample refers to an undeclared output port')
146    value = {**value, 'outputs': {k: {a: b for a, b in d.items() if a != 'sample'} for k, d in value['outputs'].items()}}
147    budget = 65536
148    for port, sample in sorted(samples.items()):
149        if budget < 256:
150            break
151        bounded = bounded_sample(sample, {**limits, 'max_bytes': min(limits['max_bytes'], budget)})
152        value['outputs'][port]['sample'] = bounded
153        budget -= len(json.dumps(bounded, allow_nan=False).encode('utf-8'))
154    return value
155
156
157def evaluate_inference(method, raw, preview=None):
158    if not callable(preview):
159        raise TypeError('every operator must implement preview_outputs and register it with preview=')
160    if len(raw) > MAX_BYTES:
161        raise ValueError('inference request too large')
162    request = json.loads(raw)
163    if not isinstance(request, dict) or not {'inputs', 'params'} <= request.keys() or request.keys() - {'inputs', 'params', 'preview'} or not isinstance(request['params'], dict):
164        raise ValueError('invalid inference request')
165    _ports(request['inputs'])
166    with redirect_stdout(sys.stderr):
167        value = method(request['inputs'], request['params'])
168        if 'preview' in request:
169            limits = preview_limits(request['preview'])
170            try:
171                samples = preview(request['inputs'], request['params'], limits)
172                value = _attach_samples(value, samples, limits)
173            except Exception as error:
174                value = {**value, 'diagnostics': list(value.get('diagnostics', [])) + [
175                    {'severity': 'warning', 'code': 'sample_unavailable', 'message': str(error) or 'Sample unavailable'}]}
176    validate_result(value)
177    raw = json.dumps(value, allow_nan=False).encode()
178    if len(raw) > MAX_BYTES:
179        raise ValueError('inference result too large')
180    return raw
181
182
183def serve_inference(method, args=None, input_stream=None, output=None, preview=None):
184    """A worker processes requests serially; errors terminate it instead of reusing state."""
185    if not callable(preview):
186        raise TypeError('every operator must implement preview_outputs')
187    import struct
188    from contextlib import ExitStack
189    args = sys.argv[1:] if args is None else args
190    if len(args) != 3 or args[0] != 'infer-worker':
191        raise ValueError('expected infer-worker REQUEST_FIFO RESPONSE_FIFO')
192    with ExitStack() as stack:
193        source = input_stream if input_stream is not None else stack.enter_context(open(args[1], 'rb', buffering=0))
194        target = output if output is not None else stack.enter_context(open(args[2], 'wb', buffering=0))
195        def read_exact(size):
196            data = bytearray()
197            while len(data) < size:
198                chunk = source.read(size - len(data))
199                if not chunk:
200                    if not data:
201                        return None
202                    raise ValueError('truncated inference frame')
203                data.extend(chunk)
204            return bytes(data)
205        def write_all(raw):
206            view = memoryview(raw)
207            while view:
208                n = target.write(view)
209                if not n:
210                    raise IOError('inference pipe closed')
211                view = view[n:]
212        write_all(struct.pack('>I', 0)); target.flush()  # Ready; no result is cached.
213        while True:
214            header = read_exact(4)
215            if header is None:
216                return
217            size, = struct.unpack('>I', header)
218            if not 0 < size <= MAX_BYTES:
219                raise ValueError('inference frame exceeds limits')
220            raw = read_exact(size)
221            if raw is None:
222                raise ValueError('truncated inference request')
223            response = evaluate_inference(method, raw, preview)
224            write_all(struct.pack('>I', len(response))); write_all(response); target.flush()
225
226
227def run_inference(method, args=None, output=None, preview=None):
228    args = sys.argv[1:] if args is None else args
229    if args and args[0] == 'infer-worker':
230        return serve_inference(method, args, output=output, preview=preview)
231    if len(args) != 2 or args[0] != 'infer':
232        raise ValueError('expected infer REQUEST_JSON')
233    path = Path(args[1])
234    if path.stat().st_size > MAX_BYTES:
235        raise ValueError('inference request too large')
236    (sys.stdout.buffer if output is None else output).write(evaluate_inference(method, path.read_bytes(), preview))
237
238
239def run_operator(process, infer_outputs, args=None, preview_outputs=None):
240    """Entrypoint for function-style operators. Both methods are required."""
241    from . import OperatorContext
242    if not callable(preview_outputs):
243        raise TypeError('every operator must implement preview_outputs')
244    args = sys.argv[1:] if args is None else args
245    if args and args[0] in ('infer', 'infer-worker'):
246        run_inference(infer_outputs, args, preview=preview_outputs)
247    else:
248        with OperatorContext(args) as ctx:
249            process(ctx)
250
251
252def preview_process(process, inputs, params, limits):
253    """Run explicitly opted-in pure transforms on bounded in-memory samples only.
254
255    No filesystem paths or remote streams are exposed. This is not a sandbox for
256    arbitrary user code; use it only for side-effect-free processing methods.
257    """
258    from types import SimpleNamespace
259    import datetime
260    import decimal
261    limits = preview_limits(limits)
262    outputs = {}
263    class EnoughSamples(BaseException):
264        pass
265    def json_value(v):
266        if isinstance(v, (datetime.date, datetime.time, datetime.datetime)):
267            return v.isoformat()
268        if isinstance(v, decimal.Decimal):
269            return str(v)
270        if isinstance(v, bytes):
271            return base64.b64encode(v).decode('ascii')
272        if isinstance(v, dict):
273            return {k: json_value(x) for k, x in v.items()}
274        if isinstance(v, (list, tuple)):
275            return [json_value(x) for x in v]
276        return v
277    class Source:
278        def __init__(self, name):
279            self.description = inputs[name]
280            sample = self.description.get('sample')
281            if sample is None:
282                raise ValueError('Upstream sample unavailable for input: ' + name)
283            self.sample = bounded_sample(sample, limits)
284            self.schema = records_schema(self.description)
285            if self.schema is None and sample['representation'] == 'records':
286                # A sample-only schema is not advertised as the inferred output schema.
287                self.schema = pa.Table.from_pylist(self.sample['rows']).schema
288            self.representation = sample['representation']
289            self.format = sample['format']
290        def bytes(self):
291            if self.representation != 'bytes':
292                raise ValueError('bytes sample required')
293            yield base64.b64decode(self.sample['data_base64'], validate=True)
294        def records(self):
295            if self.representation != 'records' or self.schema is None:
296                raise ValueError('record sample and schema required')
297            yield pa.RecordBatch.from_pylist(self.sample['rows'], schema=self.schema)
298    def records_output(port, schema):
299        sample = sample_records([])
300        outputs[port] = sample
301        def write(batch):
302            for row in batch.to_pylist():
303                if len(sample['rows']) >= limits['max_rows']:
304                    sample['truncated'] = True
305                    raise EnoughSamples()
306                candidate = bounded_sample({**sample, 'rows': sample['rows'] + [json_value(row)]}, limits)
307                if len(candidate['rows']) <= len(sample['rows']):
308                    sample['truncated'] = True
309                    raise EnoughSamples()
310                sample.update(candidate)
311        return SimpleNamespace(write=write)
312    def bytes_output(port, format='binary'):
313        outputs[port] = sample_bytes(b'', format=format)
314        data = bytearray()
315        def write(chunk):
316            data.extend(bytes(chunk)[:max(0, limits['max_bytes'] + 1 - len(data))])
317            outputs[port] = bounded_sample(sample_bytes(bytes(data), format=format), limits)
318            if outputs[port]['truncated']:
319                raise EnoughSamples()
320        return SimpleNamespace(write=write)
321    context = SimpleNamespace(params=params, input=Source, input_names=list(inputs),
322                              records_output=records_output, bytes_output=bytes_output)
323    try:
324        process(context)
325    except EnoughSamples:
326        pass
327    if any(d.get('sample', {}).get('truncated') for d in inputs.values()):
328        for sample in outputs.values():
329            sample['truncated'] = True
330    return {port: bounded_sample(sample, limits) for port, sample in outputs.items()}