Source code

Download source archive ↓

src/dacp_operator/__init__.py

1"""Helpers for packaged DACP operators. stdout is reserved for protocol frames."""
2import base64
3from contextlib import ExitStack, redirect_stdout
4import json
5from pathlib import Path
6import re
7import struct
8import sys
9
10import pyarrow as pa
11from .inference import descriptor, result, records_schema, run_inference, run_operator, sample_records, sample_bytes, bounded_sample, preview_process
12
13
14class OperatorContext:
15    def __init__(self, args=None, output=None, input_streams=None, emit=None):
16        args = sys.argv[1:] if args is None else args
17        if len(args) != 3 or args[0] != 'operator':
18            raise ValueError('expected operator INPUT PARAMS')
19        self.directory = Path(args[1])
20        self.params = json.loads(Path(args[2]).read_text(encoding='utf-8'))
21        if not isinstance(self.params, dict):
22            raise ValueError('params must be a JSON object')
23        manifest = json.loads((self.directory / 'manifest.json').read_text(encoding='utf-8'))
24        if not isinstance(manifest, list):
25            raise ValueError('manifest must be an array')
26        self._entries = {}
27        for entry in manifest:
28            name = entry['port']
29            if not _valid_name(name) or name in self._entries:
30                raise ValueError('invalid or duplicate input port')
31            self._entries[name] = entry
32        self.input_names = tuple(sorted(self._entries))
33        self._index = 0
34        self._inputs = {}
35        self._outputs = {}
36        self._writing = False
37        self._closed = False
38        self._finished = False
39        self._resources = ExitStack()
40        self._input_streams = input_streams or {}
41        self._emit = emit
42        self._output = sys.stdout.buffer if output is None else output
43        self._redirect = redirect_stdout(sys.stderr) if emit is None else None
44
45    def __enter__(self):
46        self._active()
47        if self._redirect is not None:
48            self._redirect.__enter__()
49        return self
50
51    def __exit__(self, kind, value, traceback):
52        try:
53            if kind is None:
54                if self._index != len(self.input_names):
55                    raise ValueError('all inputs must be consumed to Arrow EOS')
56                for port in self._outputs:
57                    self._frame(port, 'end')
58                self._finished = True
59        finally:
60            try:
61                self._resources.close()
62            finally:
63                self._closed = True
64                if self._redirect is not None:
65                    self._redirect.__exit__(kind, value, traceback)
66
67    def _active(self):
68        if self._closed or self._finished:
69            raise ValueError('operator context ended')
70
71    def _frame(self, port, kind, payload=b'', **metadata):
72        self._active()
73        event = dict(type=kind, port=port, **metadata)
74        header = json.dumps(event).encode('utf-8')
75        if len(header) > 65536 or len(payload) > 8 * 1024 * 1024:
76            raise ValueError('frame exceeds limits; split large record batches')
77        if self._emit is not None:
78            self._emit(event, payload)
79            return
80        self._output.write(struct.pack('>II', len(header), len(payload)) + header + payload)
81        self._output.flush()
82
83    def input(self, name):
84        self._active()
85        if name not in self._entries:
86            raise ValueError('unknown input port')
87        if name not in self._inputs:
88            self._inputs[name] = _Input(self, name, self._entries[name])
89        return self._inputs[name]
90
91    def _new_output(self, name, representation, format, schema=None):
92        self._active()
93        if not _valid_name(name) or name in self._outputs:
94            raise ValueError('invalid or duplicate output port')
95        if self._writing:
96            raise ValueError('declare all outputs before writing data')
97        metadata = dict(representation=representation, format=format)
98        if schema is not None:
99            metadata['schema'] = base64.b64encode(schema.serialize().to_pybytes()).decode('ascii')
100        self._frame(name, 'start', **metadata)
101        value = _Output(self, name, representation, schema)
102        self._outputs[name] = value
103        return value
104
105    def bytes_output(self, name, format='binary'):
106        return self._new_output(name, 'bytes', format)
107
108    def records_output(self, name, schema, format='arrow'):
109        return self._new_output(name, 'records', format, schema)
110
111
112class _Input:
113    def __init__(self, context, name, entry):
114        self._context, self.name, self._entry = context, name, entry
115        self.representation, self.format = entry['representation'], entry['format']
116        if self.representation not in ('bytes', 'records'):
117            raise ValueError('unsupported input representation')
118        self._reader = None
119        self._consumed = False
120        self._iterating = False
121
122    def _open(self):
123        ctx = self._context
124        ctx._active()
125        if (self._consumed or self._iterating or ctx._index >= len(ctx.input_names)
126                or ctx.input_names[ctx._index] != self.name):
127            raise ValueError('consume inputs once in lexical port order')
128        if self._reader is None:
129            filename = self._entry['file']
130            if not filename or Path(filename).name != filename or filename in ('.', '..'):
131                raise ValueError('invalid input filename')
132            self._raw = ctx._resources.enter_context(ctx._input_streams[self.name] if self.name in ctx._input_streams else (ctx.directory / filename).open('rb'))
133            self._reader = ctx._resources.enter_context(pa.ipc.open_stream(self._raw))
134            if self.representation == 'bytes' and self._reader.schema != pa.schema([('data', pa.binary())]):
135                raise ValueError('invalid bytes schema')
136        return self._reader
137
138    @property
139    def schema(self):
140        return self._open().schema
141
142    def _batches(self):
143        reader = self._open()
144        self._iterating = True
145        try:
146            yield from reader
147            self._consumed = True
148            self._context._index += 1
149        finally:
150            reader.close()
151            self._raw.close()
152
153    def records(self):
154        if self.representation != 'records':
155            raise ValueError('records input required')
156        return self._batches()
157
158    def bytes(self):
159        if self.representation != 'bytes':
160            raise ValueError('bytes input required')
161        for batch in self._batches():
162            for value in batch.column(0):
163                chunk = value.as_py()
164                if chunk is None:
165                    raise ValueError('null bytes chunk')
166                yield chunk
167
168
169class _Output:
170    def __init__(self, context, name, representation, schema):
171        self._context, self.name = context, name
172        self.representation, self.schema = representation, schema
173
174    def write(self, value):
175        ctx = self._context
176        ctx._active()
177        if self.representation == 'bytes':
178            if not isinstance(value, (bytes, bytearray, memoryview)):
179                raise TypeError('bytes output requires bytes-like values')
180            ctx._writing = True
181            for offset in range(0, len(value), 65536):
182                ctx._frame(self.name, 'data', bytes(value[offset:offset + 65536]))
183        else:
184            if not isinstance(value, pa.RecordBatch) or not value.schema.equals(self.schema, check_metadata=True):
185                raise ValueError('output schema mismatch')
186            ctx._writing = True
187            sink = pa.BufferOutputStream()
188            with pa.ipc.new_stream(sink, self.schema) as writer:
189                writer.write_batch(value)
190            ctx._frame(self.name, 'data', sink.getvalue().to_pybytes())
191
192
193def _valid_name(name):
194    return isinstance(name, str) and re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]{0,127}', name) is not None
195
196
197class Operator:
198    def infer_outputs(self, inputs, params):
199        raise NotImplementedError('every operator must implement infer_outputs')
200
201    def process(self, ctx):
202        raise NotImplementedError
203
204    def preview_outputs(self, inputs, params, limits):
205        raise NotImplementedError('every operator must implement preview_outputs')
206
207    def main(self):
208        if type(self).preview_outputs is Operator.preview_outputs:
209            raise TypeError('every operator must implement preview_outputs')
210        run_operator(self.process, self.infer_outputs, preview_outputs=self.preview_outputs)
211
212
213class BytesOperator(Operator):
214    def preview_outputs(self, inputs, params, limits):
215        return preview_process(self.process, inputs, params, limits)
216
217    output_format = 'binary'
218
219    def infer_outputs(self, inputs, params):
220        if set(inputs) != {'source'} or (inputs['source']['representations'] and 'bytes' not in inputs['source']['representations']):
221            raise ValueError('expected source bytes input')
222        return result({'result': descriptor(['bytes'], [self.output_format], certainty='known')})
223
224    def transform(self, chunk, params):
225        raise NotImplementedError
226
227    def process(self, ctx):
228        output = ctx.bytes_output('result', self.output_format)
229        for chunk in ctx.input('source').bytes():
230            output.write(self.transform(chunk, ctx.params))
231
232
233
234class RecordsOperator(Operator):
235    def preview_outputs(self, inputs, params, limits):
236        return preview_process(self.process, inputs, params, limits)
237
238    def infer_outputs(self, inputs, params):
239        if set(inputs) != {'source'} or (inputs['source']['representations'] and 'records' not in inputs['source']['representations']):
240            raise ValueError('expected source records input')
241        schema = records_schema(inputs['source'])
242        schema = self.output_schema(schema, params) if schema is not None else None
243        return result({'result': descriptor(['records'], ['arrow'], schema, 'known' if schema is not None else 'unknown')})
244
245    def output_schema(self, input_schema, params):
246        return input_schema
247
248    def transform(self, batch, params):
249        raise NotImplementedError
250
251    def process(self, ctx):
252        source = ctx.input('source')
253        output = ctx.records_output('result', self.output_schema(source.schema, ctx.params))
254        for batch in source.records():
255            result = self.transform(batch, ctx.params)
256            if result is not None:
257                output.write(result)