src/dacp_operator/__init__.py
1"""Helpers for packaged DACP operators. stdout is reserved for protocol frames."""
2import base64
3from contextlib import ExitStack, redirect_stdout
4import json
5from pathlib import Path
6import re
7import struct
8import sys
9
10import pyarrow as pa
11from .inference import descriptor, result, records_schema, run_inference, run_operator, sample_records, sample_bytes, bounded_sample, preview_process
12
13
14class OperatorContext:
15 def __init__(self, args=None, output=None, input_streams=None, emit=None):
16 args = sys.argv[1:] if args is None else args
17 if len(args) != 3 or args[0] != 'operator':
18 raise ValueError('expected operator INPUT PARAMS')
19 self.directory = Path(args[1])
20 self.params = json.loads(Path(args[2]).read_text(encoding='utf-8'))
21 if not isinstance(self.params, dict):
22 raise ValueError('params must be a JSON object')
23 manifest = json.loads((self.directory / 'manifest.json').read_text(encoding='utf-8'))
24 if not isinstance(manifest, list):
25 raise ValueError('manifest must be an array')
26 self._entries = {}
27 for entry in manifest:
28 name = entry['port']
29 if not _valid_name(name) or name in self._entries:
30 raise ValueError('invalid or duplicate input port')
31 self._entries[name] = entry
32 self.input_names = tuple(sorted(self._entries))
33 self._index = 0
34 self._inputs = {}
35 self._outputs = {}
36 self._writing = False
37 self._closed = False
38 self._finished = False
39 self._resources = ExitStack()
40 self._input_streams = input_streams or {}
41 self._emit = emit
42 self._output = sys.stdout.buffer if output is None else output
43 self._redirect = redirect_stdout(sys.stderr) if emit is None else None
44
45 def __enter__(self):
46 self._active()
47 if self._redirect is not None:
48 self._redirect.__enter__()
49 return self
50
51 def __exit__(self, kind, value, traceback):
52 try:
53 if kind is None:
54 if self._index != len(self.input_names):
55 raise ValueError('all inputs must be consumed to Arrow EOS')
56 for port in self._outputs:
57 self._frame(port, 'end')
58 self._finished = True
59 finally:
60 try:
61 self._resources.close()
62 finally:
63 self._closed = True
64 if self._redirect is not None:
65 self._redirect.__exit__(kind, value, traceback)
66
67 def _active(self):
68 if self._closed or self._finished:
69 raise ValueError('operator context ended')
70
71 def _frame(self, port, kind, payload=b'', **metadata):
72 self._active()
73 event = dict(type=kind, port=port, **metadata)
74 header = json.dumps(event).encode('utf-8')
75 if len(header) > 65536 or len(payload) > 8 * 1024 * 1024:
76 raise ValueError('frame exceeds limits; split large record batches')
77 if self._emit is not None:
78 self._emit(event, payload)
79 return
80 self._output.write(struct.pack('>II', len(header), len(payload)) + header + payload)
81 self._output.flush()
82
83 def input(self, name):
84 self._active()
85 if name not in self._entries:
86 raise ValueError('unknown input port')
87 if name not in self._inputs:
88 self._inputs[name] = _Input(self, name, self._entries[name])
89 return self._inputs[name]
90
91 def _new_output(self, name, representation, format, schema=None):
92 self._active()
93 if not _valid_name(name) or name in self._outputs:
94 raise ValueError('invalid or duplicate output port')
95 if self._writing:
96 raise ValueError('declare all outputs before writing data')
97 metadata = dict(representation=representation, format=format)
98 if schema is not None:
99 metadata['schema'] = base64.b64encode(schema.serialize().to_pybytes()).decode('ascii')
100 self._frame(name, 'start', **metadata)
101 value = _Output(self, name, representation, schema)
102 self._outputs[name] = value
103 return value
104
105 def bytes_output(self, name, format='binary'):
106 return self._new_output(name, 'bytes', format)
107
108 def records_output(self, name, schema, format='arrow'):
109 return self._new_output(name, 'records', format, schema)
110
111
112class _Input:
113 def __init__(self, context, name, entry):
114 self._context, self.name, self._entry = context, name, entry
115 self.representation, self.format = entry['representation'], entry['format']
116 if self.representation not in ('bytes', 'records'):
117 raise ValueError('unsupported input representation')
118 self._reader = None
119 self._consumed = False
120 self._iterating = False
121
122 def _open(self, metadata_only=False):
123 ctx = self._context
124 ctx._active()
125 if (self._consumed or self._iterating or ctx._index >= len(ctx.input_names)
126 or (not metadata_only and ctx.input_names[ctx._index] != self.name)):
127 raise ValueError('consume inputs once in lexical port order')
128 if self._reader is None:
129 filename = self._entry['file']
130 if not filename or Path(filename).name != filename or filename in ('.', '..'):
131 raise ValueError('invalid input filename')
132 self._raw = ctx._resources.enter_context(ctx._input_streams[self.name] if self.name in ctx._input_streams else (ctx.directory / filename).open('rb'))
133 self._reader = ctx._resources.enter_context(pa.ipc.open_stream(self._raw))
134 if self.representation == 'bytes' and self._reader.schema != pa.schema([('data', pa.binary())]):
135 raise ValueError('invalid bytes schema')
136 return self._reader
137
138 @property
139 def schema(self):
140 return self._open(metadata_only=True).schema
141
142 def _batches(self):
143 reader = self._open()
144 self._iterating = True
145 try:
146 yield from reader
147 self._consumed = True
148 self._context._index += 1
149 finally:
150 reader.close()
151 self._raw.close()
152
153 def records(self):
154 if self.representation != 'records':
155 raise ValueError('records input required')
156 return self._batches()
157
158 def bytes(self):
159 if self.representation != 'bytes':
160 raise ValueError('bytes input required')
161 for batch in self._batches():
162 for value in batch.column(0):
163 chunk = value.as_py()
164 if chunk is None:
165 raise ValueError('null bytes chunk')
166 yield chunk
167
168
169class _Output:
170 def __init__(self, context, name, representation, schema):
171 self._context, self.name = context, name
172 self.representation, self.schema = representation, schema
173
174 def write(self, value):
175 ctx = self._context
176 ctx._active()
177 if self.representation == 'bytes':
178 if not isinstance(value, (bytes, bytearray, memoryview)):
179 raise TypeError('bytes output requires bytes-like values')
180 ctx._writing = True
181 for offset in range(0, len(value), 65536):
182 ctx._frame(self.name, 'data', bytes(value[offset:offset + 65536]))
183 else:
184 if not isinstance(value, pa.RecordBatch) or not value.schema.equals(self.schema, check_metadata=True):
185 raise ValueError('output schema mismatch')
186 ctx._writing = True
187 sink = pa.BufferOutputStream()
188 with pa.ipc.new_stream(sink, self.schema) as writer:
189 writer.write_batch(value)
190 ctx._frame(self.name, 'data', sink.getvalue().to_pybytes())
191
192
193def _valid_name(name):
194 return isinstance(name, str) and re.fullmatch(r'[A-Za-z_][A-Za-z0-9_-]{0,127}', name) is not None
195
196
197class Operator:
198 def infer_outputs(self, inputs, params):
199 raise NotImplementedError('every operator must implement infer_outputs')
200
201 def process(self, ctx):
202 raise NotImplementedError
203
204 def preview_outputs(self, inputs, params, limits):
205 raise NotImplementedError('every operator must implement preview_outputs')
206
207 def main(self):
208 if type(self).preview_outputs is Operator.preview_outputs:
209 raise TypeError('every operator must implement preview_outputs')
210 run_operator(self.process, self.infer_outputs, preview_outputs=self.preview_outputs)
211
212
213class BytesOperator(Operator):
214 def preview_outputs(self, inputs, params, limits):
215 return preview_process(self.process, inputs, params, limits)
216
217 output_format = 'binary'
218
219 def infer_outputs(self, inputs, params):
220 if set(inputs) != {'source'} or (inputs['source']['representations'] and 'bytes' not in inputs['source']['representations']):
221 raise ValueError('expected source bytes input')
222 return result({'result': descriptor(['bytes'], [self.output_format], certainty='known')})
223
224 def transform(self, chunk, params):
225 raise NotImplementedError
226
227 def process(self, ctx):
228 output = ctx.bytes_output('result', self.output_format)
229 for chunk in ctx.input('source').bytes():
230 output.write(self.transform(chunk, ctx.params))
231
232
233
234class RecordsOperator(Operator):
235 def preview_outputs(self, inputs, params, limits):
236 return preview_process(self.process, inputs, params, limits)
237
238 def infer_outputs(self, inputs, params):
239 if set(inputs) != {'source'} or (inputs['source']['representations'] and 'records' not in inputs['source']['representations']):
240 raise ValueError('expected source records input')
241 schema = records_schema(inputs['source'])
242 schema = self.output_schema(schema, params) if schema is not None else None
243 return result({'result': descriptor(['records'], ['arrow'], schema, 'known' if schema is not None else 'unknown')})
244
245 def output_schema(self, input_schema, params):
246 return input_schema
247
248 def transform(self, batch, params):
249 raise NotImplementedError
250
251 def process(self, ctx):
252 source = ctx.input('source')
253 output = ctx.records_output('result', self.output_schema(source.schema, ctx.params))
254 for batch in source.records():
255 result = self.transform(batch, ctx.params)
256 if result is not None:
257 output.write(result)