Source code

Download source archive ↓

src/script_support.py

1"""Bounded UTF-8 line processing and mandatory inference/preview contracts."""
2from xstream_operator import descriptor, result, preview_process
3
4MAX_LINE_BYTES = 1024 * 1024
5
6
7def parameters(params, required=(), optional=()):
8    if not isinstance(params, dict) or set(params) - set(required) - set(optional) or set(required) - set(params):
9        raise ValueError('missing or unexpected parameters')
10    return params
11
12
13def lines(source):
14    """Yield UTF-8 lines including their ending; tolerate arbitrary stream chunking."""
15    pending = bytearray()
16    for block in source.bytes():
17        start = 0
18        while start < len(block):
19            end = block.find(b'\n', start)
20            stop = len(block) if end < 0 else end + 1
21            if len(pending) + stop - start > MAX_LINE_BYTES:
22                raise ValueError('line exceeds 1 MiB including its line ending')
23            pending.extend(block[start:stop])
24            start = stop
25            if end >= 0:
26                yield bytes(pending).decode('utf-8')
27                pending.clear()
28    if pending:
29        yield bytes(pending).decode('utf-8')
30
31
32def infer(inputs, params, validate, representation='bytes', format='text'):
33    if set(inputs) != {'source'} or representation not in inputs['source']['representations']:
34        raise ValueError('expected source ' + representation + ' input')
35    validate(params)
36    return result({'result': descriptor(['bytes'], [format], certainty='known')})
37
38
39def preview(process, inputs, params, limits):
40    return preview_process(process, inputs, params, limits)