Source code

Download source archive ↓

src/script_records_to_jsonl.py

1"""Serialize record batches as bounded UTF-8 JSONL rows."""
2import base64
3import datetime
4import decimal
5import json
6from script_support import parameters, infer, preview, MAX_LINE_BYTES
7from xstream_operator import run_operator
8
9
10def validate(params):
11    return parameters(params)
12
13
14def json_value(value):
15    if isinstance(value, (datetime.date, datetime.time, datetime.datetime)):
16        return value.isoformat()
17    if isinstance(value, decimal.Decimal):
18        return str(value)
19    if isinstance(value, bytes):
20        return base64.b64encode(value).decode('ascii')
21    raise TypeError('unsupported JSON value: ' + type(value).__name__)
22
23
24def process(ctx):
25    validate(ctx.params)
26    out = ctx.bytes_output('result', 'jsonl')
27    source = ctx.input('source')
28    if len(source.schema.names) != len(set(source.schema.names)):
29        raise ValueError('record field names must be unique')
30    for batch in source.records():
31        for index in range(batch.num_rows):
32            row = {field.name: batch.column(i)[index].as_py() for i, field in enumerate(batch.schema)}
33            # Incremental encoding enforces an output-row bound before joining chunks.
34            chunks, size = [], 1
35            encoder = json.JSONEncoder(ensure_ascii=False, allow_nan=False, separators=(',', ':'), default=json_value)
36            for part in encoder.iterencode(row):
37                value = part.encode('utf-8')
38                size += len(value)
39                if size > MAX_LINE_BYTES:
40                    raise ValueError('JSONL output row exceeds 1 MiB including newline')
41                chunks.append(value)
42            out.write(b''.join(chunks) + b'\n')
43
44
45def infer_outputs(inputs, params):
46    return infer(inputs, params, validate, representation='records', format='jsonl')
47
48
49def preview_outputs(inputs, params, limits):
50    return preview(process, inputs, params, limits)
51
52
53if __name__ == '__main__':
54    run_operator(process, infer_outputs, preview_outputs=preview_outputs)