src/script_records_to_jsonl.py
1"""Serialize record batches as bounded UTF-8 JSONL rows."""
2import base64
3import datetime
4import decimal
5import json
6from script_support import parameters, infer, preview, MAX_LINE_BYTES
7from xstream_operator import run_operator
8
9
10def validate(params):
11 return parameters(params)
12
13
14def json_value(value):
15 if isinstance(value, (datetime.date, datetime.time, datetime.datetime)):
16 return value.isoformat()
17 if isinstance(value, decimal.Decimal):
18 return str(value)
19 if isinstance(value, bytes):
20 return base64.b64encode(value).decode('ascii')
21 raise TypeError('unsupported JSON value: ' + type(value).__name__)
22
23
24def process(ctx):
25 validate(ctx.params)
26 out = ctx.bytes_output('result', 'jsonl')
27 source = ctx.input('source')
28 if len(source.schema.names) != len(set(source.schema.names)):
29 raise ValueError('record field names must be unique')
30 for batch in source.records():
31 for index in range(batch.num_rows):
32 row = {field.name: batch.column(i)[index].as_py() for i, field in enumerate(batch.schema)}
33 # Incremental encoding enforces an output-row bound before joining chunks.
34 chunks, size = [], 1
35 encoder = json.JSONEncoder(ensure_ascii=False, allow_nan=False, separators=(',', ':'), default=json_value)
36 for part in encoder.iterencode(row):
37 value = part.encode('utf-8')
38 size += len(value)
39 if size > MAX_LINE_BYTES:
40 raise ValueError('JSONL output row exceeds 1 MiB including newline')
41 chunks.append(value)
42 out.write(b''.join(chunks) + b'\n')
43
44
45def infer_outputs(inputs, params):
46 return infer(inputs, params, validate, representation='records', format='jsonl')
47
48
49def preview_outputs(inputs, params, limits):
50 return preview(process, inputs, params, limits)
51
52
53if __name__ == '__main__':
54 run_operator(process, infer_outputs, preview_outputs=preview_outputs)