Source code

Download source archive ↓

src/script_jsonl_project.py

1"""Select top-level keys from JSONL objects, one bounded line at a time."""
2import json
3from script_support import parameters, lines, infer, preview
4from xstream_operator import run_operator
5
6
7def validate(params):
8    p = parameters(params, ('fields',), ('missing',))
9    fields = p['fields']
10    if (not isinstance(fields, list) or not 1 <= len(fields) <= 128
11            or any(not isinstance(k, str) or not 1 <= len(k) <= 256 for k in fields)
12            or len(set(fields)) != len(fields)):
13        raise ValueError('fields must contain 1–128 unique nonempty keys, up to 256 characters each')
14    if p.get('missing', 'null') not in ('null', 'error'):
15        raise ValueError('missing must be null or error')
16    return p
17
18
19def reject_constant(value):
20    raise ValueError('non-finite JSON constant: ' + value)
21
22
23def unique_object(pairs):
24    result = {}
25    for key, value in pairs:
26        if key in result:
27            raise ValueError('duplicate JSON key: ' + key)
28        result[key] = value
29    return result
30
31
32def process(ctx):
33    p = validate(ctx.params)
34    out = ctx.bytes_output('result', 'jsonl')
35    for line in lines(ctx.input('source')):
36        if not line.strip():
37            continue
38        row = json.loads(line, parse_constant=reject_constant, object_pairs_hook=unique_object)
39        if not isinstance(row, dict):
40            raise ValueError('each JSONL value must be an object')
41        if p.get('missing', 'null') == 'error' and any(k not in row for k in p['fields']):
42            raise ValueError('JSON object is missing a selected field')
43        selected = {k: row.get(k) for k in p['fields']}
44        out.write((json.dumps(selected, ensure_ascii=False, allow_nan=False, separators=(',', ':')) + '\n').encode('utf-8'))
45
46
47def infer_outputs(inputs, params):
48    return infer(inputs, params, validate, format='jsonl')
49
50
51def preview_outputs(inputs, params, limits):
52    return preview(process, inputs, params, limits)
53
54
55if __name__ == '__main__':
56    run_operator(process, infer_outputs, preview_outputs=preview_outputs)