src/script_jsonl_project.py
1"""Select top-level keys from JSONL objects, one bounded line at a time."""
2import json
3from script_support import parameters, lines, infer, preview
4from xstream_operator import run_operator
5
6
7def validate(params):
8 p = parameters(params, ('fields',), ('missing',))
9 fields = p['fields']
10 if (not isinstance(fields, list) or not 1 <= len(fields) <= 128
11 or any(not isinstance(k, str) or not 1 <= len(k) <= 256 for k in fields)
12 or len(set(fields)) != len(fields)):
13 raise ValueError('fields must contain 1–128 unique nonempty keys, up to 256 characters each')
14 if p.get('missing', 'null') not in ('null', 'error'):
15 raise ValueError('missing must be null or error')
16 return p
17
18
19def reject_constant(value):
20 raise ValueError('non-finite JSON constant: ' + value)
21
22
23def unique_object(pairs):
24 result = {}
25 for key, value in pairs:
26 if key in result:
27 raise ValueError('duplicate JSON key: ' + key)
28 result[key] = value
29 return result
30
31
32def process(ctx):
33 p = validate(ctx.params)
34 out = ctx.bytes_output('result', 'jsonl')
35 for line in lines(ctx.input('source')):
36 if not line.strip():
37 continue
38 row = json.loads(line, parse_constant=reject_constant, object_pairs_hook=unique_object)
39 if not isinstance(row, dict):
40 raise ValueError('each JSONL value must be an object')
41 if p.get('missing', 'null') == 'error' and any(k not in row for k in p['fields']):
42 raise ValueError('JSON object is missing a selected field')
43 selected = {k: row.get(k) for k in p['fields']}
44 out.write((json.dumps(selected, ensure_ascii=False, allow_nan=False, separators=(',', ':')) + '\n').encode('utf-8'))
45
46
47def infer_outputs(inputs, params):
48 return infer(inputs, params, validate, format='jsonl')
49
50
51def preview_outputs(inputs, params, limits):
52 return preview(process, inputs, params, limits)
53
54
55if __name__ == '__main__':
56 run_operator(process, infer_outputs, preview_outputs=preview_outputs)