src/xstream_piflow_mock_data.py
1"""Generate a bounded random record stream; consume the declared input as a trigger."""
2import random
3import string
4import pyarrow as pa
5from dacp_piflow_support import params
6
7
8def process(ctx):
9 p = params(ctx, ['schema', 'count'])
10 if not isinstance(p['schema'], str) or not isinstance(p['count'], str):
11 raise ValueError('schema and count must be strings')
12 count = int(p['count'])
13 if not 0 <= count <= 1000000:
14 raise ValueError('count must be between 0 and 1000000')
15 types = dict(String=pa.string(), Int=pa.int32(), Long=pa.int64(), Float=pa.float32(), Double=pa.float64(), Boolean=pa.bool_())
16 fields, kinds = [], []
17 for definition in p['schema'].split(','):
18 parts = [v.strip() for v in definition.split(':')]
19 if len(parts) not in (2, 3) or not parts[0] or parts[1] not in types or (len(parts)==3 and parts[2] not in ('true','false')):
20 raise ValueError('expected name:Type[:true|false]')
21 fields.append(pa.field(parts[0], types[parts[1]], nullable=len(parts)==3 and parts[2]=='true'))
22 kinds.append(parts[1])
23 if len(set(f.name for f in fields))!=len(fields):
24 raise ValueError('unique field names required')
25 source=ctx.input('source')
26 for _ in source.records():
27 pass
28 rng = random.Random()
29 generators = dict(String=lambda: ''.join(rng.choices(string.ascii_letters+string.digits,k=10)), Int=lambda:rng.randrange(100),
30 Long=lambda:rng.randrange(-(2**63),2**63), Float=rng.random, Double=rng.random, Boolean=lambda:bool(rng.getrandbits(1)))
31 schema=pa.schema(fields)
32 out=ctx.records_output('result',schema)
33 for start in range(0,count,1024):
34 rows=[{f.name:None if f.nullable and rng.getrandbits(1) else generators[k]() for f,k in zip(fields,kinds)} for _ in range(min(1024,count-start))]
35 out.write(pa.RecordBatch.from_pylist(rows,schema=schema))
36
37
38def infer_outputs(inputs, params):
39 import pyarrow as pa
40 from xstream_operator import descriptor, result, records_schema
41 expected = {'source'}
42 if set(inputs) != expected:
43 raise ValueError('unexpected input ports')
44 outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
45 return result(outputs)
46
47
48def preview_outputs(inputs, params, limits):
49 from xstream_operator import preview_process
50 return preview_process(process, inputs, params, limits)
51
52
53if __name__ == '__main__':
54 from xstream_operator import run_operator
55 run_operator(process, infer_outputs, preview_outputs=preview_outputs)