Source code

Download source archive ↓

src/xstream_piflow_mock_data.py

1"""Generate a bounded random record stream; consume the declared input as a trigger."""
2import random
3import string
4import pyarrow as pa
5from dacp_piflow_support import params
6
7
8def process(ctx):
9    p = params(ctx, ['schema', 'count'])
10    if not isinstance(p['schema'], str) or not isinstance(p['count'], str):
11        raise ValueError('schema and count must be strings')
12    count = int(p['count'])
13    if not 0 <= count <= 1000000:
14        raise ValueError('count must be between 0 and 1000000')
15    types = dict(String=pa.string(), Int=pa.int32(), Long=pa.int64(), Float=pa.float32(), Double=pa.float64(), Boolean=pa.bool_())
16    fields, kinds = [], []
17    for definition in p['schema'].split(','):
18        parts = [v.strip() for v in definition.split(':')]
19        if len(parts) not in (2, 3) or not parts[0] or parts[1] not in types or (len(parts)==3 and parts[2] not in ('true','false')):
20            raise ValueError('expected name:Type[:true|false]')
21        fields.append(pa.field(parts[0], types[parts[1]], nullable=len(parts)==3 and parts[2]=='true'))
22        kinds.append(parts[1])
23    if len(set(f.name for f in fields))!=len(fields):
24        raise ValueError('unique field names required')
25    source=ctx.input('source')
26    for _ in source.records():
27        pass
28    rng = random.Random()
29    generators = dict(String=lambda: ''.join(rng.choices(string.ascii_letters+string.digits,k=10)), Int=lambda:rng.randrange(100),
30                      Long=lambda:rng.randrange(-(2**63),2**63), Float=rng.random, Double=rng.random, Boolean=lambda:bool(rng.getrandbits(1)))
31    schema=pa.schema(fields)
32    out=ctx.records_output('result',schema)
33    for start in range(0,count,1024):
34        rows=[{f.name:None if f.nullable and rng.getrandbits(1) else generators[k]() for f,k in zip(fields,kinds)} for _ in range(min(1024,count-start))]
35        out.write(pa.RecordBatch.from_pylist(rows,schema=schema))
36
37
38def infer_outputs(inputs, params):
39    import pyarrow as pa
40    from xstream_operator import descriptor, result, records_schema
41    expected = {'source'}
42    if set(inputs) != expected:
43        raise ValueError('unexpected input ports')
44    outputs = {'result': {'representations': ['records'], 'formats': ['arrow'], 'certainty': 'unknown'}}
45    return result(outputs)
46
47
48def preview_outputs(inputs, params, limits):
49    from xstream_operator import preview_process
50    return preview_process(process, inputs, params, limits)
51
52
53if __name__ == '__main__':
54    from xstream_operator import run_operator
55    run_operator(process, infer_outputs, preview_outputs=preview_outputs)