Source code

Download source archive ↓

audit-publication.py

1#!/usr/bin/env python3
2"""Audit public Galaxy manifests, source archives and runnable artifacts."""
3import argparse
4import hashlib
5import io
6import json
7from pathlib import Path
8from urllib.parse import quote
9from urllib.request import urlopen
10import zipfile
11import yaml
12
13HERE = Path(__file__).resolve().parent
14
15
16def main():
17    parser = argparse.ArgumentParser(description=__doc__)
18    parser.add_argument('--url', default='http://127.0.0.1:8820')
19    parser.add_argument('--verification-directory', type=Path, help='Private publisher output parent; match passed artifact tests by SHA-256.')
20    parser.add_argument('--report', type=Path, default=HERE / 'verification-public-catalog.json')
21    parser.add_argument('--expected-count', type=int)
22    args = parser.parse_args()
23    def fetch(path):
24        with urlopen(args.url.rstrip('/') + path, timeout=60) as response:
25            return response.read()
26    def blob(ref):
27        data = fetch('/v1/blobs/sha256/' + ref['sha256'] + '/' + quote(ref['filename']))
28        assert len(data) == ref['size'], ref
29        assert hashlib.sha256(data).hexdigest() == ref['sha256'], ref
30        return data
31    specs = json.loads((HERE / 'catalog.json').read_text())
32    expected = args.expected_count if args.expected_count is not None else len(specs)
33    assert len(specs) == expected
34    public = json.loads(fetch('/v1/operators'))
35    galaxy = {x['name'] for x in public if x['name'].startswith('galaxy_')}
36    assert galaxy == {s['name'] for s in specs.values()}, ('public distinct count', len(galaxy), 'expected', expected)
37    tests = {}
38    if args.verification_directory:
39        for path in args.verification_directory.glob('*/*-test.json'):
40            result = json.loads(path.read_text())
41            if result.get('status') == 'passed':
42                tests[result['artifact_sha256']] = result
43    results = []
44    for name, spec in specs.items():
45        manifest = json.loads(fetch('/v1/operators/' + spec['name'] + '/' + spec['version']))
46        assert manifest['name'] == spec['name'] and manifest['version'] == spec['version']
47        assert manifest['execution']['executor'] == 'jvm'
48        assert manifest['source_public'] is True
49        assert 'Galaxy' in manifest['categories']
50        assert manifest['provenance']['source_revision']
51        doc = manifest['documentation']
52        assert set(doc['inputs']) == set(manifest['inputs'])
53        assert doc['outputs']['result'].strip()
54        assert set(doc['parameters']) == set(manifest['parameters']['properties'])
55        for param in doc['parameters'].values():
56            assert param['description'].strip() and param['help'].strip() and 'example' in param
57        example = yaml.safe_load(doc['example_job'])
58        assert any(s['operator'] == spec['name'] + '@' + spec['version'] for s in example['steps'])
59        artifact = blob(manifest['artifact'])
60        with zipfile.ZipFile(io.BytesIO(artifact)) as jar:
61            entries = set(jar.namelist())
62            assert manifest['execution']['entrypoint'].replace('.', '/') + '.class' in entries
63            assert {'META-INF/LICENSE', 'META-INF/NOTICE', 'META-INF/galaxy-operator.json'} <= entries
64        source = blob(manifest['provenance']['source_archive'])
65        with zipfile.ZipFile(io.BytesIO(source)) as archive:
66            entries = set(archive.namelist())
67            assert {'LICENSE', 'NOTICE', 'README.md', 'README_zh.md'} <= entries
68            assert any(n.startswith('src/') and n.endswith('.scala') for n in entries)
69            assert any(n.startswith('upstream/') for n in entries)
70            evidence = json.loads(archive.read('upstream-manifest.json'))
71            for path, digest in evidence['files'].items():
72                assert hashlib.sha256(archive.read('upstream/' + path)).hexdigest() == digest
73            if 'upstream-extended-manifest.json' in entries:
74                for family, evidence in json.loads(archive.read('upstream-extended-manifest.json')).items():
75                    for path, digest in evidence['files'].items():
76                        assert hashlib.sha256(archive.read('upstream-extended/' + family + '/' + path)).hexdigest() == digest
77            if 'upstream-dependencies-manifest.json' in entries:
78                for family, evidence in json.loads(archive.read('upstream-dependencies-manifest.json')).items():
79                    for path, digest in evidence['files'].items():
80                        assert hashlib.sha256(archive.read('upstream-dependencies/' + family + '/' + path)).hexdigest() == digest
81        if args.verification_directory:
82            assert manifest['artifact']['sha256'] in tests, ('missing artifact tests', spec['name'])
83        results.append({'operator': spec['name'] + '@' + spec['version'],
84                        'artifact_sha256': manifest['artifact']['sha256'],
85                        'source_sha256': manifest['provenance']['source_archive']['sha256'],
86                        'runtime': manifest['runtime']['image'], 'status': 'passed',
87                        'artifact_tests': tests.get(manifest['artifact']['sha256'])})
88        print('PASS public manifest, documentation, source and artifact', spec['name'], flush=True)
89    args.report.write_text(json.dumps({'status': 'passed', 'distinct_operators': len(galaxy),
90                                      'operators': results}, indent=2) + '\n')
91
92
93if __name__ == '__main__':
94    main()