Source code

Download source archive ↓

audit-publication.py

1#!/usr/bin/env python3
2"""Audit all public public Galaxy manifests, source archives and runnable artifacts."""
3import argparse
4import hashlib
5import io
6import json
7from pathlib import Path
8from urllib.parse import quote
9from urllib.request import urlopen
10import zipfile
11import yaml
12
13HERE = Path(__file__).resolve().parent
14
15
16def main():
17    parser = argparse.ArgumentParser(description=__doc__)
18    parser.add_argument('--url', default='http://127.0.0.1:8820')
19    parser.add_argument('--verification-directory', type=Path, help='Private publisher output parent; match passed artifact tests by SHA-256.')
20    parser.add_argument('--report', type=Path, default=HERE / 'verification-public-catalog.json')
21    args = parser.parse_args()
22    def fetch(path):
23        with urlopen(args.url.rstrip('/') + path, timeout=60) as response:
24            return response.read()
25    def blob(ref):
26        data = fetch('/v1/blobs/sha256/' + ref['sha256'] + '/' + quote(ref['filename']))
27        assert len(data) == ref['size'], ref
28        assert hashlib.sha256(data).hexdigest() == ref['sha256'], ref
29        return data
30    specs = json.loads((HERE / 'catalog.json').read_text())
31    assert len(specs) == 100
32    public = json.loads(fetch('/v1/operators'))
33    galaxy = {x['name'] for x in public if x['name'].startswith('galaxy_')}
34    assert len(galaxy) == 100, ('public distinct count', len(galaxy))
35    tests = {}
36    if args.verification_directory:
37        for path in args.verification_directory.glob('*/*-test.json'):
38            result = json.loads(path.read_text())
39            if result.get('status') == 'passed':
40                tests[result['artifact_sha256']] = result
41    results = []
42    for name, spec in specs.items():
43        manifest = json.loads(fetch('/v1/operators/' + spec['name'] + '/' + spec['version']))
44        assert manifest['name'] == spec['name'] and manifest['version'] == spec['version']
45        assert manifest['execution']['executor'] == 'jvm'
46        assert manifest['source_public'] is True
47        assert 'Galaxy' in manifest['categories']
48        assert manifest['provenance']['source_revision']
49        doc = manifest['documentation']
50        assert set(doc['inputs']) == set(manifest['inputs'])
51        assert doc['outputs']['result'].strip()
52        assert set(doc['parameters']) == set(manifest['parameters']['properties'])
53        for param in doc['parameters'].values():
54            assert param['description'].strip() and param['help'].strip() and 'example' in param
55        example = yaml.safe_load(doc['example_job'])
56        assert any(s['operator'] == spec['name'] + '@' + spec['version'] for s in example['steps'])
57        artifact = blob(manifest['artifact'])
58        with zipfile.ZipFile(io.BytesIO(artifact)) as jar:
59            entries = set(jar.namelist())
60            assert manifest['execution']['entrypoint'].replace('.', '/') + '.class' in entries
61            assert {'META-INF/LICENSE', 'META-INF/NOTICE', 'META-INF/galaxy-operator.json'} <= entries
62        source = blob(manifest['provenance']['source_archive'])
63        with zipfile.ZipFile(io.BytesIO(source)) as archive:
64            entries = set(archive.namelist())
65            assert {'LICENSE', 'NOTICE', 'README.md', 'README_zh.md'} <= entries
66            assert any(n.startswith('src/') and n.endswith('.scala') for n in entries)
67            assert any(n.startswith('upstream/') for n in entries)
68            evidence = json.loads(archive.read('upstream-manifest.json'))
69            for path, digest in evidence['files'].items():
70                assert hashlib.sha256(archive.read('upstream/' + path)).hexdigest() == digest
71            if 'upstream-extended-manifest.json' in entries:
72                for family, evidence in json.loads(archive.read('upstream-extended-manifest.json')).items():
73                    for path, digest in evidence['files'].items():
74                        assert hashlib.sha256(archive.read('upstream-extended/' + family + '/' + path)).hexdigest() == digest
75            if 'upstream-dependencies-manifest.json' in entries:
76                for family, evidence in json.loads(archive.read('upstream-dependencies-manifest.json')).items():
77                    for path, digest in evidence['files'].items():
78                        assert hashlib.sha256(archive.read('upstream-dependencies/' + family + '/' + path)).hexdigest() == digest
79        if args.verification_directory:
80            assert manifest['artifact']['sha256'] in tests, ('missing artifact tests', spec['name'])
81        results.append({'operator': spec['name'] + '@' + spec['version'],
82                        'artifact_sha256': manifest['artifact']['sha256'],
83                        'source_sha256': manifest['provenance']['source_archive']['sha256'],
84                        'runtime': manifest['runtime']['image'], 'status': 'passed',
85                        'artifact_tests': tests.get(manifest['artifact']['sha256'])})
86        print('PASS public manifest, documentation, source and artifact', spec['name'], flush=True)
87    args.report.write_text(json.dumps({'status': 'passed', 'distinct_operators': len(galaxy),
88                                      'operators': results}, indent=2) + '\n')
89
90
91if __name__ == '__main__':
92    main()