audit-publication.py
1#!/usr/bin/env python3
2"""Audit public Galaxy manifests, source archives and runnable artifacts."""
3import argparse
4import hashlib
5import io
6import json
7from pathlib import Path
8from urllib.parse import quote
9from urllib.request import urlopen
10import zipfile
11import yaml
12
13HERE = Path(__file__).resolve().parent
14
15
16def main():
17 parser = argparse.ArgumentParser(description=__doc__)
18 parser.add_argument('--url', default='http://127.0.0.1:8820')
19 parser.add_argument('--verification-directory', type=Path, help='Private publisher output parent; match passed artifact tests by SHA-256.')
20 parser.add_argument('--report', type=Path, default=HERE / 'verification-public-catalog.json')
21 parser.add_argument('--expected-count', type=int)
22 args = parser.parse_args()
23 def fetch(path):
24 with urlopen(args.url.rstrip('/') + path, timeout=60) as response:
25 return response.read()
26 def blob(ref):
27 data = fetch('/v1/blobs/sha256/' + ref['sha256'] + '/' + quote(ref['filename']))
28 assert len(data) == ref['size'], ref
29 assert hashlib.sha256(data).hexdigest() == ref['sha256'], ref
30 return data
31 specs = json.loads((HERE / 'catalog.json').read_text())
32 expected = args.expected_count if args.expected_count is not None else len(specs)
33 assert len(specs) == expected
34 public = json.loads(fetch('/v1/operators'))
35 galaxy = {x['name'] for x in public if x['name'].startswith('galaxy_')}
36 assert galaxy == {s['name'] for s in specs.values()}, ('public distinct count', len(galaxy), 'expected', expected)
37 tests = {}
38 if args.verification_directory:
39 for path in args.verification_directory.glob('*/*-test.json'):
40 result = json.loads(path.read_text())
41 if result.get('status') == 'passed':
42 tests[result['artifact_sha256']] = result
43 results = []
44 for name, spec in specs.items():
45 manifest = json.loads(fetch('/v1/operators/' + spec['name'] + '/' + spec['version']))
46 assert manifest['name'] == spec['name'] and manifest['version'] == spec['version']
47 assert manifest['execution']['executor'] == 'jvm'
48 assert manifest['source_public'] is True
49 assert 'Galaxy' in manifest['categories']
50 assert manifest['provenance']['source_revision']
51 doc = manifest['documentation']
52 assert set(doc['inputs']) == set(manifest['inputs'])
53 assert doc['outputs']['result'].strip()
54 assert set(doc['parameters']) == set(manifest['parameters']['properties'])
55 for param in doc['parameters'].values():
56 assert param['description'].strip() and param['help'].strip() and 'example' in param
57 example = yaml.safe_load(doc['example_job'])
58 assert any(s['operator'] == spec['name'] + '@' + spec['version'] for s in example['steps'])
59 artifact = blob(manifest['artifact'])
60 with zipfile.ZipFile(io.BytesIO(artifact)) as jar:
61 entries = set(jar.namelist())
62 assert manifest['execution']['entrypoint'].replace('.', '/') + '.class' in entries
63 assert {'META-INF/LICENSE', 'META-INF/NOTICE', 'META-INF/galaxy-operator.json'} <= entries
64 source = blob(manifest['provenance']['source_archive'])
65 with zipfile.ZipFile(io.BytesIO(source)) as archive:
66 entries = set(archive.namelist())
67 assert {'LICENSE', 'NOTICE', 'README.md', 'README_zh.md'} <= entries
68 assert any(n.startswith('src/') and n.endswith('.scala') for n in entries)
69 assert any(n.startswith('upstream/') for n in entries)
70 evidence = json.loads(archive.read('upstream-manifest.json'))
71 for path, digest in evidence['files'].items():
72 assert hashlib.sha256(archive.read('upstream/' + path)).hexdigest() == digest
73 if 'upstream-extended-manifest.json' in entries:
74 for family, evidence in json.loads(archive.read('upstream-extended-manifest.json')).items():
75 for path, digest in evidence['files'].items():
76 assert hashlib.sha256(archive.read('upstream-extended/' + family + '/' + path)).hexdigest() == digest
77 if 'upstream-dependencies-manifest.json' in entries:
78 for family, evidence in json.loads(archive.read('upstream-dependencies-manifest.json')).items():
79 for path, digest in evidence['files'].items():
80 assert hashlib.sha256(archive.read('upstream-dependencies/' + family + '/' + path)).hexdigest() == digest
81 if args.verification_directory:
82 assert manifest['artifact']['sha256'] in tests, ('missing artifact tests', spec['name'])
83 results.append({'operator': spec['name'] + '@' + spec['version'],
84 'artifact_sha256': manifest['artifact']['sha256'],
85 'source_sha256': manifest['provenance']['source_archive']['sha256'],
86 'runtime': manifest['runtime']['image'], 'status': 'passed',
87 'artifact_tests': tests.get(manifest['artifact']['sha256'])})
88 print('PASS public manifest, documentation, source and artifact', spec['name'], flush=True)
89 args.report.write_text(json.dumps({'status': 'passed', 'distinct_operators': len(galaxy),
90 'operators': results}, indent=2) + '\n')
91
92
93if __name__ == '__main__':
94 main()