audit-publication.py
1#!/usr/bin/env python3
2"""Audit all public public Galaxy manifests, source archives and runnable artifacts."""
3import argparse
4import hashlib
5import io
6import json
7from pathlib import Path
8from urllib.parse import quote
9from urllib.request import urlopen
10import zipfile
11import yaml
12
13HERE = Path(__file__).resolve().parent
14
15
16def main():
17 parser = argparse.ArgumentParser(description=__doc__)
18 parser.add_argument('--url', default='http://127.0.0.1:8820')
19 parser.add_argument('--verification-directory', type=Path, help='Private publisher output parent; match passed artifact tests by SHA-256.')
20 parser.add_argument('--report', type=Path, default=HERE / 'verification-public-catalog.json')
21 parser.add_argument('--expected-count', type=int, default=200)
22 args = parser.parse_args()
23 def fetch(path):
24 with urlopen(args.url.rstrip('/') + path, timeout=60) as response:
25 return response.read()
26 def blob(ref):
27 data = fetch('/v1/blobs/sha256/' + ref['sha256'] + '/' + quote(ref['filename']))
28 assert len(data) == ref['size'], ref
29 assert hashlib.sha256(data).hexdigest() == ref['sha256'], ref
30 return data
31 specs = json.loads((HERE / 'catalog.json').read_text())
32 assert len(specs) == args.expected_count
33 public = json.loads(fetch('/v1/operators'))
34 galaxy = {x['name'] for x in public if x['name'].startswith('galaxy_')}
35 assert len(galaxy) == args.expected_count, ('public distinct count', len(galaxy))
36 tests = {}
37 if args.verification_directory:
38 for path in args.verification_directory.glob('*/*-test.json'):
39 result = json.loads(path.read_text())
40 if result.get('status') == 'passed':
41 tests[result['artifact_sha256']] = result
42 results = []
43 for name, spec in specs.items():
44 manifest = json.loads(fetch('/v1/operators/' + spec['name'] + '/' + spec['version']))
45 assert manifest['name'] == spec['name'] and manifest['version'] == spec['version']
46 assert manifest['execution']['executor'] == 'jvm'
47 assert manifest['source_public'] is True
48 assert 'Galaxy' in manifest['categories']
49 assert manifest['provenance']['source_revision']
50 doc = manifest['documentation']
51 assert set(doc['inputs']) == set(manifest['inputs'])
52 assert doc['outputs']['result'].strip()
53 assert set(doc['parameters']) == set(manifest['parameters']['properties'])
54 for param in doc['parameters'].values():
55 assert param['description'].strip() and param['help'].strip() and 'example' in param
56 example = yaml.safe_load(doc['example_job'])
57 assert any(s['operator'] == spec['name'] + '@' + spec['version'] for s in example['steps'])
58 artifact = blob(manifest['artifact'])
59 with zipfile.ZipFile(io.BytesIO(artifact)) as jar:
60 entries = set(jar.namelist())
61 assert manifest['execution']['entrypoint'].replace('.', '/') + '.class' in entries
62 assert {'META-INF/LICENSE', 'META-INF/NOTICE', 'META-INF/galaxy-operator.json'} <= entries
63 source = blob(manifest['provenance']['source_archive'])
64 with zipfile.ZipFile(io.BytesIO(source)) as archive:
65 entries = set(archive.namelist())
66 assert {'LICENSE', 'NOTICE', 'README.md', 'README_zh.md'} <= entries
67 assert any(n.startswith('src/') and n.endswith('.scala') for n in entries)
68 assert any(n.startswith('upstream/') for n in entries)
69 evidence = json.loads(archive.read('upstream-manifest.json'))
70 for path, digest in evidence['files'].items():
71 assert hashlib.sha256(archive.read('upstream/' + path)).hexdigest() == digest
72 if 'upstream-extended-manifest.json' in entries:
73 for family, evidence in json.loads(archive.read('upstream-extended-manifest.json')).items():
74 for path, digest in evidence['files'].items():
75 assert hashlib.sha256(archive.read('upstream-extended/' + family + '/' + path)).hexdigest() == digest
76 if 'upstream-dependencies-manifest.json' in entries:
77 for family, evidence in json.loads(archive.read('upstream-dependencies-manifest.json')).items():
78 for path, digest in evidence['files'].items():
79 assert hashlib.sha256(archive.read('upstream-dependencies/' + family + '/' + path)).hexdigest() == digest
80 if args.verification_directory:
81 assert manifest['artifact']['sha256'] in tests, ('missing artifact tests', spec['name'])
82 results.append({'operator': spec['name'] + '@' + spec['version'],
83 'artifact_sha256': manifest['artifact']['sha256'],
84 'source_sha256': manifest['provenance']['source_archive']['sha256'],
85 'runtime': manifest['runtime']['image'], 'status': 'passed',
86 'artifact_tests': tests.get(manifest['artifact']['sha256'])})
87 print('PASS public manifest, documentation, source and artifact', spec['name'], flush=True)
88 args.report.write_text(json.dumps({'status': 'passed', 'distinct_operators': len(galaxy),
89 'operators': results}, indent=2) + '\n')
90
91
92if __name__ == '__main__':
93 main()