verify.py
1#!/usr/bin/env python3
2"""Verify Galaxy golden fixtures and real Arrow operator/inference protocols offline."""
3import argparse,base64,hashlib,json,shlex,struct,subprocess,tempfile
4from pathlib import Path
5import pyarrow as pa
6HERE=Path(__file__).resolve().parent
7RUNTIME='sha256:23732819bf6777f314efc0454034fb8509caac9dc81b84538e52104274cb4af3'
8def main():
9 p=argparse.ArgumentParser(description=__doc__);p.add_argument('--docker-prefix',default='sudo -n -u cnic -g docker');p.add_argument('--artifact',type=Path);p.add_argument('--runtime',default=RUNTIME);p.add_argument('--report',type=Path);p.add_argument('--only',choices=list(json.loads((HERE/'catalog.json').read_text())));a=p.parse_args()
10 jar=(a.artifact or HERE/'target/scala-2.13/xstream-galaxy-operators_2.13-1.0.jar').resolve()
11 specs=json.loads((HERE/'catalog.json').read_text());prefix=shlex.split(a.docker_prefix);cases=[];golden_counts={k:0 for k in specs}
12 def golden(op,d,source,expected,params):
13 golden_counts[op]+=1;folder=HERE/'upstream'/d/'test-data';cases.append((op,folder.joinpath(source).read_bytes(),folder.joinpath(expected).read_bytes(),params))
14 golden('change_case','change_case','1.txt','changeCase_out1.tabular',{'columns':'c1','delimiter':'whitespace','casing':'upper'})
15 golden('change_case','change_case','1.bed','changeCase_out2.tabular',{'columns':'c1'})
16 golden('condense_characters','condense_characters','1.bed','eq-condense.dat',{'character':'tab'})
17 golden('convert_delimiters','convert_characters','1.bed','eq-convert.dat',{'character':'whitespace'})
18 golden('convert_delimiters','convert_characters','a.txt','a.tab',{'character':'whitespace'})
19 golden('cut_columns','cut_columns','1.bed','eq-cut.dat',{'columns':'c1,c4,c2,c3'})
20 golden('cut_columns','cut_columns','1.bed','eq-cut.dat',{'columns':'c1,c4,c2-c3'})
21 golden('merge_columns','merge_cols','1.bed','mergeCols.dat',{'columns':'c4,c1,c6'})
22 golden('add_column','add_value','1.bed','eq-addvalue.dat',{'value':'1','iterate':False})
23 golden('fasta_length','fasta_compute_length','454.fasta','fasta_tool_compute_length_1.out',{'keep_first':0,'id_only':False})
24 golden('fasta_length','fasta_compute_length','extract_genomic_dna_out1.fasta','fasta_tool_compute_length_2.out',{'keep_first':0})
25 golden('fasta_length','fasta_compute_length','454.fasta','fasta_tool_compute_length_3.out',{'keep_first':14})
26 golden('fasta_length','fasta_compute_length','merged.fa','merged.tab',{'id_only':True})
27 golden('fasta_filter_length','fasta_filter_by_length','454.fasta','fasta_tool_filter_length_1.out',{'min_length':10,'max_length':0})
28 golden('fasta_filter_length','fasta_filter_by_length','4.fasta','fasta_tool_filter_length_2.out',{'min_length':0,'max_length':60})
29 for source,expected,params in [('454.fasta','fasta_to_tabular_out1.tabular',{}),('4.fasta','fasta_to_tabular_out2.tabular',{}),('454.fasta','fasta_to_tabular_out3.tabular',{'keep_first':14}),('454.fasta','fasta_to_tabular_out4.tabular',{'description_columns':2}),('454.fasta','fasta_to_tabular_out5.tabular',{'description_columns':5})]:
30 golden('fasta_to_tabular','fasta_to_tabular',source,expected,params)
31 golden('remove_beginning','remove_beginning','1.bed','eq-removebeginning.dat',{'lines':5})
32 golden('head','show_beginning','1.bed','eq-showbeginning.dat',{'lines':10})
33 golden('tail','show_tail','1.bed','eq-showtail.dat',{'lines':10})
34 cases += [
35 ('change_case','éa\t好\r\nza\t世'.encode(),'ÉA\t好\nZA\t世\n'.encode(),{}),
36 ('cut_columns',b'\n',b'.\t.\n',{'columns':'c1,c2'}),
37 ('cut_columns',b'#skip\na\tb\nc',b'b\ta\tb\t.\n.\tc\t.\t.\n',{'columns':'c2,c1,c2,c4'}),
38 ('merge_columns',b'#keep\n\na\tb',b'#keep\n\na\tb\tba\n',{'columns':'c2,c8,c1'}),
39 ('condense_characters',b'a...b...\nc',b'a.b.\nc\n',{'character':'dot'}),
40 ('condense_characters',b'a \t b',b'a b\n',{'character':'whitespace'}),
41 ('convert_delimiters',b' a:::b \r\nc',b'a\tb\nc\n',{'character':'colon'}),
42 ('add_column',b'a\r\nb',b'a\t8\nb\t9\n',{'value':'8','iterate':True}),
43 ('add_column',b'a\nb',b'a\tid-0\nb\tid-1\n',{'value':'id','iterate':True})]
44 cases += [
45 ('fasta_length',b'#comment\n>one title\r\nACG\r\nTA\n>two\nNN',b'one\t5\ntwo\t2\n',{'id_only':True}),
46 ('fasta_filter_length',b'>one\nACGT\n>two\nA\n',b'>one\nACGT\n',{'min_length':2,'max_length':5}),
47 ('fasta_filter_length',b'>one\nACGT\n>two\nA',b'>two\nA',{'min_length':0,'max_length':2}),
48 ('fasta_to_tabular',b'#comment\n>one title here\nACG\nTA',b'one\ttitle here\tACGTA\n',{'description_columns':2}),
49 ('fasta_to_tabular',b'>id\nA',b'id\t\t\tA\n',{'description_columns':3}),
50 ('remove_beginning',b'a\r\nb\rc',b'b\rc',{'lines':1}),
51 ('head',b'a\r\nb\nc',b'a\r\nb\n',{'lines':2}),
52 ('tail',b'a\r\nb\nc',b'b\nc',{'lines':2}),
53 ('tail',b'a\nb',b'',{'lines':0})]
54 for op in specs:cases.append((op,b'',b'',{}))
55 if a.only:
56 specs={a.only:specs[a.only]};cases=[c for c in cases if c[0]==a.only]
57 with tempfile.TemporaryDirectory(prefix='xstream-galaxy-verify-') as tmp:
58 root=Path(tmp);root.chmod(0o755)
59 def call(op,verb,d,check=True):
60 command=prefix+['docker','run','--rm','--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--memory','512m','--pids-limit','64','--user','65534:65534','--tmpfs','/tmp:rw,nosuid,nodev,size=32m','-v',str(jar)+':/operator/galaxy.jar:ro','-v',str(root)+':/fixtures:ro','--entrypoint','/opt/java/bin/java',a.runtime,'--add-opens=java.base/java.nio=ALL-UNNAMED','-Dio.netty.tryReflectionSetAccessible=true','-cp','/operator/galaxy.jar:/opt/lib/*',specs[op]['entrypoint'],verb]
61 if verb=='operator':command += ['/fixtures/'+d.name,'/fixtures/'+d.name+'/params.json']
62 else:command += ['/fixtures/'+d.name+'/request.json']
63 r=subprocess.run(command,capture_output=True,timeout=60)
64 if check and r.returncode:raise AssertionError(r.stderr.decode(errors='replace'))
65 return r
66 def fixture(index,data,params):
67 d=root/str(index);d.mkdir();d.chmod(0o755)
68 table=pa.table({'data':pa.array([data[i:i+2] for i in range(0,len(data),2)],type=pa.binary())})
69 with (d/'input.arrow').open('wb') as f:
70 with pa.ipc.new_stream(f,table.schema) as w:w.write_table(table,max_chunksize=2)
71 (d/'manifest.json').write_text(json.dumps([{'port':'source','file':'input.arrow','representation':'bytes','format':'text'}]))
72 (d/'params.json').write_text(json.dumps(params));return d
73 def payload(raw):
74 pos=0;data=bytearray();started=ended=False
75 while pos<len(raw):
76 hn,dn=struct.unpack_from('>ii',raw,pos);pos+=8;h=json.loads(raw[pos:pos+hn]);pos+=hn;chunk=raw[pos:pos+dn];pos+=dn
77 assert h['port']=='result'
78 if h['type']=='start':assert not started and h['representation']=='bytes';started=True
79 elif h['type']=='data':assert started and not ended;data.extend(chunk)
80 elif h['type']=='end':assert started and not ended;ended=True
81 else:raise AssertionError(h)
82 assert started and ended;return bytes(data)
83 for i,(op,data,expected,params) in enumerate(cases):
84 d=fixture(i,data,params);actual=payload(call(op,'operator',d).stdout)
85 assert actual==expected,(op,i,actual[:500],expected[:500]);print('PASS execution',op,i,flush=True)
86 for i,(op,s) in enumerate(specs.items()):
87 d=fixture('infer-'+str(i),b'',s['params'])
88 sample_data=(b'>seq example\nACGTACGTACGT\n'*20) if op.startswith('fasta_') else (b'hello\tworld\n'*200)
89 sample={'representation':'bytes','format':'text','data_base64':base64.b64encode(sample_data).decode(),'truncated':False}
90 request={'inputs':{'source':{'representations':['bytes'],'formats':['text'],'certainty':'known','sample':sample}},'params':s['params'],'preview':{'max_rows':10,'max_bytes':256}}
91 (d/'request.json').write_text(json.dumps(request));out=json.loads(call(op,'infer',d).stdout);desc=out['outputs']['result'];assert desc['representations']==['bytes'];assert len(json.dumps(desc['sample'],separators=(',',':')).encode())<=256;print('PASS inference/preview',op,flush=True)
92 bad={'columns':'c0'} if op in ['change_case','cut_columns','merge_columns'] else {'value':'a\nb'} if op=='add_column' else {'character':'(?evil)'}
93 request['params']=bad;(d/'request.json').write_text(json.dumps(request));assert call(op,'infer',d,False).returncode!=0
94 for index,data,params in ([('huge',b'x'*1048577,{}),('utf8',b'\xff',{})] if 'add_column' in specs else []):
95 d=fixture(index,data,params);assert call('add_column','operator',d,False).returncode!=0
96 if not a.only:
97 # A three-step bytes pipeline through the real IPC protocol, including a split Unicode character.
98 data='éa,,,b\nxy,,z'.encode()
99 for i,(op,params) in enumerate([('convert_delimiters',{'character':'comma'}),('change_case',{}),('add_column',{'value':'row','iterate':True})]):
100 d=fixture('pipeline-'+str(i),data,params);data=payload(call(op,'operator',d).stdout)
101 assert data=='ÉA\tb\trow-0\nXY\tz\trow-1\n'.encode()
102 report={'artifact_sha256':hashlib.sha256(jar.read_bytes()).hexdigest(),'runtime':a.runtime,'upstream_golden_cases':sum(golden_counts[k] for k in specs),'execution_cases':len(cases),'inference_preview_cases':len(specs),'invalid_parameter_cases':len(specs),'resource_encoding_cases':2 if 'add_column' in specs else 0,'pipeline':'passed' if not a.only else 'not requested','status':'passed'}
103 if a.report:a.report.parent.mkdir(parents=True,exist_ok=True);a.report.write_text(json.dumps(report,indent=2)+'\n')
104 print(json.dumps(report),flush=True)
105if __name__=='__main__':main()