Source code

Download source archive ↓

verify.py

1#!/usr/bin/env python3
2"""Verify Galaxy golden fixtures and real Arrow operator/inference protocols offline."""
3import argparse,base64,hashlib,json,shlex,struct,subprocess,tempfile
4from pathlib import Path
5import pyarrow as pa
6HERE=Path(__file__).resolve().parent
7RUNTIME='sha256:23732819bf6777f314efc0454034fb8509caac9dc81b84538e52104274cb4af3'
8def main():
9 p=argparse.ArgumentParser(description=__doc__);p.add_argument('--docker-prefix',default='sudo -n -u cnic -g docker');p.add_argument('--artifact',type=Path);p.add_argument('--runtime',default=RUNTIME);p.add_argument('--report',type=Path);p.add_argument('--only',choices=list(json.loads((HERE/'catalog.json').read_text())));a=p.parse_args()
10 jar=(a.artifact or HERE/'target/scala-2.13/xstream-galaxy-operators_2.13-1.0.jar').resolve()
11 specs=json.loads((HERE/'catalog.json').read_text());prefix=shlex.split(a.docker_prefix);cases=[];golden_counts={k:0 for k in specs}
12 def golden(op,d,source,expected,params):
13  golden_counts[op]+=1;folder=HERE/'upstream'/d/'test-data';cases.append((op,folder.joinpath(source).read_bytes(),folder.joinpath(expected).read_bytes(),params))
14 golden('change_case','change_case','1.txt','changeCase_out1.tabular',{'columns':'c1','delimiter':'whitespace','casing':'upper'})
15 golden('change_case','change_case','1.bed','changeCase_out2.tabular',{'columns':'c1'})
16 golden('condense_characters','condense_characters','1.bed','eq-condense.dat',{'character':'tab'})
17 golden('convert_delimiters','convert_characters','1.bed','eq-convert.dat',{'character':'whitespace'})
18 golden('convert_delimiters','convert_characters','a.txt','a.tab',{'character':'whitespace'})
19 golden('cut_columns','cut_columns','1.bed','eq-cut.dat',{'columns':'c1,c4,c2,c3'})
20 golden('cut_columns','cut_columns','1.bed','eq-cut.dat',{'columns':'c1,c4,c2-c3'})
21 golden('merge_columns','merge_cols','1.bed','mergeCols.dat',{'columns':'c4,c1,c6'})
22 golden('add_column','add_value','1.bed','eq-addvalue.dat',{'value':'1','iterate':False})
23 golden('fasta_length','fasta_compute_length','454.fasta','fasta_tool_compute_length_1.out',{'keep_first':0,'id_only':False})
24 golden('fasta_length','fasta_compute_length','extract_genomic_dna_out1.fasta','fasta_tool_compute_length_2.out',{'keep_first':0})
25 golden('fasta_length','fasta_compute_length','454.fasta','fasta_tool_compute_length_3.out',{'keep_first':14})
26 golden('fasta_length','fasta_compute_length','merged.fa','merged.tab',{'id_only':True})
27 golden('fasta_filter_length','fasta_filter_by_length','454.fasta','fasta_tool_filter_length_1.out',{'min_length':10,'max_length':0})
28 golden('fasta_filter_length','fasta_filter_by_length','4.fasta','fasta_tool_filter_length_2.out',{'min_length':0,'max_length':60})
29 for source,expected,params in [('454.fasta','fasta_to_tabular_out1.tabular',{}),('4.fasta','fasta_to_tabular_out2.tabular',{}),('454.fasta','fasta_to_tabular_out3.tabular',{'keep_first':14}),('454.fasta','fasta_to_tabular_out4.tabular',{'description_columns':2}),('454.fasta','fasta_to_tabular_out5.tabular',{'description_columns':5})]:
30  golden('fasta_to_tabular','fasta_to_tabular',source,expected,params)
31 golden('remove_beginning','remove_beginning','1.bed','eq-removebeginning.dat',{'lines':5})
32 golden('head','show_beginning','1.bed','eq-showbeginning.dat',{'lines':10})
33 golden('tail','show_tail','1.bed','eq-showtail.dat',{'lines':10})
34 golden('tabular_to_fasta','tabular_to_fasta','solexa.tabular','tabular_to_fasta_out1.fasta',{'title_columns':'1,2,3,4','sequence_column':5})
35 golden('sam_filter_flags','sam_bitwise_flag_filter','sam_bw_filter.sam','sam_bw_filter_0002-yes.sam',{'require_flags':2})
36 # Original interval expectations contain a CIGAR padding bug: P does not consume the reference.
37 for source,target,params in [('sam_bioinf_example.sam','sam2interval_printAll.dat',{'print_all':True}),('sam_bioinf_example.sam','sam2interval_noprintAll.dat',{}),('sam2interval-test3.sam','sam2interval_with_unmapped_reads_noprintAll.dat',{})]:
38  golden('sam_to_interval','sam2interval',source,target,params)
39  op,data,expected,params=cases[-1]
40  expected=expected.replace(b'ref\t8\t19\t+',b'ref\t8\t18\t+')
41  cases[-1]=(op,data,expected,params)
42 golden('trim','trimmer','trimmer_tab_delimited.dat','trimmer_a_f_c0_s1_e13_i62.dat',{'column':0,'start':1,'end':13,'ignore_prefixes':'>'})
43 golden('trim','trimmer','trimmer_tab_delimited.dat','trimmer_a_f_c2_s1_e2_i62.dat',{'column':2,'start':1,'end':2,'ignore_prefixes':'>'})
44 golden('make_windows','windowsplitter','4.bed','4_windows.bed',{'window_size':5000,'offset':4000})
45 for target,params in [('test_extract_pass_filter_quality_9_DP_2000_lt.vcf',{'pass_only':True}),('test_extract_region_80000_100000.vcf',{'region':'20:80000..100000'}),('test_extract_quality_90_ge.vcf',{'quality':90,'quality_operator':'ge'}),('test_extract_keep_info_TV.vcf',{'keep_info':'TV'}),('test_extract_discard_info_TV.vcf',{'discard_info':'TV'})]:
46  golden('vcf_extract','vcf_extract','test_filter_quality_9_DP_2000_lt.vcf' if params.get('pass_only') else 'test.small.vcf',target,params)
47  op,data,expected,params=cases[-1]
48  # Preserve original headers exactly; the Python tool reorders metadata and injects its own provenance.
49  headers=b''.join(line+b'\n' for line in data.splitlines() if line.startswith(b'#'))
50  records=b''.join(line+b'\n' for line in expected.splitlines() if line and not line.startswith(b'#'))
51  cases[-1]=(op,data,headers+records,params)
52 cases += [
53 ('change_case','éa\t好\r\nza\t世'.encode(),'ÉA\t好\nZA\t世\n'.encode(),{}),
54 ('cut_columns',b'\n',b'.\t.\n',{'columns':'c1,c2'}),
55 ('cut_columns',b'#skip\na\tb\nc',b'b\ta\tb\t.\n.\tc\t.\t.\n',{'columns':'c2,c1,c2,c4'}),
56 ('merge_columns',b'#keep\n\na\tb',b'#keep\n\na\tb\tba\n',{'columns':'c2,c8,c1'}),
57 ('condense_characters',b'a...b...\nc',b'a.b.\nc\n',{'character':'dot'}),
58 ('condense_characters',b'a \t b',b'a b\n',{'character':'whitespace'}),
59 ('convert_delimiters',b'  a:::b  \r\nc',b'a\tb\nc\n',{'character':'colon'}),
60 ('add_column',b'a\r\nb',b'a\t8\nb\t9\n',{'value':'8','iterate':True}),
61 ('add_column',b'a\nb',b'a\tid-0\nb\tid-1\n',{'value':'id','iterate':True})]
62 cases += [
63 ('fasta_length',b'#comment\n>one title\r\nACG\r\nTA\n>two\nNN',b'one\t5\ntwo\t2\n',{'id_only':True}),
64 ('fasta_filter_length',b'>one\nACGT\n>two\nA\n',b'>one\nACGT\n',{'min_length':2,'max_length':5}),
65 ('fasta_filter_length',b'>one\nACGT\n>two\nA',b'>two\nA',{'min_length':0,'max_length':2}),
66 ('fasta_to_tabular',b'#comment\n>one title here\nACG\nTA',b'one\ttitle here\tACGTA\n',{'description_columns':2}),
67 ('fasta_to_tabular',b'>id\nA',b'id\t\t\tA\n',{'description_columns':3}),
68 ('remove_beginning',b'a\r\nb\rc',b'b\rc',{'lines':1}),
69 ('head',b'a\r\nb\nc',b'a\r\nb\n',{'lines':2}),
70 ('tail',b'a\r\nb\nc',b'b\nc',{'lines':2}),
71 ('tail',b'a\nb',b'',{'lines':0})]
72 cases += [
73 ('tabular_to_fasta',b'#comment\n>id\tACGT\r\nbad\nnext\tNN',b'>id\nACGT\n>next\nNN\n',{}),
74 ('tabular_to_fasta','α\tβ\tACGT'.encode(),'>α_β\nACGT\n'.encode(),{'title_columns':'1,2','sequence_column':3}),
75 ('sam_filter_flags',b'@HD\tVN:1.6\nr1\t18\tchr1\t10\t20\t2=1X1P2D3N1I\t*\t0\t0\tAAAA\t*\nr2\t4\t*\t0\t0\t*\t*\t0\t0\tN\t*',b'r1\t18\tchr1\t10\t20\t2=1X1P2D3N1I\t*\t0\t0\tAAAA\t*\n',{'require_flags':2,'exclude_flags':4}),
76 ('sam_to_interval',b'r1\t16\tchr1\t10\t20\t2=1X1P2D3N1I\t*\t0\t0\tAAAA\t*\nr2\t4\t*\t0\t0\t*\t*\t0\t0\tN\t*',b'#chrom\tstart\tend\tstrand\tread_name\nchr1\t9\t17\t-\tr1\n',{})]
77 cases += [
78 ('trim','α😀βγ\tkeep\r\n>header\n'.encode(),'😀β\tkeep\n>header\n'.encode(),{'column':1,'start':2,'end':3,'ignore_prefixes':'>'}),
79 ('trim',b'@id\nACGT\n+\nIIII\n',b'@id\nCG\n+\nII\n',{'start':2,'end':3,'fastq':True}),
80 ('trim',b'\na\r\nbc',b'a\nbc\n',{}),
81 ('make_windows',b'chr1\t0\t3\nchr1\t5\t15\tgene',b'chr1\t5\t9\tgene\nchr1\t7\t11\tgene\nchr1\t9\t13\tgene\nchr1\t11\t15\tgene\n',{'window_size':4,'offset':2}),
82 ('make_windows',b'chr1\t0\t6',b'chr1\t0\t4\n',{'window_size':4,'offset':10}),
83 ('make_windows',b'chr1\t0\t10',b'chr1\t0\t4\nchr1\t4\t8\n',{'window_size':4}),
84 ('vcf_extract',b'#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n20\t10\t.\tA\tG\t.\tPASS\tTV\n20\t11\t.\tA\tG\t90\tPASS\tTV=1;DP=2',b'#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n20\t11\t.\tA\tG\t90\tPASS\tTV=1;DP=2\n',{'quality':90,'quality_operator':'ge','keep_info':'TV','pass_only':True}),
85 ('vcf_extract',b'20\t10\t.\tA\tG\t90\tPASS\tTV\n20\t11\t.\tA\tG\t95\t.\t.',b'20\t10\t.\tA\tG\t90\tPASS\tTV\n',{'region':'20:10..10'})]
86 for op in specs:cases.append((op,b'',b'#chrom\tstart\tend\tstrand\tread_name\n' if op=='sam_to_interval' else b'',{}))
87 if a.only:
88  specs={a.only:specs[a.only]};cases=[c for c in cases if c[0]==a.only]
89 with tempfile.TemporaryDirectory(prefix='xstream-galaxy-verify-') as tmp:
90  root=Path(tmp);root.chmod(0o755)
91  def call(op,verb,d,check=True):
92   command=prefix+['docker','run','--rm','--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--memory','512m','--pids-limit','64','--user','65534:65534','--tmpfs','/tmp:rw,nosuid,nodev,size=32m','-v',str(jar)+':/operator/galaxy.jar:ro','-v',str(root)+':/fixtures:ro','--entrypoint','/opt/java/bin/java',a.runtime,'--add-opens=java.base/java.nio=ALL-UNNAMED','-Dio.netty.tryReflectionSetAccessible=true','-cp','/operator/galaxy.jar:/opt/lib/*',specs[op]['entrypoint'],verb]
93   if verb=='operator':command += ['/fixtures/'+d.name,'/fixtures/'+d.name+'/params.json']
94   else:command += ['/fixtures/'+d.name+'/request.json']
95   r=subprocess.run(command,capture_output=True,timeout=60)
96   if check and r.returncode:raise AssertionError(r.stderr.decode(errors='replace'))
97   return r
98  def fixture(index,data,params):
99   d=root/str(index);d.mkdir();d.chmod(0o755)
100   table=pa.table({'data':pa.array([data[i:i+2] for i in range(0,len(data),2)],type=pa.binary())})
101   with (d/'input.arrow').open('wb') as f:
102    with pa.ipc.new_stream(f,table.schema) as w:w.write_table(table,max_chunksize=2)
103   (d/'manifest.json').write_text(json.dumps([{'port':'source','file':'input.arrow','representation':'bytes','format':'text'}]))
104   (d/'params.json').write_text(json.dumps(params));return d
105  def payload(raw):
106   pos=0;data=bytearray();started=ended=False
107   while pos<len(raw):
108    hn,dn=struct.unpack_from('>ii',raw,pos);pos+=8;h=json.loads(raw[pos:pos+hn]);pos+=hn;chunk=raw[pos:pos+dn];pos+=dn
109    assert h['port']=='result'
110    if h['type']=='start':assert not started and h['representation']=='bytes';started=True
111    elif h['type']=='data':assert started and not ended;data.extend(chunk)
112    elif h['type']=='end':assert started and not ended;ended=True
113    else:raise AssertionError(h)
114   assert started and ended;return bytes(data)
115  for i,(op,data,expected,params) in enumerate(cases):
116   d=fixture(i,data,params);actual=payload(call(op,'operator',d).stdout)
117   assert actual==expected,(op,i,actual[:500],expected[:500]);print('PASS execution',op,i,flush=True)
118  for i,(op,s) in enumerate(specs.items()):
119   d=fixture('infer-'+str(i),b'',s['params'])
120   sample_data=(b'>seq example\nACGTACGTACGT\n'*20) if op.startswith('fasta_') else (b'hello\tworld\n'*200)
121   if op=='make_windows':sample_data=b'chr1\t0\t20\n'
122   if op=='vcf_extract':sample_data=b'#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n20\t90000\t.\tA\tG\t100\tPASS\tTV\n'
123   if op.startswith('sam_'):sample_data=b'@HD\tVN:1.6\nr1\t2\tchr1\t10\t20\t4M\t*\t0\t0\tACGT\t*\n'
124   sample={'representation':'bytes','format':'text','data_base64':base64.b64encode(sample_data).decode(),'truncated':False}
125   request={'inputs':{'source':{'representations':['bytes'],'formats':['text'],'certainty':'known','sample':sample}},'params':s['params'],'preview':{'max_rows':10,'max_bytes':256}}
126   (d/'request.json').write_text(json.dumps(request));out=json.loads(call(op,'infer',d).stdout);desc=out['outputs']['result'];assert desc['representations']==['bytes'];assert len(json.dumps(desc['sample'],separators=(',',':')).encode())<=256;print('PASS inference/preview',op,flush=True)
127   bad={'columns':'c0'} if op in ['change_case','cut_columns','merge_columns'] else {'value':'a\nb'} if op=='add_column' else {'character':'(?evil)'}
128   request['params']=bad;(d/'request.json').write_text(json.dumps(request));assert call(op,'infer',d,False).returncode!=0
129  for index,data,params in ([('huge',b'x'*1048577,{}),('utf8',b'\xff',{})] if 'add_column' in specs else []):
130   d=fixture(index,data,params);assert call('add_column','operator',d,False).returncode!=0
131  if not a.only:
132   # A three-step bytes pipeline through the real IPC protocol, including a split Unicode character.
133   data='éa,,,b\nxy,,z'.encode()
134   for i,(op,params) in enumerate([('convert_delimiters',{'character':'comma'}),('change_case',{}),('add_column',{'value':'row','iterate':True})]):
135    d=fixture('pipeline-'+str(i),data,params);data=payload(call(op,'operator',d).stdout)
136   assert data=='ÉA\tb\trow-0\nXY\tz\trow-1\n'.encode()
137 report={'artifact_sha256':hashlib.sha256(jar.read_bytes()).hexdigest(),'runtime':a.runtime,'upstream_golden_cases':sum(golden_counts[k] for k in specs),'corrected_upstream_interval_cases':2 if 'sam_to_interval' in specs else 0,'preserved_header_vcf_fixture_cases':5 if 'vcf_extract' in specs else 0,'execution_cases':len(cases),'inference_preview_cases':len(specs),'invalid_parameter_cases':len(specs),'resource_encoding_cases':2 if 'add_column' in specs else 0,'pipeline':'passed' if not a.only else 'not requested','status':'passed'}
138 if a.report:a.report.parent.mkdir(parents=True,exist_ok=True);a.report.write_text(json.dumps(report,indent=2)+'\n')
139 print(json.dumps(report),flush=True)
140if __name__=='__main__':main()