Source code

Download source archive ↓

verify.py

1#!/usr/bin/env python3
2"""Verify Galaxy golden fixtures and real Arrow operator/inference protocols offline."""
3import argparse,base64,hashlib,json,shlex,struct,subprocess,tempfile
4from pathlib import Path
5import pyarrow as pa
6HERE=Path(__file__).resolve().parent
7RUNTIME='sha256:23732819bf6777f314efc0454034fb8509caac9dc81b84538e52104274cb4af3'
8def main():
9 p=argparse.ArgumentParser(description=__doc__);p.add_argument('--docker-prefix',default='sudo -n -u cnic -g docker');p.add_argument('--artifact',type=Path);p.add_argument('--runtime',default=RUNTIME);p.add_argument('--report',type=Path);p.add_argument('--only',choices=['change_case','condense_characters','convert_delimiters','cut_columns','merge_columns','add_column']);a=p.parse_args()
10 jar=(a.artifact or HERE/'target/scala-2.13/xstream-galaxy-operators_2.13-1.0.jar').resolve()
11 specs=json.loads((HERE/'catalog.json').read_text());prefix=shlex.split(a.docker_prefix);cases=[]
12 def golden(op,d,source,expected,params):
13  folder=HERE/'upstream'/d/'test-data';cases.append((op,folder.joinpath(source).read_bytes(),folder.joinpath(expected).read_bytes(),params))
14 golden('change_case','change_case','1.txt','changeCase_out1.tabular',{'columns':'c1','delimiter':'whitespace','casing':'upper'})
15 golden('change_case','change_case','1.bed','changeCase_out2.tabular',{'columns':'c1'})
16 golden('condense_characters','condense_characters','1.bed','eq-condense.dat',{'character':'tab'})
17 golden('convert_delimiters','convert_characters','1.bed','eq-convert.dat',{'character':'whitespace'})
18 golden('convert_delimiters','convert_characters','a.txt','a.tab',{'character':'whitespace'})
19 golden('cut_columns','cut_columns','1.bed','eq-cut.dat',{'columns':'c1,c4,c2,c3'})
20 golden('cut_columns','cut_columns','1.bed','eq-cut.dat',{'columns':'c1,c4,c2-c3'})
21 golden('merge_columns','merge_cols','1.bed','mergeCols.dat',{'columns':'c4,c1,c6'})
22 golden('add_column','add_value','1.bed','eq-addvalue.dat',{'value':'1','iterate':False})
23 cases += [
24 ('change_case','éa\t好\r\nza\t世'.encode(),'ÉA\t好\nZA\t世\n'.encode(),{}),
25 ('cut_columns',b'\n',b'.\t.\n',{'columns':'c1,c2'}),
26 ('cut_columns',b'#skip\na\tb\nc',b'b\ta\tb\t.\n.\tc\t.\t.\n',{'columns':'c2,c1,c2,c4'}),
27 ('merge_columns',b'#keep\n\na\tb',b'#keep\n\na\tb\tba\n',{'columns':'c2,c8,c1'}),
28 ('condense_characters',b'a...b...\nc',b'a.b.\nc\n',{'character':'dot'}),
29 ('condense_characters',b'a \t b',b'a b\n',{'character':'whitespace'}),
30 ('convert_delimiters',b'  a:::b  \r\nc',b'a\tb\nc\n',{'character':'colon'}),
31 ('add_column',b'a\r\nb',b'a\t8\nb\t9\n',{'value':'8','iterate':True}),
32 ('add_column',b'a\nb',b'a\tid-0\nb\tid-1\n',{'value':'id','iterate':True})]
33 for op in specs:cases.append((op,b'',b'',{}))
34 if a.only:
35  specs={a.only:specs[a.only]};cases=[c for c in cases if c[0]==a.only]
36 with tempfile.TemporaryDirectory(prefix='xstream-galaxy-verify-') as tmp:
37  root=Path(tmp);root.chmod(0o755)
38  def call(op,verb,d,check=True):
39   command=prefix+['docker','run','--rm','--network','none','--read-only','--cap-drop','ALL','--security-opt','no-new-privileges','--memory','512m','--pids-limit','64','--user','65534:65534','--tmpfs','/tmp:rw,nosuid,nodev,size=32m','-v',str(jar)+':/operator/galaxy.jar:ro','-v',str(root)+':/fixtures:ro','--entrypoint','/opt/java/bin/java',a.runtime,'--add-opens=java.base/java.nio=ALL-UNNAMED','-Dio.netty.tryReflectionSetAccessible=true','-cp','/operator/galaxy.jar:/opt/lib/*',specs[op]['entrypoint'],verb]
40   if verb=='operator':command += ['/fixtures/'+d.name,'/fixtures/'+d.name+'/params.json']
41   else:command += ['/fixtures/'+d.name+'/request.json']
42   r=subprocess.run(command,capture_output=True,timeout=60)
43   if check and r.returncode:raise AssertionError(r.stderr.decode(errors='replace'))
44   return r
45  def fixture(index,data,params):
46   d=root/str(index);d.mkdir();d.chmod(0o755)
47   table=pa.table({'data':pa.array([data[i:i+2] for i in range(0,len(data),2)],type=pa.binary())})
48   with (d/'input.arrow').open('wb') as f:
49    with pa.ipc.new_stream(f,table.schema) as w:w.write_table(table,max_chunksize=2)
50   (d/'manifest.json').write_text(json.dumps([{'port':'source','file':'input.arrow','representation':'bytes','format':'text'}]))
51   (d/'params.json').write_text(json.dumps(params));return d
52  def payload(raw):
53   pos=0;data=bytearray();started=ended=False
54   while pos<len(raw):
55    hn,dn=struct.unpack_from('>ii',raw,pos);pos+=8;h=json.loads(raw[pos:pos+hn]);pos+=hn;chunk=raw[pos:pos+dn];pos+=dn
56    assert h['port']=='result'
57    if h['type']=='start':assert not started and h['representation']=='bytes';started=True
58    elif h['type']=='data':assert started and not ended;data.extend(chunk)
59    elif h['type']=='end':assert started and not ended;ended=True
60    else:raise AssertionError(h)
61   assert started and ended;return bytes(data)
62  for i,(op,data,expected,params) in enumerate(cases):
63   d=fixture(i,data,params);actual=payload(call(op,'operator',d).stdout)
64   assert actual==expected,(op,i,actual[:500],expected[:500]);print('PASS execution',op,i,flush=True)
65  for i,(op,s) in enumerate(specs.items()):
66   d=fixture('infer-'+str(i),b'',s['params'])
67   sample={'representation':'bytes','format':'text','data_base64':base64.b64encode(b'hello\tworld\n'*200).decode(),'truncated':False}
68   request={'inputs':{'source':{'representations':['bytes'],'formats':['text'],'certainty':'known','sample':sample}},'params':s['params'],'preview':{'max_rows':10,'max_bytes':256}}
69   (d/'request.json').write_text(json.dumps(request));out=json.loads(call(op,'infer',d).stdout);desc=out['outputs']['result'];assert desc['representations']==['bytes'];assert len(json.dumps(desc['sample'],separators=(',',':')).encode())<=256;print('PASS inference/preview',op,flush=True)
70   bad={'columns':'c0'} if op in ['change_case','cut_columns','merge_columns'] else {'value':'a\nb'} if op=='add_column' else {'character':'(?evil)'}
71   request['params']=bad;(d/'request.json').write_text(json.dumps(request));assert call(op,'infer',d,False).returncode!=0
72  for index,data,params in ([('huge',b'x'*1048577,{}),('utf8',b'\xff',{})] if 'add_column' in specs else []):
73   d=fixture(index,data,params);assert call('add_column','operator',d,False).returncode!=0
74  if not a.only:
75   # A three-step bytes pipeline through the real IPC protocol, including a split Unicode character.
76   data='éa,,,b\nxy,,z'.encode()
77   for i,(op,params) in enumerate([('convert_delimiters',{'character':'comma'}),('change_case',{}),('add_column',{'value':'row','iterate':True})]):
78    d=fixture('pipeline-'+str(i),data,params);data=payload(call(op,'operator',d).stdout)
79   assert data=='ÉA\tb\trow-0\nXY\tz\trow-1\n'.encode()
80 report={'artifact_sha256':hashlib.sha256(jar.read_bytes()).hexdigest(),'runtime':a.runtime,'upstream_golden_cases':(9 if not a.only else {'change_case':2,'condense_characters':1,'convert_delimiters':2,'cut_columns':2,'merge_columns':1,'add_column':1}[a.only]),'execution_cases':len(cases),'inference_preview_cases':len(specs),'invalid_parameter_cases':len(specs),'resource_encoding_cases':2 if 'add_column' in specs else 0,'pipeline':'passed' if not a.only else 'not requested','status':'passed'}
81 if a.report:a.report.parent.mkdir(parents=True,exist_ok=True);a.report.write_text(json.dumps(report,indent=2)+'\n')
82 print(json.dumps(report),flush=True)
83if __name__=='__main__':main()