upstream/tools/query_tabular/query_tabular.xml
1<tool id="query_tabular" name="Query Tabular" version="3.3.2">
2 <description>using sqlite sql</description>
3
4 <macros>
5 <import>macros.xml</import>
6 </macros>
7 <expand macro="requirements"/>
8 <command detect_errors="exit_code"><![CDATA[
9 cat '$query_file' &&
10 #if $add_to_database.withdb:
11 #if $save_db:
12 cp '$add_to_database.withdb' '$sqlitedb' &&
13 #else:
14 cp '$add_to_database.withdb' '$workdb' &&
15 #end if
16 #end if
17 python '$__tool_directory__/query_tabular.py' -d
18 #if $save_db
19 -s '$sqlitedb'
20 #else
21 -s '$workdb'
22 #end if
23 -j '$table_json'
24 #if $sqlquery:
25 -Q '$query_file'
26 @RESULT_HEADER@
27 -o '$output'
28 #end if
29 ]]></command>
30 <configfiles>
31 <configfile name="query_file">
32$sqlquery
33 </configfile>
34 <configfile name="table_json">
35#import json
36#set $jtbldef = dict()
37#set $jtbls = []
38#set $jtbldef['tables'] = $jtbls
39#for $i,$tbl in enumerate($tables):
40 #set $table = $tbl.table
41 #set $jtbl = dict()
42 #set $jtbl['file_path'] = str($tbl.table)
43 #if $tbl.tbl_opts.table_name:
44 #set $tname = str($tbl.tbl_opts.table_name)
45 #else
46 #set $tname = 't' + str($i + 1)
47 #end if
48 #set $jtbl['table_name'] = $tname
49 ## #if $tbl.tbl_opts.sel_cols:
50 ## #set $jtbl['sel_cols'] = $tbl.tbl_opts.sel_cols el_cols
51 ## #end if
52 #if $tbl.tbl_opts.pkey_autoincr:
53 #set $jtbl['pkey_autoincr'] = str($tbl.tbl_opts.pkey_autoincr)
54 #end if
55 #if $tbl.tbl_opts.column_names_from_first_line:
56 #set $jtbl['firstlinenames'] = True
57 #end if
58 #if $tbl.tbl_opts.col_names:
59 #set $col_names = str($tbl.tbl_opts.col_names)
60 #if $tbl.tbl_opts.load_named_columns:
61 #set $jtbl['load_named_columns'] = True
62 #end if
63 #else
64 #set $col_names = ''
65 #end if
66 #set $jtbl['column_names'] = $col_names
67 #set $idx_unique = []
68 #set $idx_non = []
69 #for $idx in $tbl.tbl_opts.indexes:
70 #if $idx.unique:
71 #silent $idx_unique.append(str($idx.index_columns))
72 #else:
73 #silent $idx_non.append(str($idx.index_columns))
74 #end if
75 #end for
76 #if len($idx_unique) > 0:
77 #set $jtbl['unique'] = $idx_unique
78 #end if
79 #if len($idx_non) > 0:
80 #set $jtbl['index'] = $idx_non
81 #end if
82 #set $dataset_name = $tbl.table.element_identifier
83 #set $linefilters = $tbl.input_opts.linefilters
84 @LINEFILTERS@
85 #if $input_filters:
86 #set $jtbl['filters'] = $input_filters
87 #end if
88 #set $jtbls += [$jtbl]
89#end for
90#set $jstmts = []
91#for $i,$stmt in enumerate($modify_database.sql_stmts):
92 #set $jstmts += [str($stmt.sqlstmt)]
93#end for
94#if len($jstmts) > 0:
95 #set $jtbldef['sql_stmts'] = $jstmts
96#end if
97#set $jqueries = []
98#for $i,$query in enumerate($addqueries.queries):
99 #set $jquery = dict()
100 #set $jquery['query'] = str($query.sqlquery)
101 #set $jquery['result_file'] = 'results' + str($i) + '.tsv'
102 #if $query.query_result.header == 'yes':
103 #set $header_prefix = ''
104 #if $query.query_result.header_prefix:
105 #set $header_prefix = chr(int(str($query.query_result.header_prefix)))
106 #end if
107 #set $jquery['header'] = $header_prefix
108 #end if
109 #set $jqueries += [$jquery]
110#end for
111#if len($jqueries) > 0:
112 #set $jtbldef['queries'] = $jqueries
113#end if
114#echo $json.dumps($jtbldef)
115 </configfile>
116 </configfiles>
117 <inputs>
118 <param name="workdb" type="hidden" value="workdb.sqlite" label=""/>
119 <section name="add_to_database" expanded="false" title="Add tables to an existing database">
120 <param name="withdb" type="data" format="sqlite" optional="true" label="Add tables to this Database"
121 help="Make sure your added table names are not already in this database"/>
122 </section>
123 <repeat name="tables" title="Database Table" min="0">
124 <param name="table" type="data" format="tabular" label="Tabular Dataset for Table"/>
125 <section name="input_opts" expanded="false" title="Filter Dataset Input">
126 <expand macro="macro_line_filters" />
127 </section>
128 <section name="tbl_opts" expanded="false" title="Table Options">
129 <param name="table_name" type="text" value="" optional="true" label="Specify Name for Table">
130 <help>By default, tables will be named: t1,t2,...,tn (table names must be unique)</help>
131 <validator type="regex" message="Table name should start with a letter and may contain additional letters, digits, and underscores">^[A-Za-z]\w*$</validator>
132 </param>
133 <param name="column_names_from_first_line" type="boolean" truevalue="True" falsevalue="False" checked="false"
134 label="Use first line as column names"
135 help="The names will be quoted if they are not valid SQLite column names."/>
136 <param name="col_names" type="text" value="" optional="true" label="Specify Column Names (comma-separated list)">
137 <help>By default, table columns will be named: c1,c2,c3,...,cn (column names for a table must be unique)
138 You can override the default names by entering a comma -separated list of names, e.g. ',name1,,,name2' would rename the second and fifth columns.
139 </help>
140 <sanitizer sanitize="False"/>
141 <validator type="regex" message="A List of names separated by commas: Column names should start with a letter and may contain additional letters, digits, and underscores. Otherwise, the name must be eclosed in: double quotes, back quotes, or square brackets.">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])?(,([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])?)*$</validator>
142 </param>
143 <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/>
144 <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name"
145 help="Only creates this additional column when a name is entered. (This can not be the same name as any of the other columns in this table.)">
146 <validator type="regex" message="Column name">^([A-Za-z]\w*)?$</validator>
147 </param>
148 <repeat name="indexes" title="Table Index">
149 <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/>
150 <param name="index_columns" type="text" value="" label="Index on Columns">
151 <help>Create an index on the column names: e.g. for default column names: c1 or c2,c4 ( use the names you gave for columns)</help>
152 <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])(,([A-Za-z]\w*|"\S+[^,"]"|`\S+[^,`]*`|\[\S+[^,"]*\])?)*$</validator>
153 <sanitizer sanitize="False"/>
154 </param>
155 </repeat>
156 </section>
157 </repeat>
158 <section name="modify_database" expanded="false" title="Modify the database">
159 <repeat name="sql_stmts" title="Database Manipulation SQL Statements" min="0">
160 <param name="sqlstmt" type="text" area="true" size="20x80" value="" optional="true" label="SQL Query to modify the database">
161 <help>These modify the SQLite database</help>
162 <sanitizer sanitize="False"/>
163 </param>
164 </repeat>
165 </section>
166 <param name="save_db" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Save the sqlite database in your history"
167 help="SQLite to tabular tool can run additional queries on this database"/>
168 <expand macro="sql_query_input"/>
169 <expand macro="result_results_header_line" />
170 <section name="addqueries" expanded="false" title="Additional Queries">
171 <repeat name="queries" title="SQL Query" min="0" max="3">
172 <expand macro="sql_query_input"/>
173 <expand macro="result_results_header_line" />
174 </repeat>
175 </section>
176 </inputs>
177 <outputs>
178 <data format="sqlite" name="sqlitedb" label="sqlite db of ${on_string}">
179 <filter>save_db</filter>
180 </data>
181 <data format="tabular" name="output" label="query results on ${on_string}">
182 <filter>not save_db or (sqlquery and len(sqlquery.strip()) > 0)</filter>
183 </data>
184 <data format="tabular" name="output1" label="query 1 results on ${on_string}" from_work_dir="results0.tsv">
185 <filter>len(addqueries['queries']) > 0</filter>
186 </data>
187 <data format="tabular" name="output2" label="query 2 results on ${on_string}" from_work_dir="results1.tsv">
188 <filter>len(addqueries['queries']) > 1</filter>
189 </data>
190 <data format="tabular" name="output3" label="query 3 results on ${on_string}" from_work_dir="results2.tsv">
191 <filter>len(addqueries['queries']) > 2</filter>
192 </data>
193 </outputs>
194 <tests>
195 <!-- Test 1 -->
196 <test expect_num_outputs="1">
197 <repeat name="tables">
198 <param name="table" ftype="tabular" value="customers.tsv"/>
199 <section name="input_opts">
200 <repeat name="linefilters">
201 <conditional name="filter">
202 <param name="filter_type" value="comment"/>
203 <param name="comment_char" value="35"/>
204 </conditional>
205 </repeat>
206 </section>
207 <section name="tbl_opts">
208 <param name="table_name" value="customers"/>
209 <param name="col_names" value="CustomerID,FirstName,LastName,Email,DOB,Phone"/>
210 </section>
211 </repeat>
212 <repeat name="tables">
213 <param name="table" ftype="tabular" value="sales.tsv"/>
214 <section name="input_opts">
215 <repeat name="linefilters">
216 <conditional name="filter">
217 <param name="filter_type" value="comment"/>
218 <param name="comment_char" value="35"/>
219 </conditional>
220 </repeat>
221 </section>
222 <section name="tbl_opts">
223 <param name="table_name" value="sales"/>
224 <param name="col_names" value="CustomerID,Date,SaleAmount"/>
225 </section>
226 </repeat>
227 <param name="sqlquery" value="SELECT FirstName,LastName,sum(SaleAmount) as "TotalSales" FROM customers join sales on customers.CustomerID = sales.CustomerID GROUP BY customers.CustomerID ORDER BY TotalSales DESC"/>
228 <output name="output" file="sales_results.tsv"/>
229 </test>
230
231 <!-- Test 2 -->
232 <test expect_num_outputs="1">
233 <repeat name="tables">
234 <param name="table" ftype="tabular" value="customers.tsv"/>
235 <section name="input_opts">
236 <repeat name="linefilters">
237 <conditional name="filter">
238 <param name="filter_type" value="comment"/>
239 <param name="comment_char" value="35"/>
240 </conditional>
241 </repeat>
242 </section>
243 <section name="tbl_opts">
244 <param name="col_names" value=",FirstName,LastName,,DOB,"/>
245 </section>
246 </repeat>
247 <repeat name="tables">
248 <param name="table" ftype="tabular" value="sales.tsv"/>
249 <section name="input_opts">
250 <repeat name="linefilters">
251 <conditional name="filter">
252 <param name="filter_type" value="skip"/>
253 <param name="skip_lines" value="1"/>
254 </conditional>
255 </repeat>
256 </section>
257 </repeat>
258 <param name="sqlquery" value="SELECT FirstName,LastName,sum(t2.c3) as "TotalSales" FROM t1 join t2 on t1.c1 = t2.c1 GROUP BY t1.c1 ORDER BY TotalSales DESC;"/>
259 <output name="output" file="sales_results.tsv"/>
260 </test>
261
262 <!-- Test 3 -->
263 <test expect_num_outputs="1">
264 <repeat name="tables">
265 <param name="table" ftype="tabular" value="customers.tsv"/>
266 <section name="input_opts">
267 <repeat name="linefilters">
268 <conditional name="filter">
269 <param name="filter_type" value="skip"/>
270 <param name="skip_lines" value="1"/>
271 </conditional>
272 </repeat>
273 </section>
274 <section name="tbl_opts">
275 <param name="col_names" value=",FirstName,LastName,,BirthDate,"/>
276 </section>
277 </repeat>
278 <param name="sqlquery" value="select FirstName,LastName,re_sub('^\d{2}(\d{2})-(\d\d)-(\d\d)','\3/\2/\1',BirthDate) as "DOB" from t1 WHERE re_search('[hp]er',c4)"/>
279 <output name="output" file="regex_results.tsv"/>
280 </test>
281
282 <!-- Test 4 -->
283 <test expect_num_outputs="1">
284 <repeat name="tables">
285 <param name="table" ftype="tabular" value="IEDB.tsv"/>
286 <section name="input_opts">
287 <repeat name="linefilters">
288 <conditional name="filter">
289 <param name="filter_type" value="comment"/>
290 <param name="comment_char" value="35"/>
291 </conditional>
292 </repeat>
293 </section>
294 <section name="tbl_opts">
295 <param name="table_name" value="iedb"/>
296 <param name="col_names" value="ID,allele,seq_num,start,end,length,peptide,method,percentile_rank,ann_ic50,ann_rank,smm_ic50,smm_rank,comblib_sidney2008_score,comblib_sidney2008_rank,netmhcpan_ic50,netmhcpan_rank"/>
297 </section>
298 </repeat>
299 <repeat name="tables">
300 <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
301 <section name="input_opts">
302 <repeat name="linefilters">
303 <conditional name="filter">
304 <param name="filter_type" value="skip"/>
305 <param name="skip_lines" value="1"/>
306 </conditional>
307 </repeat>
308 </section>
309 <section name="tbl_opts">
310 <param name="table_name" value="mhc_summary"/>
311 <param name="col_names" value="pos,peptide,logscore,affinity,Bind_Level,Protein,Allele"/>
312 </section>
313 </repeat>
314 <param name="sqlquery" value="select iedb.ID,iedb.peptide,iedb.start,iedb.end,iedb.percentile_rank,mhc_summary.logscore,mhc_summary.affinity,mhc_summary.Bind_Level from iedb left outer join mhc_summary on iedb.peptide = mhc_summary.peptide order by affinity,Bind_Level,percentile_rank"/>
315 <output name="output" file="query_results.tsv"/>
316 </test>
317
318 <!-- Test 5 -->
319 <test expect_num_outputs="1">
320 <section name="add_to_database">
321 <param name="withdb" ftype="sqlite" value="testdb.sqlite"/>
322 </section>
323 <repeat name="tables">
324 <param name="table" ftype="tabular" value="pets.tsv"/>
325 <section name="input_opts">
326 <repeat name="linefilters">
327 <conditional name="filter">
328 <param name="filter_type" value="regex"/>
329 <param name="regex_pattern" value="^\d+"/>
330 <param name="regex_action" value="include_find"/>
331 </conditional>
332 </repeat>
333 <repeat name="linefilters">
334 <conditional name="filter">
335 <param name="filter_type" value="comment"/>
336 <param name="comment_char" value="35"/>
337 </conditional>
338 </repeat>
339 <repeat name="linefilters">
340 <conditional name="filter">
341 <param name="filter_type" value="append_line_num"/>
342 </conditional>
343 </repeat>
344 <repeat name="linefilters">
345 <conditional name="filter">
346 <param name="filter_type" value="select_columns"/>
347 <param name="columns" value="7,2,3,4,1"/>
348 </conditional>
349 </repeat>
350 <repeat name="linefilters">
351 <conditional name="filter">
352 <param name="filter_type" value="replace"/>
353 <param name="column" value="c4"/>
354 <param name="regex_pattern" value="(\d+)/(\d+)/(\d+)"/>
355 <param name="regex_replace" value="19\3-\2-\1"/>
356 </conditional>
357 </repeat>
358 </section>
359 <section name="tbl_opts">
360 <param name="table_name" value="people"/>
361 <param name="col_names" value="id,first,last,dob,pets"/>
362 </section>
363 </repeat>
364 <param name="sqlquery" value="SELECT people.id,first,last,pets,quote FROM people JOIN contacts ON people.first = contacts.first_name"/>
365 <output name="output" file="add_to_db_results.tsv"/>
366 </test>
367
368 <!-- Test 6 -->
369 <test expect_num_outputs="1">
370 <repeat name="tables">
371 <param name="table" ftype="tabular" value="pets.tsv"/>
372 <section name="input_opts">
373 <repeat name="linefilters">
374 <conditional name="filter">
375 <param name="filter_type" value="regex"/>
376 <param name="regex_pattern" value="^\d+"/>
377 <param name="regex_action" value="include_find"/>
378 </conditional>
379 </repeat>
380 <repeat name="linefilters">
381 <conditional name="filter">
382 <param name="filter_type" value="comment"/>
383 <param name="comment_char" value="35"/>
384 </conditional>
385 </repeat>
386 <repeat name="linefilters">
387 <conditional name="filter">
388 <param name="filter_type" value="append_line_num"/>
389 </conditional>
390 </repeat>
391 <repeat name="linefilters">
392 <conditional name="filter">
393 <param name="filter_type" value="select_columns"/>
394 <param name="columns" value="7,2,3,4,1"/>
395 </conditional>
396 </repeat>
397 <repeat name="linefilters">
398 <conditional name="filter">
399 <param name="filter_type" value="replace"/>
400 <param name="column" value="c4"/>
401 <param name="regex_pattern" value="(\d+)/(\d+)/(\d+)"/>
402 <param name="regex_replace" value="19\3-\2-\1"/>
403 </conditional>
404 </repeat>
405 </section>
406 <section name="tbl_opts">
407 <param name="table_name" value="people"/>
408 <param name="col_names" value="id,first,last,dob,pets"/>
409 </section>
410 </repeat>
411 <repeat name="tables">
412 <param name="table" ftype="tabular" value="pets.tsv"/>
413 <section name="input_opts">
414 <repeat name="linefilters">
415 <conditional name="filter">
416 <param name="filter_type" value="regex"/>
417 <param name="regex_pattern" value="^\d+"/>
418 <param name="regex_action" value="include_find"/>
419 </conditional>
420 </repeat>
421 <repeat name="linefilters">
422 <conditional name="filter">
423 <param name="filter_type" value="append_line_num"/>
424 </conditional>
425 </repeat>
426 <repeat name="linefilters">
427 <conditional name="filter">
428 <param name="filter_type" value="select_columns"/>
429 <param name="columns" value="c7,c5,c6"/>
430 </conditional>
431 </repeat>
432 <repeat name="linefilters">
433 <conditional name="filter">
434 <param name="filter_type" value="normalize"/>
435 <param name="columns" value="c2,c3"/>
436 <param name="separator" value=","/>
437 </conditional>
438 </repeat>
439 </section>
440 <section name="tbl_opts">
441 <param name="table_name" value="pet"/>
442 <param name="col_names" value="id,name,animal"/>
443 </section>
444 </repeat>
445 <param name="sqlquery" value="SELECT people.id,first,last,dob,name,animal,pets FROM people JOIN pet ON people.id = pet.id WHERE animal = 'cat'"/>
446 <output name="output" file="pet_normalized_query_results.tsv"/>
447 </test>
448
449 <!-- Test 7 -->
450 <test expect_num_outputs="1">
451 <repeat name="tables">
452 <param name="table" ftype="tabular" value="psm_report.tsv"/>
453 <section name="input_opts">
454 <repeat name="linefilters">
455 <conditional name="filter">
456 <param name="filter_type" value="select_columns"/>
457 <param name="columns" value="1,3,2,6,14,19"/>
458 </conditional>
459 </repeat>
460 </section>
461 <section name="tbl_opts">
462 <param name="table_name" value="PSMs"/>
463 <param name="column_names_from_first_line" value="True"/>
464 <param name="col_names" value="Scan"/>
465 </section>
466 </repeat>
467 <param name="sqlquery" value="SELECT * from PSMs"/>
468 <conditional name="query_result">
469 <param name="header" value="yes"/>
470 <param name="header_prefix" value=""/>
471 </conditional>
472 <output name="output" file="psm_report_out1.tsv" compare="re_match"/>
473 </test>
474
475 <!-- Test 8 column_names_from_first_line -->
476 <test expect_num_outputs="1">
477 <repeat name="tables">
478 <param name="table" ftype="tabular" value="psm_report.tsv"/>
479 <section name="input_opts">
480 <repeat name="linefilters">
481 <conditional name="filter">
482 <param name="filter_type" value="select_columns"/>
483 <param name="columns" value="1,3,2,6,14,19"/>
484 </conditional>
485 </repeat>
486 </section>
487 <section name="tbl_opts">
488 <param name="table_name" value="PSMs"/>
489 <param name="column_names_from_first_line" value="True"/>
490 <param name="col_names" value="Scan"/>
491 </section>
492 </repeat>
493 <param name="sqlquery" value="SELECT Scan,"m/z", "Precursor m/z Error [ppm]", Sequence, "Protein(s)" FROM PSMs WHERE NOT re_search(', ',"Protein(s)")"/>
494 <conditional name="query_result">
495 <param name="header" value="yes"/>
496 <param name="header_prefix" value=""/>
497 </conditional>
498 <output name="output" file="psm_report_out2.tsv" compare="re_match"/>
499 </test>
500
501 <!-- Test 9 modify database -->
502 <test expect_num_outputs="2">
503 <repeat name="tables">
504 <param name="table" ftype="tabular" value="psm_report.tsv"/>
505 <section name="input_opts">
506 <repeat name="linefilters">
507 <conditional name="filter">
508 <param name="filter_type" value="select_columns"/>
509 <param name="columns" value="1,3,2,6,14,19,23"/>
510 </conditional>
511 </repeat>
512 </section>
513 <section name="tbl_opts">
514 <param name="table_name" value="PSMs"/>
515 <param name="column_names_from_first_line" value="True"/>
516 <param name="col_names" value="scan,,,,,,confidence"/>
517 </section>
518 </repeat>
519 <section name="modify_database">
520 <repeat name="sql_stmts">
521 <param name="sqlstmt" value="UPDATE psms SET confidence = 99.999 WHERE confidence = 100.0"/>
522 </repeat>
523 </section>
524 <param name="sqlquery" value="SELECT scan,"m/z", "Precursor m/z Error [ppm]", Sequence, "Protein(s)", confidence FROM PSMs WHERE NOT re_search(', ',"Protein(s)")"/>
525 <conditional name="query_result">
526 <param name="header" value="yes"/>
527 <param name="header_prefix" value="35"/>
528 </conditional>
529 <section name="addqueries">
530 <repeat name="queries">
531 <param name="sqlquery" value="SELECT * FROM psms WHERE confidence > 97.0"/>
532 <conditional name="query_result">
533 <param name="header" value="yes"/>
534 <param name="header_prefix" value=""/>
535 </conditional>
536 </repeat>
537 </section>
538 <output name="output" file="psm_dbmod_output.tsv" compare="re_match"/>
539 <output name="output1" file="psm_dbmod_output1.tsv" compare="re_match"/>
540 </test>
541
542 <!-- Test 10 column_names_from_first_line -->
543 <test expect_num_outputs="1">
544 <repeat name="tables">
545 <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
546 <section name="input_opts">
547 <repeat name="linefilters">
548 <conditional name="filter">
549 <param name="filter_type" value="regex"/>
550 <param name="regex_pattern" value="peptide"/>
551 <param name="regex_action" value="include_find"/>
552 </conditional>
553 </repeat>
554 </section>
555 <section name="tbl_opts">
556 <param name="table_name" value="epitope"/>
557 <param name="column_names_from_first_line" value="True"/>
558 </section>
559 </repeat>
560 <param name="sqlquery" value="SELECT * FROM epitope"/>
561 <conditional name="query_result">
562 <param name="header" value="yes"/>
563 <param name="header_prefix" value=""/>
564 </conditional>
565 <output name="output" file="netMHC_summary_out1.tsv" ftype="tabular"/>
566 </test>
567
568 <!-- Test 11 -->
569 <test expect_num_outputs="1">
570 <repeat name="tables">
571 <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
572 <section name="input_opts">
573 <repeat name="linefilters">
574 <conditional name="filter">
575 <param name="filter_type" value="regex"/>
576 <param name="regex_pattern" value="NOT TO BE FOUND"/>
577 <param name="regex_action" value="include_find"/>
578 </conditional>
579 </repeat>
580 </section>
581 <section name="tbl_opts">
582 <param name="table_name" value="epitope"/>
583 <param name="column_names_from_first_line" value="False"/>
584 <param name="col_names" value="pos,peptide,logscore,,,,Allele"/>
585 </section>
586 </repeat>
587 <param name="sqlquery" value="SELECT pos,peptide,logscore,Allele FROM epitope"/>
588 <conditional name="query_result">
589 <param name="header" value="yes"/>
590 <param name="header_prefix" value=""/>
591 </conditional>
592 <output name="output" file="netMHC_summary_out2.tsv" ftype="tabular"/>
593 </test>
594
595 <!-- Test 12 math functions -->
596 <test expect_num_outputs="1">
597 <repeat name="tables">
598 <param name="table" ftype="tabular" value="math_input.tsv"/>
599 </repeat>
600 <param name="sqlquery" value="SELECT c1, c2, trunc(c2), floor(c2), ceil(c2), abs(c1), fabs(c2), mod(c1,2), fmod(c2, 1.5), sqrt(c1), sqrt(c2), degrees(c1), radians(c1), log(c1), log10(c1), pow(c1,2), exp(c2), exp(c1), cos(c1), sin(c1), tan(c1), cosh(c1), sinh(c1), tanh(c1), acos(c1), asin(c1),acosh(c1), asinh(c1) FROM t1"/>
601 <output name="output">
602 <assert_contents>
603 <has_text_matching expression="-2\t-2.222\t-2\t-3\t-2\t2\t2.222\t0\t-0.722\t\t\t-114.59155\d*\t-0.03490\d*\t\t\t4.0\t0.10839\d*\t0.13533\d*\t-0.41614\d*\t-0.90929\d*\t2.18503\d*\t3.76219\d*\t-3.62686\d*\t-0.96402\d*\t\t\t\t-1.44363\d*" />
604 <has_text_matching expression="-1\t-1.999\t-1\t-2\t-1\t1\t1.999\t1\t-0.49900\d*\t\t\t-57.29577\d*\t-0.01745\d*\t\t\t1.0\t0.13547\d*\t0.36787\d*\t0.54030\d*\t-0.84147\d*\t-1.55740\d*\t1.54308\d*\t-1.17520\d*\t-0.76159\d*\t3.14159\d*\t-1.57079\d*\t\t-0.88137\d*" />
605 <has_text_matching expression="0\t0.0\t0\t0\t0\t0\t0.0\t0\t0.0\t0.0\t0.0\t0.0\t0.0\t\t\t0.0\t1.0\t1.0\t1.0\t0.0\t0.0\t1.0\t0.0\t0.0\t1.57079\d*\t0.0\t\t0.0" />
606 <has_text_matching expression="1\t1.0\t1\t1\t1\t1\t1.0\t1\t1.0\t1.0\t1.0\t57.29577\d*\t0.01745\d*\t0.0\t0.0\t1.0\t2.71828\d*\t2.71828\d*\t0.54030\d*\t0.84147\d*\t1.55740\d*\t1.54308\d*\t1.17520\d*\t0.76159\d*\t0.0\t1.57079\d*\t0.0\t0.88137\d*" />
607 <has_text_matching expression="2\t2.22\t2\t2\t3\t2\t2.22\t0\t0.72000\d*\t1.41421\d*\t1.48996\d*\t114.59155\d*\t0.03490\d*\t0.69314\d*\t0.30102\d*\t4.0\t9.20733\d*\t7.38905\d*\t-0.41614\d*\t0.90929\d*\t-2.18503\d*\t3.76219\d*\t3.62686\d*\t0.96402\d*\t\t\t1.31695\d*\t1.44363\d*" />
608 <has_text_matching expression="3\t3.33333\d*\t3\t3\t4\t3\t3.33333\d*\t1\t0.33333\d*\t1.73205\d*\t1.82574\d*\t171.88733\d*\t0.05235\d*\t1.09861\d*\t0.47712\d*\t9.0\t28.03153\d*\t20.08553\d*\t-0.98999\d*\t0.14112\d*\t-0.14254\d*\t10.06766\d*\t10.01787\d*\t0.99505\d*\t\t\t1.76274\d*\t1.81844\d*" />
609 <has_text_matching expression="4\t4.00444\d*\t4\t4\t5\t4\t4.00444\d*\t0\t1.00443\d*\t2.0\t2.00110\d*\t229.18311\d*\t0.06981\d*\t1.38629\d*\t0.60205\d*\t16.0\t54.84110\d*\t54.59815\d*\t-0.65364\d*\t-0.75680\d*\t1.15782\d*\t27.30823\d*\t27.28991\d*\t0.99932\d*\t\t\t2.06343\d*\t2.09471\d*" />
610 <has_text_matching expression="5\t5.555\t5\t5\t6\t5\t5.555\t1\t1.05499\d*\t2.23606\d*\t2.35690\d*\t286.47889\d*\t0.08726\d*\t1.60943\d*\t0.69897\d*\t25.0\t258.52696\d*\t148.41315\d*\t0.28366\d*\t-0.95892\d*\t-3.38051\d*\t74.20994\d*\t74.20321\d*\t0.99990\d*\t\t\t2.29243\d*\t2.31243\d*" />
611 </assert_contents>
612 </output>
613 </test>
614
615 </tests>
616
617 <help><![CDATA[
618=============
619Query Tabular
620=============
621
622**Inputs**
623
624 Loads tabular datasets into a SQLite_ data base.
625
626 An existing SQLite_ data base can be used as input, and any selected tabular datasets will be added as new tables in that data base.
627
628 **NOTE:** If there are no data rows in an input tabular dataset, query_tabular will fail unless:
629
630 - There is a row header line in the dataset and **Use first line as column names** is selected
631 - **Specify Column Names** is used to provide column names
632
633 in which case an empty table will be created with those columns
634
635
636@LINEFILTERS_HELP@
637
638**Table Options**
639
640 Specify Name for Table:
641
642 - By default tables will be named: t1, t2, t3, etc.
643
644 - **Specify Name for Table**
645 You can provide your own name for a database table, the name should begin with a letter and only contain letters, digits, or underscores.
646 The name should not be a SQLite key word: https://sqlite.org/lang_keywords.html
647
648
649 There are several ways to name columns in a table:
650
651 - By default table columns will be named: c1, c2, c3, etc.
652
653 - If **Use first line as column names** is selected, the first line is interpreted as column names, and not loaded into the table as a data row.
654 Any missing column name will be assigned cn where *n* is the ordinal position of the column,
655 e.g. a blank header for the second column would be named: c2.
656 The column names will be quoted is they are not valid identifiers,
657 e.g. if they are SQLite keywords, or start with a non letter character, or contain a character other than a letter, digit, or underscore.
658 The precedent for quoting is to enclose the identifier in double quotes,
659 else if it contains a double quote enclose in square brackets,
660 else if it contains a square bracket enclose in grave accents.
661 **NOTE:** that this is the first line after line filtering has been applied.
662 If a line filter *prepend a line number column* had been used, the name of the first column would be "1".
663 You could rename that column using **Specify Column Names**
664
665 - **Specify Column Names** (comma-separated list)
666 This will override any previously assigned column names.
667 You can also choose to only load those columns for which you provided a name,
668 but that is better accomplished with the line filter: *select columns*.
669
670
671 **Table Index**
672
673 Queries on larger tables can be much faster if indexes are specified.
674 In general, specifiy an index for table columns used in joins with other tables,
675 or on columns used in SQL query WHERE clauses or in GROUP BY clauses.
676
677
678**Outputs**
679
680 The results of a SQL query are output to the history as a tabular file.
681
682 The SQLite_ data base can also be saved and output as a dataset in the history.
683
684 *(The* **SQLite to tabular** *tool can run additional queries on this database.)*
685
686
687@QUERY_HELP@
688
689@LINEFILTERS_HELP_EXAMPLE@
690
691
692 Table name: pets
693
694 Table columns: Pets,FirstName,LastName,Birthdate,PetNames,PetType,line_num,entry_num,row_num
695
696 Query: SELECT * FROM pets
697
698 Result:
699
700 ====== ========== ======== ========== ========= ======== ========= ========== ========
701 #Pets FirstName LastName BirthDate PetNames PetType line_num entry_num row_num
702 ====== ========== ======== ========== ========= ======== ========= ========== ========
703 2 Paula Brown 1978-05-24 Rex dog 3 1 1
704 2 Paula Brown 1978-05-24 Fluff cat 3 1 2
705 1 Steven Jones 1974-04-04 Allie cat 4 2 3
706 0 Jane Doe 1978-05-24 5 3 4
707 1 James Smith 1980-10-20 Spot 6 4 5
708 ====== ========== ======== ========== ========= ======== ========= ========== ========
709
710
711**Normalizing by Line Filtering into 2 Tables**
712
713*Relational database opertions work with single-valued column entries.
714To apply relational operations to tabular files that contain fields with lists of values,
715we need to "normalize" those fields, duplicating lines for each item in the list.
716In this example we create 2 tables, one for single-valued fields and a second with list-valued fields normalized.
717Becauce we add a line number first for each table, we can join the 2 tables on the line number column.*
718https://en.wikipedia.org/wiki/First_normal_form
719
720 *People Table*
721
722 ::
723
724 Filter 1 - by regex expression matching [include]: '^\d+' (include lines that start with a number)
725 Filter 2 - append a line number column:
726 Filter 3 - regex replace value in column[4]: '(\d+)/(\d+)/(\d+)' '19\3-\2-\1' (convert dates to sqlite format)
727 Filter 4 - select columns 7,2,3,4,1
728
729 Table: People
730 Columns: id,FirstName,LastName,DOB,Pets
731
732 == ========= ======== ========== ====
733 id FirstName LastName DOB Pets
734 == ========= ======== ========== ====
735 1 Paula Brown 1978-05-24 2
736 2 Steven Jones 1974-04-04 1
737 3 Jane Doe 1978-05-24 0
738 4 James Smith 1980-10-20 1
739 == ========= ======== ========== ====
740
741
742 *Pet Table*
743
744 ::
745
746 Filter 1 - by regex expression matching [include]: '^\d+' (include lines that start with a number)
747 Filter 2 - append a line number column:
748 Filter 3 - by regex expression matching [exclude]: '^0\t' (exclude lines with no pets)
749 Filter 4 - normalize list columns[5,6]:
750 Filter 5 - select columns 7,5,6
751
752 Table: Pet
753 Columns: id,PetName,PetType
754
755 == ======== ========
756 id PetName PetType
757 == ======== ========
758 1 Rex dog
759 1 Fluff cat
760 2 Allie cat
761 4 Spot
762 == ======== ========
763
764
765 Query: SELECT FirstName,LastName,PetName FROM People JOIN Pet ON People.id = Pet.id WHERE PetType = 'cat';
766
767 Result:
768
769 ========= ======== ========
770 FirstName LastName PetName
771 ========= ======== ========
772 Paula Brown Fluff
773 Steven Jones Allie
774 ========= ======== ========
775
776
777 ]]></help>
778 <expand macro="citations" />
779</tool>