Source code

Download source archive ↓

upstream/tools/query_tabular/query_tabular.xml

1<tool id="query_tabular" name="Query Tabular" version="3.3.2">
2    <description>using sqlite sql</description>
3
4    <macros>
5         <import>macros.xml</import>
6    </macros>
7    <expand macro="requirements"/>
8    <command detect_errors="exit_code"><![CDATA[
9        cat '$query_file' &&
10        #if $add_to_database.withdb: 
11            #if $save_db:
12                cp '$add_to_database.withdb' '$sqlitedb' &&
13            #else:
14                cp '$add_to_database.withdb' '$workdb' &&
15            #end if 
16        #end if
17        python '$__tool_directory__/query_tabular.py' -d
18        #if $save_db
19        -s '$sqlitedb'
20        #else
21        -s '$workdb'
22        #end if
23        -j '$table_json'
24        #if $sqlquery:
25          -Q '$query_file' 
26          @RESULT_HEADER@
27          -o '$output'
28        #end if
29    ]]></command>
30    <configfiles>
31        <configfile name="query_file">
32$sqlquery
33        </configfile>
34        <configfile name="table_json">
35#import json
36#set $jtbldef = dict()
37#set $jtbls = []
38#set $jtbldef['tables'] = $jtbls
39#for $i,$tbl in enumerate($tables):
40  #set $table = $tbl.table
41  #set $jtbl = dict()
42  #set $jtbl['file_path'] = str($tbl.table)
43  #if $tbl.tbl_opts.table_name:
44  #set $tname = str($tbl.tbl_opts.table_name)
45  #else
46  #set $tname = 't' + str($i + 1) 
47  #end if
48  #set $jtbl['table_name'] = $tname
49  ## #if $tbl.tbl_opts.sel_cols:
50  ##   #set $jtbl['sel_cols'] = $tbl.tbl_opts.sel_cols el_cols
51  ## #end if
52  #if $tbl.tbl_opts.pkey_autoincr:
53    #set $jtbl['pkey_autoincr'] = str($tbl.tbl_opts.pkey_autoincr)
54  #end if
55  #if $tbl.tbl_opts.column_names_from_first_line:
56    #set $jtbl['firstlinenames'] = True
57  #end if
58  #if $tbl.tbl_opts.col_names:
59  #set $col_names = str($tbl.tbl_opts.col_names)
60    #if $tbl.tbl_opts.load_named_columns:
61      #set $jtbl['load_named_columns'] = True
62    #end if
63  #else 
64  #set $col_names = ''
65  #end if
66  #set $jtbl['column_names'] = $col_names
67  #set $idx_unique = []
68  #set $idx_non = []
69  #for $idx in $tbl.tbl_opts.indexes:
70    #if $idx.unique:
71      #silent $idx_unique.append(str($idx.index_columns))
72    #else:
73      #silent $idx_non.append(str($idx.index_columns))
74    #end if
75  #end for
76  #if len($idx_unique) > 0:
77    #set $jtbl['unique'] = $idx_unique
78  #end if
79  #if len($idx_non) > 0:
80    #set $jtbl['index'] = $idx_non
81  #end if
82  #set $dataset_name = $tbl.table.element_identifier
83  #set $linefilters = $tbl.input_opts.linefilters
84  @LINEFILTERS@
85  #if $input_filters:
86    #set $jtbl['filters'] = $input_filters
87  #end if
88  #set $jtbls += [$jtbl]
89#end for
90#set $jstmts = []
91#for $i,$stmt in enumerate($modify_database.sql_stmts):
92    #set $jstmts += [str($stmt.sqlstmt)]
93#end for
94#if len($jstmts) > 0:
95   #set $jtbldef['sql_stmts'] = $jstmts
96#end if
97#set $jqueries = []
98#for $i,$query in enumerate($addqueries.queries):
99    #set $jquery = dict()
100    #set $jquery['query'] = str($query.sqlquery)
101    #set $jquery['result_file'] = 'results' + str($i) + '.tsv'
102    #if $query.query_result.header == 'yes':
103        #set $header_prefix = ''
104        #if $query.query_result.header_prefix:
105            #set $header_prefix = chr(int(str($query.query_result.header_prefix)))
106        #end if
107        #set $jquery['header'] = $header_prefix
108    #end if
109    #set $jqueries += [$jquery]
110#end for
111#if len($jqueries) > 0:
112   #set $jtbldef['queries'] = $jqueries
113#end if
114#echo $json.dumps($jtbldef)
115        </configfile>
116    </configfiles>
117    <inputs>
118        <param name="workdb" type="hidden" value="workdb.sqlite" label=""/>
119        <section name="add_to_database" expanded="false" title="Add tables to an existing database">
120            <param name="withdb" type="data" format="sqlite" optional="true" label="Add tables to this Database" 
121               help="Make sure your added table names are not already in this database"/>
122        </section>
123        <repeat name="tables" title="Database Table" min="0">
124            <param name="table" type="data" format="tabular" label="Tabular Dataset for Table"/>
125            <section name="input_opts" expanded="false" title="Filter Dataset Input">
126                <expand macro="macro_line_filters" />
127            </section>
128            <section name="tbl_opts" expanded="false" title="Table Options">
129                <param name="table_name" type="text" value="" optional="true" label="Specify Name for Table">
130                    <help>By default, tables will be named: t1,t2,...,tn (table names must be unique)</help>
131                    <validator type="regex" message="Table name should start with a letter and may contain additional letters, digits, and underscores">^[A-Za-z]\w*$</validator>
132                </param>
133                <param name="column_names_from_first_line" type="boolean" truevalue="True" falsevalue="False" checked="false" 
134                    label="Use first line as column names"
135                    help="The names will be quoted if they are not valid SQLite column names."/>
136                <param name="col_names" type="text" value="" optional="true" label="Specify Column Names (comma-separated list)">
137                    <help>By default, table columns will be named: c1,c2,c3,...,cn  (column names for a table must be unique)
138                          You can override the default names by entering a comma -separated list of names, e.g. ',name1,,,name2' would rename the second and fifth columns.
139                    </help>
140                    <sanitizer sanitize="False"/>
141                    <validator type="regex" message="A List of names separated by commas: Column names should start with a letter and may contain additional letters, digits, and underscores. Otherwise, the name must be eclosed in: double quotes, back quotes, or square brackets.">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])?(,([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])?)*$</validator>
142                </param>
143                <param name="load_named_columns" type="boolean" truevalue="load_named_columns" falsevalue="" checked="false" label="Only load the columns you have named into database"/>
144                <param name="pkey_autoincr" type="text" value="" optional="true" label="Add an auto increment primary key column with this name"
145                       help="Only creates this additional column when a name is entered. (This can not be the same name as any of the other columns in this table.)">
146                        <validator type="regex" message="Column name">^([A-Za-z]\w*)?$</validator>
147                </param>
148                <repeat name="indexes" title="Table Index">
149                    <param name="unique" type="boolean" truevalue="yes" falsevalue="no" checked="False" label="This is a unique index"/>
150                    <param name="index_columns" type="text" value="" label="Index on Columns">
151                        <help>Create an index on the column names: e.g. for default column names:  c1  or c2,c4 ( use the names you gave for columns)</help>
152                        <validator type="regex" message="Column name, separated by commes if more than one">^([A-Za-z]\w*|"\S+[^,"]*"|`\S+[^,`]*`|\[\S+[^,"]*\])(,([A-Za-z]\w*|"\S+[^,"]"|`\S+[^,`]*`|\[\S+[^,"]*\])?)*$</validator>
153                        <sanitizer sanitize="False"/>
154                    </param>
155                </repeat>
156            </section>
157        </repeat>
158        <section name="modify_database" expanded="false" title="Modify the database">
159            <repeat name="sql_stmts" title="Database Manipulation SQL Statements" min="0">
160                <param name="sqlstmt" type="text" area="true" size="20x80" value="" optional="true" label="SQL Query to modify the database">
161                    <help>These modify the SQLite database</help>
162                    <sanitizer sanitize="False"/>
163                </param>
164            </repeat>
165        </section>
166        <param name="save_db" type="boolean" truevalue="yes" falsevalue="no" checked="false" label="Save the sqlite database in your history"
167            help="SQLite to tabular tool can run additional queries on this database"/>
168        <expand macro="sql_query_input"/>
169        <expand macro="result_results_header_line" />
170        <section name="addqueries" expanded="false" title="Additional Queries">
171            <repeat name="queries" title="SQL Query" min="0" max="3">
172                <expand macro="sql_query_input"/>
173                <expand macro="result_results_header_line" />
174            </repeat>
175        </section>
176    </inputs>
177    <outputs>
178        <data format="sqlite" name="sqlitedb" label="sqlite db of ${on_string}">
179            <filter>save_db</filter>
180        </data>
181        <data format="tabular" name="output" label="query results on ${on_string}">
182            <filter>not save_db or (sqlquery and len(sqlquery.strip()) > 0)</filter>
183        </data>
184        <data format="tabular" name="output1" label="query 1 results on ${on_string}" from_work_dir="results0.tsv">
185            <filter>len(addqueries['queries']) > 0</filter>
186        </data>
187        <data format="tabular" name="output2" label="query 2 results on ${on_string}" from_work_dir="results1.tsv">
188            <filter>len(addqueries['queries']) > 1</filter>
189        </data>
190        <data format="tabular" name="output3" label="query 3 results on ${on_string}" from_work_dir="results2.tsv">
191            <filter>len(addqueries['queries']) > 2</filter>
192        </data>
193    </outputs>
194    <tests>
195        <!-- Test 1 -->
196        <test expect_num_outputs="1">
197            <repeat name="tables">
198                <param name="table" ftype="tabular" value="customers.tsv"/>
199                <section name="input_opts">
200                    <repeat name="linefilters">
201                        <conditional name="filter">
202                            <param name="filter_type" value="comment"/>
203                            <param name="comment_char" value="35"/>
204                        </conditional>
205                    </repeat>
206                </section>
207                <section name="tbl_opts">
208                    <param name="table_name" value="customers"/>
209                    <param name="col_names" value="CustomerID,FirstName,LastName,Email,DOB,Phone"/>
210                </section>
211            </repeat>
212            <repeat name="tables">
213                <param name="table" ftype="tabular" value="sales.tsv"/>
214                <section name="input_opts">
215                    <repeat name="linefilters">
216                        <conditional name="filter">
217                            <param name="filter_type" value="comment"/>
218                            <param name="comment_char" value="35"/>
219                        </conditional>
220                    </repeat>
221                </section>
222                <section name="tbl_opts">
223                    <param name="table_name" value="sales"/>
224                    <param name="col_names" value="CustomerID,Date,SaleAmount"/>
225                </section>
226            </repeat>
227            <param name="sqlquery" value="SELECT FirstName,LastName,sum(SaleAmount) as &quot;TotalSales&quot; FROM customers join sales on customers.CustomerID = sales.CustomerID GROUP BY customers.CustomerID ORDER BY TotalSales DESC"/>
228            <output name="output" file="sales_results.tsv"/>
229        </test>
230
231        <!-- Test 2 -->
232        <test expect_num_outputs="1">
233            <repeat name="tables">
234                <param name="table" ftype="tabular" value="customers.tsv"/>
235                <section name="input_opts">
236                    <repeat name="linefilters">
237                        <conditional name="filter">
238                            <param name="filter_type" value="comment"/>
239                            <param name="comment_char" value="35"/>
240                        </conditional>
241                    </repeat>
242                </section>
243                <section name="tbl_opts">
244                    <param name="col_names" value=",FirstName,LastName,,DOB,"/>
245                </section>
246            </repeat>
247            <repeat name="tables">
248                <param name="table" ftype="tabular" value="sales.tsv"/>
249                <section name="input_opts">
250                    <repeat name="linefilters">
251                        <conditional name="filter">
252                            <param name="filter_type" value="skip"/>
253                            <param name="skip_lines" value="1"/>
254                        </conditional>
255                    </repeat>
256                </section>
257            </repeat>
258            <param name="sqlquery" value="SELECT FirstName,LastName,sum(t2.c3) as &quot;TotalSales&quot; FROM t1 join t2 on t1.c1 = t2.c1 GROUP BY t1.c1 ORDER BY TotalSales DESC;"/>
259            <output name="output" file="sales_results.tsv"/>
260        </test>
261
262        <!-- Test 3 -->
263        <test expect_num_outputs="1">
264            <repeat name="tables">
265                <param name="table" ftype="tabular" value="customers.tsv"/>
266                <section name="input_opts">
267                    <repeat name="linefilters">
268                        <conditional name="filter">
269                            <param name="filter_type" value="skip"/>
270                            <param name="skip_lines" value="1"/>
271                        </conditional>
272                    </repeat>
273                </section>
274                <section name="tbl_opts">
275                    <param name="col_names" value=",FirstName,LastName,,BirthDate,"/>
276                </section>
277            </repeat>
278            <param name="sqlquery" value="select FirstName,LastName,re_sub('^\d{2}(\d{2})-(\d\d)-(\d\d)','\3/\2/\1',BirthDate) as &quot;DOB&quot; from t1 WHERE re_search('[hp]er',c4)"/>
279            <output name="output" file="regex_results.tsv"/>
280        </test>
281
282        <!-- Test 4 -->
283        <test expect_num_outputs="1">
284            <repeat name="tables">
285                <param name="table" ftype="tabular" value="IEDB.tsv"/>
286                <section name="input_opts">
287                    <repeat name="linefilters">
288                        <conditional name="filter">
289                            <param name="filter_type" value="comment"/>
290                            <param name="comment_char" value="35"/>
291                        </conditional>
292                    </repeat>
293                </section>
294                <section name="tbl_opts">
295                    <param name="table_name" value="iedb"/>
296                    <param name="col_names" value="ID,allele,seq_num,start,end,length,peptide,method,percentile_rank,ann_ic50,ann_rank,smm_ic50,smm_rank,comblib_sidney2008_score,comblib_sidney2008_rank,netmhcpan_ic50,netmhcpan_rank"/>
297                </section>
298            </repeat>
299            <repeat name="tables">
300                <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
301                <section name="input_opts">
302                    <repeat name="linefilters">
303                        <conditional name="filter">
304                            <param name="filter_type" value="skip"/>
305                            <param name="skip_lines" value="1"/>
306                        </conditional>
307                    </repeat>
308                </section>
309                <section name="tbl_opts">
310                    <param name="table_name" value="mhc_summary"/>
311                    <param name="col_names" value="pos,peptide,logscore,affinity,Bind_Level,Protein,Allele"/>
312                </section>
313            </repeat>
314            <param name="sqlquery" value="select iedb.ID,iedb.peptide,iedb.start,iedb.end,iedb.percentile_rank,mhc_summary.logscore,mhc_summary.affinity,mhc_summary.Bind_Level from iedb left outer join mhc_summary on iedb.peptide = mhc_summary.peptide order by affinity,Bind_Level,percentile_rank"/>
315            <output name="output" file="query_results.tsv"/>
316        </test>
317
318        <!-- Test 5 -->
319        <test expect_num_outputs="1">
320            <section name="add_to_database">
321                <param name="withdb" ftype="sqlite" value="testdb.sqlite"/>
322            </section>
323            <repeat name="tables">
324                <param name="table" ftype="tabular" value="pets.tsv"/>
325                <section name="input_opts">
326                    <repeat name="linefilters">
327                        <conditional name="filter">
328                            <param name="filter_type" value="regex"/>
329                            <param name="regex_pattern" value="^\d+"/>
330                            <param name="regex_action" value="include_find"/>
331                        </conditional>
332                    </repeat>
333                    <repeat name="linefilters">
334                        <conditional name="filter">
335                            <param name="filter_type" value="comment"/>
336                            <param name="comment_char" value="35"/>
337                        </conditional>
338                    </repeat>
339                    <repeat name="linefilters">
340                        <conditional name="filter">
341                            <param name="filter_type" value="append_line_num"/>
342                        </conditional>
343                    </repeat>
344                    <repeat name="linefilters">
345                        <conditional name="filter">
346                            <param name="filter_type" value="select_columns"/>
347                            <param name="columns" value="7,2,3,4,1"/>
348                        </conditional>
349                    </repeat>
350                    <repeat name="linefilters">
351                        <conditional name="filter">
352                            <param name="filter_type" value="replace"/>
353                            <param name="column" value="c4"/>
354                            <param name="regex_pattern" value="(\d+)/(\d+)/(\d+)"/>
355                            <param name="regex_replace" value="19\3-\2-\1"/>
356                        </conditional>
357                    </repeat>
358                </section>
359                <section name="tbl_opts">
360                    <param name="table_name" value="people"/>
361                    <param name="col_names" value="id,first,last,dob,pets"/>
362                </section>
363            </repeat>
364            <param name="sqlquery" value="SELECT people.id,first,last,pets,quote FROM people JOIN contacts ON people.first = contacts.first_name"/>
365            <output name="output" file="add_to_db_results.tsv"/>
366        </test>
367
368        <!-- Test 6 -->
369        <test expect_num_outputs="1">
370            <repeat name="tables">
371                <param name="table" ftype="tabular" value="pets.tsv"/>
372                <section name="input_opts">
373                    <repeat name="linefilters">
374                        <conditional name="filter">
375                            <param name="filter_type" value="regex"/>
376                            <param name="regex_pattern" value="^\d+"/>
377                            <param name="regex_action" value="include_find"/>
378                        </conditional>
379                    </repeat>
380                    <repeat name="linefilters">
381                        <conditional name="filter">
382                            <param name="filter_type" value="comment"/>
383                            <param name="comment_char" value="35"/>
384                        </conditional>
385                    </repeat>
386                    <repeat name="linefilters">
387                        <conditional name="filter">
388                            <param name="filter_type" value="append_line_num"/>
389                        </conditional>
390                    </repeat>
391                    <repeat name="linefilters">
392                        <conditional name="filter">
393                            <param name="filter_type" value="select_columns"/>
394                            <param name="columns" value="7,2,3,4,1"/>
395                        </conditional>
396                    </repeat>
397                    <repeat name="linefilters">
398                        <conditional name="filter">
399                            <param name="filter_type" value="replace"/>
400                            <param name="column" value="c4"/>
401                            <param name="regex_pattern" value="(\d+)/(\d+)/(\d+)"/>
402                            <param name="regex_replace" value="19\3-\2-\1"/>
403                        </conditional>
404                    </repeat>
405                </section>
406                <section name="tbl_opts">
407                    <param name="table_name" value="people"/>
408                    <param name="col_names" value="id,first,last,dob,pets"/>
409                </section>
410            </repeat>
411            <repeat name="tables">
412                <param name="table" ftype="tabular" value="pets.tsv"/>
413                <section name="input_opts">
414                    <repeat name="linefilters">
415                        <conditional name="filter">
416                            <param name="filter_type" value="regex"/>
417                            <param name="regex_pattern" value="^\d+"/>
418                            <param name="regex_action" value="include_find"/>
419                        </conditional>
420                    </repeat>
421                    <repeat name="linefilters">
422                        <conditional name="filter">
423                            <param name="filter_type" value="append_line_num"/>
424                        </conditional>
425                    </repeat>
426                    <repeat name="linefilters">
427                        <conditional name="filter">
428                            <param name="filter_type" value="select_columns"/>
429                            <param name="columns" value="c7,c5,c6"/>
430                        </conditional>
431                    </repeat>
432                    <repeat name="linefilters">
433                        <conditional name="filter">
434                            <param name="filter_type" value="normalize"/>
435                            <param name="columns" value="c2,c3"/>
436                            <param name="separator" value=","/>
437                        </conditional>
438                    </repeat>
439                </section>
440                <section name="tbl_opts">
441                    <param name="table_name" value="pet"/>
442                    <param name="col_names" value="id,name,animal"/>
443                </section>
444            </repeat>
445            <param name="sqlquery" value="SELECT people.id,first,last,dob,name,animal,pets FROM people JOIN pet ON people.id = pet.id WHERE animal = 'cat'"/>
446            <output name="output" file="pet_normalized_query_results.tsv"/>
447        </test>
448
449        <!-- Test 7 -->
450        <test expect_num_outputs="1">
451            <repeat name="tables">
452                <param name="table" ftype="tabular" value="psm_report.tsv"/>
453                <section name="input_opts">
454                    <repeat name="linefilters">
455                        <conditional name="filter">
456                            <param name="filter_type" value="select_columns"/>
457                            <param name="columns" value="1,3,2,6,14,19"/>
458                        </conditional>
459                    </repeat>
460                </section>
461                <section name="tbl_opts">
462                    <param name="table_name" value="PSMs"/>
463                    <param name="column_names_from_first_line" value="True"/>
464                    <param name="col_names" value="Scan"/>
465                </section>
466            </repeat>
467            <param name="sqlquery" value="SELECT * from PSMs"/>
468            <conditional name="query_result">
469                <param name="header" value="yes"/>
470                <param name="header_prefix" value=""/>
471            </conditional>
472            <output name="output" file="psm_report_out1.tsv" compare="re_match"/>
473        </test>
474
475        <!-- Test 8 column_names_from_first_line -->
476        <test expect_num_outputs="1">
477            <repeat name="tables">
478                <param name="table" ftype="tabular" value="psm_report.tsv"/>
479                <section name="input_opts">
480                    <repeat name="linefilters">
481                        <conditional name="filter">
482                            <param name="filter_type" value="select_columns"/>
483                            <param name="columns" value="1,3,2,6,14,19"/>
484                        </conditional>
485                    </repeat>
486                </section>
487                <section name="tbl_opts">
488                    <param name="table_name" value="PSMs"/>
489                    <param name="column_names_from_first_line" value="True"/>
490                    <param name="col_names" value="Scan"/>
491                </section>
492            </repeat>
493            <param name="sqlquery" value="SELECT Scan,&quot;m/z&quot;, &quot;Precursor m/z Error [ppm]&quot;, Sequence, &quot;Protein(s)&quot; FROM PSMs WHERE NOT re_search(', ',&quot;Protein(s)&quot;)"/>
494            <conditional name="query_result">
495                <param name="header" value="yes"/>
496                <param name="header_prefix" value=""/>
497            </conditional>
498            <output name="output" file="psm_report_out2.tsv" compare="re_match"/>
499        </test>
500
501        <!-- Test 9  modify database -->
502        <test expect_num_outputs="2">
503            <repeat name="tables">
504                <param name="table" ftype="tabular" value="psm_report.tsv"/>
505                <section name="input_opts">
506                    <repeat name="linefilters">
507                        <conditional name="filter">
508                            <param name="filter_type" value="select_columns"/>
509                            <param name="columns" value="1,3,2,6,14,19,23"/>
510                        </conditional>
511                    </repeat>
512                </section>
513                <section name="tbl_opts">
514                    <param name="table_name" value="PSMs"/>
515                    <param name="column_names_from_first_line" value="True"/>
516                    <param name="col_names" value="scan,,,,,,confidence"/>
517                </section>
518            </repeat>
519            <section name="modify_database">
520                <repeat name="sql_stmts">
521                    <param name="sqlstmt" value="UPDATE psms SET confidence = 99.999 WHERE confidence = 100.0"/>
522                </repeat>
523            </section>
524            <param name="sqlquery" value="SELECT scan,&quot;m/z&quot;, &quot;Precursor m/z Error [ppm]&quot;, Sequence, &quot;Protein(s)&quot;, confidence FROM PSMs WHERE NOT re_search(', ',&quot;Protein(s)&quot;)"/>
525            <conditional name="query_result">
526                <param name="header" value="yes"/>
527                <param name="header_prefix" value="35"/>
528            </conditional>
529            <section name="addqueries">
530                <repeat name="queries">
531                    <param name="sqlquery" value="SELECT * FROM psms WHERE confidence > 97.0"/>
532                    <conditional name="query_result">
533                        <param name="header" value="yes"/>
534                        <param name="header_prefix" value=""/>
535                    </conditional>
536                </repeat>
537            </section>
538            <output name="output" file="psm_dbmod_output.tsv" compare="re_match"/>
539            <output name="output1" file="psm_dbmod_output1.tsv" compare="re_match"/>
540        </test>
541
542        <!-- Test 10 column_names_from_first_line -->
543        <test expect_num_outputs="1">
544            <repeat name="tables">
545                <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
546                <section name="input_opts">
547                    <repeat name="linefilters">
548                        <conditional name="filter">
549                            <param name="filter_type" value="regex"/>
550                            <param name="regex_pattern" value="peptide"/>
551                            <param name="regex_action" value="include_find"/>
552                        </conditional>
553                    </repeat>
554                </section>
555                <section name="tbl_opts">
556                    <param name="table_name" value="epitope"/>
557                    <param name="column_names_from_first_line" value="True"/>
558                </section>
559            </repeat>
560            <param name="sqlquery" value="SELECT * FROM epitope"/>
561            <conditional name="query_result">
562                <param name="header" value="yes"/>
563                <param name="header_prefix" value=""/>
564            </conditional>
565            <output name="output" file="netMHC_summary_out1.tsv" ftype="tabular"/>
566        </test>
567
568        <!-- Test 11 -->
569        <test expect_num_outputs="1">
570            <repeat name="tables">
571                <param name="table" ftype="tabular" value="netMHC_summary.tsv"/>
572                <section name="input_opts">
573                    <repeat name="linefilters">
574                        <conditional name="filter">
575                            <param name="filter_type" value="regex"/>
576                            <param name="regex_pattern" value="NOT TO BE FOUND"/>
577                            <param name="regex_action" value="include_find"/>
578                        </conditional>
579                    </repeat>
580                </section>
581                <section name="tbl_opts">
582                    <param name="table_name" value="epitope"/>
583                    <param name="column_names_from_first_line" value="False"/>
584                    <param name="col_names" value="pos,peptide,logscore,,,,Allele"/>
585                </section>
586            </repeat>
587            <param name="sqlquery" value="SELECT pos,peptide,logscore,Allele FROM epitope"/>
588            <conditional name="query_result">
589                <param name="header" value="yes"/>
590                <param name="header_prefix" value=""/>
591            </conditional>
592            <output name="output" file="netMHC_summary_out2.tsv" ftype="tabular"/>
593        </test>
594
595        <!-- Test 12 math functions -->
596        <test expect_num_outputs="1">
597            <repeat name="tables">
598                <param name="table" ftype="tabular" value="math_input.tsv"/>
599            </repeat>
600            <param name="sqlquery" value="SELECT c1, c2, trunc(c2), floor(c2), ceil(c2), abs(c1), fabs(c2), mod(c1,2), fmod(c2, 1.5), sqrt(c1), sqrt(c2), degrees(c1), radians(c1), log(c1), log10(c1), pow(c1,2),  exp(c2), exp(c1), cos(c1), sin(c1), tan(c1), cosh(c1), sinh(c1), tanh(c1), acos(c1), asin(c1),acosh(c1), asinh(c1) FROM t1"/>
601            <output name="output">
602                <assert_contents>
603                    <has_text_matching expression="-2\t-2.222\t-2\t-3\t-2\t2\t2.222\t0\t-0.722\t\t\t-114.59155\d*\t-0.03490\d*\t\t\t4.0\t0.10839\d*\t0.13533\d*\t-0.41614\d*\t-0.90929\d*\t2.18503\d*\t3.76219\d*\t-3.62686\d*\t-0.96402\d*\t\t\t\t-1.44363\d*" />
604                    <has_text_matching expression="-1\t-1.999\t-1\t-2\t-1\t1\t1.999\t1\t-0.49900\d*\t\t\t-57.29577\d*\t-0.01745\d*\t\t\t1.0\t0.13547\d*\t0.36787\d*\t0.54030\d*\t-0.84147\d*\t-1.55740\d*\t1.54308\d*\t-1.17520\d*\t-0.76159\d*\t3.14159\d*\t-1.57079\d*\t\t-0.88137\d*" />
605                    <has_text_matching expression="0\t0.0\t0\t0\t0\t0\t0.0\t0\t0.0\t0.0\t0.0\t0.0\t0.0\t\t\t0.0\t1.0\t1.0\t1.0\t0.0\t0.0\t1.0\t0.0\t0.0\t1.57079\d*\t0.0\t\t0.0" />
606                    <has_text_matching expression="1\t1.0\t1\t1\t1\t1\t1.0\t1\t1.0\t1.0\t1.0\t57.29577\d*\t0.01745\d*\t0.0\t0.0\t1.0\t2.71828\d*\t2.71828\d*\t0.54030\d*\t0.84147\d*\t1.55740\d*\t1.54308\d*\t1.17520\d*\t0.76159\d*\t0.0\t1.57079\d*\t0.0\t0.88137\d*" />
607                    <has_text_matching expression="2\t2.22\t2\t2\t3\t2\t2.22\t0\t0.72000\d*\t1.41421\d*\t1.48996\d*\t114.59155\d*\t0.03490\d*\t0.69314\d*\t0.30102\d*\t4.0\t9.20733\d*\t7.38905\d*\t-0.41614\d*\t0.90929\d*\t-2.18503\d*\t3.76219\d*\t3.62686\d*\t0.96402\d*\t\t\t1.31695\d*\t1.44363\d*" />
608                    <has_text_matching expression="3\t3.33333\d*\t3\t3\t4\t3\t3.33333\d*\t1\t0.33333\d*\t1.73205\d*\t1.82574\d*\t171.88733\d*\t0.05235\d*\t1.09861\d*\t0.47712\d*\t9.0\t28.03153\d*\t20.08553\d*\t-0.98999\d*\t0.14112\d*\t-0.14254\d*\t10.06766\d*\t10.01787\d*\t0.99505\d*\t\t\t1.76274\d*\t1.81844\d*" />
609                    <has_text_matching expression="4\t4.00444\d*\t4\t4\t5\t4\t4.00444\d*\t0\t1.00443\d*\t2.0\t2.00110\d*\t229.18311\d*\t0.06981\d*\t1.38629\d*\t0.60205\d*\t16.0\t54.84110\d*\t54.59815\d*\t-0.65364\d*\t-0.75680\d*\t1.15782\d*\t27.30823\d*\t27.28991\d*\t0.99932\d*\t\t\t2.06343\d*\t2.09471\d*" />
610                    <has_text_matching expression="5\t5.555\t5\t5\t6\t5\t5.555\t1\t1.05499\d*\t2.23606\d*\t2.35690\d*\t286.47889\d*\t0.08726\d*\t1.60943\d*\t0.69897\d*\t25.0\t258.52696\d*\t148.41315\d*\t0.28366\d*\t-0.95892\d*\t-3.38051\d*\t74.20994\d*\t74.20321\d*\t0.99990\d*\t\t\t2.29243\d*\t2.31243\d*" />
611                </assert_contents>
612            </output>
613        </test>
614
615    </tests>
616
617    <help><![CDATA[
618=============
619Query Tabular
620=============
621
622**Inputs**
623
624  Loads tabular datasets into a SQLite_ data base.  
625
626  An existing SQLite_ data base can be used as input, and any selected tabular datasets will be added as new tables in that data base.
627
628    **NOTE:** If there are no data rows in an input tabular dataset, query_tabular will fail unless:
629
630      - There is a row header line in the dataset and **Use first line as column names** is selected  
631      - **Specify Column Names** is used to provide column names
632
633        in which case an empty table will be created with those columns
634
635
636@LINEFILTERS_HELP@
637
638**Table Options**
639
640  Specify Name for Table:
641
642    - By default tables will be named: t1, t2, t3, etc.
643 
644    - **Specify Name for Table** 
645      You can provide your own name for a database table, the name should begin with a letter and only contain letters, digits, or underscores. 
646      The name should not be a SQLite key word: https://sqlite.org/lang_keywords.html
647
648
649  There are several ways to name columns in a table:  
650
651    - By default table columns will be named: c1, c2, c3, etc.
652
653    - If **Use first line as column names** is selected, the first line is interpreted as column names, and not loaded into the table as a data row.  
654      Any missing column name will be assigned cn where *n* is the ordinal position of the column, 
655      e.g. a blank header for the second column would be named: c2.
656      The column names will be quoted is they are not valid identifiers, 
657      e.g. if they are SQLite keywords, or start with a non letter character, or contain a character other than a letter, digit, or underscore.
658      The precedent for quoting is to enclose the identifier in double quotes, 
659      else if it contains a double quote enclose in square brackets,
660      else if it contains a square bracket enclose in grave accents.
661      **NOTE:** that this is the first line after line filtering has been applied. 
662      If a line filter *prepend a line number column* had been used, the name of the first column would be "1".
663      You could rename that column using **Specify Column Names**
664
665    - **Specify Column Names** (comma-separated list)
666      This will override any previously assigned column names. 
667      You can also choose to only load those columns for which you provided a name, 
668      but that is better accomplished with the line filter: *select columns*. 
669
670
671  **Table Index**
672
673    Queries on larger tables can be much faster if indexes are specified.  
674    In general, specifiy an index for table columns used in joins with other tables, 
675    or on columns used in SQL query WHERE clauses or in GROUP BY clauses.
676
677
678**Outputs**
679
680  The results of a SQL query are output to the history as a tabular file.
681
682  The SQLite_ data base can also be saved and output as a dataset in the history.  
683
684    *(The* **SQLite to tabular** *tool can run additional queries on this database.)*
685
686
687@QUERY_HELP@
688
689@LINEFILTERS_HELP_EXAMPLE@
690
691
692  Table name: pets
693
694  Table columns: Pets,FirstName,LastName,Birthdate,PetNames,PetType,line_num,entry_num,row_num
695
696  Query: SELECT * FROM pets 
697
698  Result:
699
700     ======  ==========  ========  ==========  =========  ========  =========  ==========  ========
701     #Pets   FirstName   LastName  BirthDate   PetNames   PetType   line_num   entry_num    row_num
702     ======  ==========  ========  ==========  =========  ========  =========  ==========  ========
703     2       Paula       Brown     1978-05-24  Rex        dog              3           1         1
704     2       Paula       Brown     1978-05-24  Fluff      cat              3           1         2
705     1       Steven      Jones     1974-04-04  Allie      cat              4           2         3
706     0       Jane        Doe       1978-05-24                              5           3         4
707     1       James       Smith     1980-10-20  Spot                        6           4         5          
708     ======  ==========  ========  ==========  =========  ========  =========  ==========  ======== 
709
710
711**Normalizing by Line Filtering into 2 Tables** 
712
713*Relational database opertions work with single-valued column entries.  
714To apply relational operations to tabular files that contain fields with lists of values,
715we need to "normalize" those fields, duplicating lines for each item in the list.  
716In this example we create 2 tables, one for single-valued fields and a second with list-valued fields normalized.  
717Becauce we add a line number first for each table, we can join the 2 tables on the line number column.*
718https://en.wikipedia.org/wiki/First_normal_form 
719
720    *People Table*
721
722      ::
723  
724        Filter 1 - by regex expression matching [include]: '^\d+' (include lines that start with a number) 
725        Filter 2 - append a line number column:
726        Filter 3 - regex replace value in column[4]: '(\d+)/(\d+)/(\d+)' '19\3-\2-\1' (convert dates to sqlite format) 
727        Filter 4 - select columns 7,2,3,4,1
728
729      Table: People
730      Columns: id,FirstName,LastName,DOB,Pets
731
732      ==  =========  ========   ==========  ====
733      id  FirstName  LastName   DOB         Pets
734      ==  =========  ========   ==========  ====
735      1     Paula      Brown    1978-05-24  2
736      2     Steven     Jones    1974-04-04  1
737      3     Jane       Doe      1978-05-24  0
738      4     James      Smith    1980-10-20  1
739      ==  =========  ========   ==========  ====
740
741
742    *Pet Table*
743
744      :: 
745
746        Filter 1 - by regex expression matching [include]: '^\d+' (include lines that start with a number) 
747        Filter 2 - append a line number column:
748        Filter 3 - by regex expression matching [exclude]: '^0\t' (exclude lines with no pets)
749        Filter 4 - normalize list columns[5,6]:
750        Filter 5 - select columns 7,5,6
751
752      Table: Pet
753      Columns: id,PetName,PetType
754
755      ==  ========  ========
756      id  PetName   PetType 
757      ==  ========  ========
758      1   Rex       dog     
759      1   Fluff     cat     
760      2   Allie     cat     
761      4   Spot              
762      ==  ========  ========
763
764
765    Query: SELECT FirstName,LastName,PetName FROM People JOIN Pet ON People.id = Pet.id WHERE PetType = 'cat';     
766
767    Result:
768
769     =========  ========  ========
770     FirstName  LastName  PetName 
771     =========  ========  ========
772     Paula      Brown     Fluff   
773     Steven     Jones     Allie   
774     =========  ========  ========
775
776
777    ]]></help>
778    <expand macro="citations" />
779</tool>