Source code

Download source archive ↓

upstream/tools/query_tabular/macros.xml

1<macros>
2  <token name="@LINEFILTERS@">
3<![CDATA[
4  ## set linefilters to the 
5  #set $input_filters = []
6  #for $fi in $linefilters:
7    #if $fi.filter.filter_type == 'skip':
8      #set $skip_lines = None
9      #if str($fi.filter.skip_lines) != '':
10        #set $skip_lines = int($fi.filter.skip_lines)
11      #elif $table.metadata.comment_lines and int($table.metadata.comment_lines) > 0:
12        #set $skip_lines = int($table.metadata.comment_lines)
13      #end if
14      #if $skip_lines is not None:
15        #set $filter_dict = dict()
16        #set $filter_dict['filter'] = str($fi.filter.filter_type)
17        #set $filter_dict['count'] = $skip_lines
18        #silent $input_filters.append($filter_dict)
19      #end if
20    #elif $fi.filter.filter_type == 'comment':
21      #set $filter_dict = dict()
22      #set $filter_dict['filter'] = 'regex'
23      #set $filter_dict['pattern'] = '^(%s).*$' % '|'.join([chr(int(x)).replace('|','[|]') for x in (str($fi.filter.comment_char)).split(',')])
24      #set $filter_dict['action'] = 'exclude_match'
25      #silent $input_filters.append($filter_dict)
26    #elif $fi.filter.filter_type == 'regex':
27      #set $filter_dict = dict()
28      #set $filter_dict['filter'] = str($fi.filter.filter_type)
29      #set $filter_dict['pattern'] = str($fi.filter.regex_pattern)
30      #set $filter_dict['action'] = str($fi.filter.regex_action)
31      #silent $input_filters.append($filter_dict)
32    #elif $fi.filter.filter_type == 'select_columns':
33      #set $filter_dict = dict()
34      #set $filter_dict['filter'] = str($fi.filter.filter_type)
35      #set $filter_dict['columns'] = [int($c) for $c in str($fi.filter.columns).replace('c','').split(',')]
36      #silent $input_filters.append($filter_dict)
37    #elif $fi.filter.filter_type == 'select_column_slices':
38      #set $filter_dict = dict()
39      #set $filter_dict['filter'] = str($fi.filter.filter_type)
40      #set $filter_dict['columns'] = [$c for $c in str($fi.filter.columns).split(',')]
41      #silent $input_filters.append($filter_dict)
42    #elif $fi.filter.filter_type == 'replace':
43      #set $filter_dict = dict()
44      #set $filter_dict['filter'] = str($fi.filter.filter_type)
45      #set $filter_dict['column'] = int(str($fi.filter.column).replace('c',''))
46      #set $filter_dict['pattern'] = str($fi.filter.regex_pattern)
47      #set $filter_dict['replace'] = str($fi.filter.regex_replace)
48      #if $fi.filter.add:
49          #set $filter_dict['add'] = str($fi.filter.add)
50      #end if
51      #silent $input_filters.append($filter_dict)
52    #elif str($fi.filter.filter_type).endswith('pend_line_num'):
53      #set $filter_dict = dict()
54      #set $filter_dict['filter'] = str($fi.filter.filter_type)
55      #silent $input_filters.append($filter_dict)
56    #elif str($fi.filter.filter_type).endswith('pend_text'):
57      #set $filter_dict = dict()
58      #set $filter_dict['filter'] = str($fi.filter.filter_type)
59      #set $filter_dict['column_text'] = str($fi.filter.column_text)
60      #silent $input_filters.append($filter_dict)
61    #elif str($fi.filter.filter_type).endswith('pend_dataset_name'):
62      #set $filter_dict = dict()
63      #set $filter_dict['filter'] = str($fi.filter.filter_type).replace('dataset_name', 'text')
64      #set $filter_dict['column_text'] = $dataset_name
65      #silent $input_filters.append($filter_dict)
66    #elif $fi.filter.filter_type == 'normalize':
67      #set $filter_dict = dict()
68      #set $filter_dict['filter'] = str($fi.filter.filter_type)
69      #set $filter_dict['columns'] = [int(str($ci).replace('c','')) for $ci in str($fi.filter.columns).split(',')]
70      #set $filter_dict['separator'] = str($fi.filter.separator)
71      #silent $input_filters.append($filter_dict)
72    #end if
73  #end for
74]]>
75  </token>
76  <token name="@RESULT_HEADER@">
77<![CDATA[
78    #if $query_result.header == 'yes':
79        #if $query_result.header_prefix:
80            #set $header_prefix = chr(int(str($query_result.header_prefix)))
81            --comment_char='$header_prefix'
82        #end if
83    #else
84        --no_header
85    #end if
86]]>
87  </token>
88  <xml name="requirements">
89      <requirements>
90          <requirement type="package" version="3.7">python</requirement>
91      </requirements>
92  </xml>
93  <xml name="comment_char_options">
94        <option value="62">&gt;</option>
95        <option value="64">@</option>
96        <option value="43">+</option>
97        <option value="60">&lt;</option>
98        <option value="42">*</option>
99        <option value="45">-</option>
100        <option value="61">=</option>
101        <option value="124">|</option>
102        <option value="63">?</option>
103        <option value="36">$</option>
104        <option value="46">.</option>
105        <option value="58">:</option>
106        <option value="38">&amp;</option>
107        <option value="37">%</option>
108        <option value="94">^</option>
109        <option value="35" selected="true">&#35;</option>
110        <option value="33">!</option>
111  </xml>
112  <xml name="result_results_header_line">
113      <conditional name="query_result">
114          <param name="header" type="select" label="include query result column headers">
115              <option value="yes">Yes</option>
116              <option value="no">No</option>
117          </param>
118          <when value="yes">
119              <param name="header_prefix" type="select" optional="true" label="Prefix character for column_header line">
120                  <option value="">no comment character prefix</option>
121                  <expand macro="comment_char_options" />
122              </param>
123          </when>
124          <when value="no"/>
125      </conditional>
126  </xml>
127  <xml name="sql_query_input">
128        <param name="sqlquery" type="text" area="true" size="20x80" value="" optional="true" label="SQL Query to generate tabular output">
129            <help>By default: tables are named: t1,t2,...,tn and columns in each table: c1,c2,...,cn</help>
130            <sanitizer sanitize="False"/>
131            <validator type="regex" message="">(?ims)^\s*select\s+.*\s+from\s+.*$</validator>
132        </param>
133  </xml>
134  <xml name="macro_line_filters">
135                <repeat name="linefilters" title="Filter Tabular Input Lines">
136                    <conditional name="filter">
137                        <param name="filter_type" type="select" label="Filter By">
138                            <option value="skip">skip leading lines</option>
139                            <option value="comment">comment char</option>
140                            <option value="regex">by regex expression matching</option>
141                            <option value="select_columns">select columns</option>
142                            <option value="select_column_slices">select columns by indices/slices</option>
143                            <option value="replace">regex replace value in column</option>
144                            <option value="prepend_line_num">prepend a line number column</option>
145                            <option value="append_line_num">append a line number column</option>
146                            <option value="prepend_dataset_name">prepend a column with the dataset name</option>
147                            <option value="append_dataset_name">append a column with the dataset name</option>
148                            <option value="prepend_text">prepend a column with the given text</option>
149                            <option value="append_text">append a column with the given text</option>
150                            <option value="normalize">normalize list columns, replicates row for each item in list</option>
151                        </param>
152                        <when value="skip">
153                             <param name="skip_lines" type="integer" value="" min="0" optional="true" label="Skip lines" 
154                                 help="Leave blank to use the comment lines metadata for this dataset" />
155                        </when>
156                        <when value="comment">
157                            <param name="comment_char" type="select" display="checkboxes" multiple="True" label="Ignore lines beginning with these characters" help="lines beginning with these are skipped">
158                                <expand macro="comment_char_options" />
159                            </param>
160                        </when>
161                        <when value="prepend_line_num"/>
162                        <when value="append_line_num"/>
163                        <when value="prepend_dataset_name"/>
164                        <when value="append_dataset_name"/>
165                        <when value="prepend_text">
166                            <param name="column_text" type="text" value="" label="text for column">
167                            </param>
168                        </when>
169                        <when value="append_text">
170                            <param name="column_text" type="text" value="" label="text for column">
171                            </param>
172                        </when>
173                        <when value="regex">
174                            <param name="regex_pattern" type="text" value="" label="regex pattern">
175                                <sanitizer sanitize="False"/>
176                            </param>
177                            <param name="regex_action" type="select" label="action for regex match">
178                                <option value="exclude_match">exclude line on pattern match</option>
179                                <option value="include_match">include line on pattern match</option>
180                                <option value="exclude_find">exclude line if pattern found</option>
181                                <option value="include_find">include line if pattern found</option>
182                            </param>
183                        </when>
184                        <when value="select_columns">
185                            <param name="columns" type="text" value="" label="enter column numbers to keep"
186                                help="example: 1,4,2 or c1,c4,c2(selects the first,fourth, and second columns)">
187                                <validator type="regex" message="Column ordinal positions separated by commas">^(c?[1-9]\d*)(,c?[1-9]\d*)*$</validator>
188                            </param>
189                        </when>
190                        <when value="select_column_slices">
191                            <param name="columns" type="text" value="" label="enter indices or slices of the columns to keep">
192                                <help><![CDATA[ Python offset indexes or slices.  Examples:
193                                 <ul>
194                                 <li>Column offset indexes: 0,3,1 (selects the first,fourth, and second columns)</li>
195                                 <li>Negative column numbers: -1,-2 (selects the last, and second last columns)</li>
196                                 <li>python slices ( slice(start, stop[, step]) select a range of columns): <li>
197                                   <ul>
198                                     <li>0:3 or :3 (selects the first 3 columns)</li>
199                                     <li>3:5 (selects the fourth and fifth columns)</li>
200                                     <li>2: (selects all columns after the second)</li>
201                                     <li>-2: (selects the last 2 columns)</li>
202                                     <li>2::-1 (selects the first 3 columns n reverse order: third,second,first)</li>
203                                   </ul>
204                                 </ul>
205                                ]]></help>
206                                <validator type="regex" message="Column ordinal positions separated by commas">^(-?[1-9]\d*|((-?\d+)?:(-?\d*(:-?\d*)?)))(,(-?[1-9]\d*|((-?\d+)?:(-?\d*(:-?\d*)?))))*$</validator>
207                            </param>
208                        </when>
209
210                        <when value="replace">
211                            <param name="column" type="text" value="" label="enter column number to replace"
212                                help="example: 1 or c1 (selects the first column)">
213                                <validator type="regex" message="Column ordinal position">^(c?[1-9]\d*)$</validator>
214                            </param>
215                            <param name="regex_pattern" type="text" value="" label="regex pattern">
216                                <sanitizer sanitize="False"/>
217                            </param>
218                            <param name="regex_replace" type="text" value="" label="replacement expression">
219                                <sanitizer sanitize="False"/>
220                            </param>
221                            <param name="add" type="select" optional="true" label="Instead of replacing, Add as new column:">
222                                <option value="prepend">prepend to beginning of row</option>
223                                <option value="append">append to the end of row</option>
224                                <option value="before">insert before column field</option>
225                                <option value="after">insert after column field</option>
226                            </param>
227                        </when>
228                        <when value="normalize">
229                            <param name="columns" type="text" value="" label="enter column numbers to normalize">
230                                <help><![CDATA[
231                                example: 2,4 or c2,c4 (selects the second, and fourth columns)
232                                If multiple columns are selected, they should have the same length and separator on each line
233                                ]]></help>
234                                <validator type="regex" message="Column ordinal positions separated by commas">^(c?[1-9]\d*)(,c?[1-9]\d*)*$</validator>
235                            </param>
236                            <param name="separator" type="text" value="," label="List item delimiter in column">
237                                <sanitizer sanitize="False"/>
238                                <validator type="regex" message="Anything but TAB or Newline">^[^\t\n\r\f\v]+$</validator>
239                            </param>
240                        </when>
241                    </conditional>
242                </repeat>
243  </xml>
244
245  <token name="@LINEFILTERS_HELP@">
246<![CDATA[
247**Input Line Filters**
248
249  As a tabular file is being read, line filters may be applied:
250
251    - skip leading lines                - skip the first *number* of lines
252    - comment char                      - omit any lines that start with the specified comment character 
253    - by regex expression matching      - *include/exclude* lines that match the regex expression 
254    - select columns                    - choose to include only selected columns in the order specified 
255    - select columns by indices/slices  - *indices or slices* of the columns to keep (python_list_ indexing)
256    - regex replace value in column     - replace a field in a column using a regex substitution (good for date reformatting)
257    - regex replace value in column     - add a new column using a regex substitution of a column value
258    - prepend a line number column      - each line has the *ordinal* value of the line read by this filter as the first column
259    - append a line number column       - each line has the *ordinal* value of the line read by this filter as the last column
260    - prepend a text column             - each line has the text string as the first column
261    - append a text column              - each line has the text string as the last column
262    - prepend the dataset name          - each line has the *dataset name* as the first column
263    - append the dataset name           - each line has the *dataset name* as the last column
264    - normalize list columns            - replicates the line for each item in the specified list *columns*
265
266.. _python_list: https://docs.python.org/3/library/stdtypes.html#common-sequence-operations
267]]>
268  </token>
269
270  <token name="@LINEFILTERS_HELP_EXAMPLE@">
271<![CDATA[
272**Line Filtering Example** 
273  *(Six filters are applied as the following file is read)*
274
275  ::
276
277    Input Tabular File:
278
279    #People with pets
280    Pets FirstName           LastName   DOB       PetNames  PetType
281    2    Paula               Brown      24/05/78  Rex,Fluff dog,cat
282    1    Steven              Jones      04/04/74  Allie     cat
283    0    Jane                Doe        24/05/78  
284    1    James               Smith      20/10/80  Spot
285
286
287    Filter 1 - append a line number column:
288
289    #People with pets                                                 1
290    Pets FirstName           LastName   DOB       PetNames  PetType   2
291    2    Paula               Brown      24/05/78  Rex,Fluff dog,cat   3
292    1    Steven              Jones      04/04/74  Allie     cat       4
293    0    Jane                Doe        24/05/78                      5
294    1    James               Smith      20/10/80  Spot                6
295
296    Filter 2 - by regex expression matching [include]: '^\d+' (include lines that start with a number) 
297
298    2    Paula               Brown      24/05/78  Rex,Fluff dog,cat   3
299    1    Steven              Jones      04/04/74  Allie     cat       4
300    0    Jane                Doe        24/05/78                      5
301    1    James               Smith      20/10/80  Spot                6
302
303    Filter 3 - append a line number column:
304
305    2    Paula               Brown      24/05/78  Rex,Fluff dog,cat   3  1
306    1    Steven              Jones      04/04/74  Allie     cat       4  2
307    0    Jane                Doe        24/05/78                      5  3
308    1    James               Smith      20/10/80  Spot                6  4
309
310    Filter 4 - regex replace value in column[4]: '(\d+)/(\d+)/(\d+)' '19\3-\2-\1' (convert dates to sqlite format) 
311
312    2    Paula               Brown      1978-05-24  Rex,Fluff dog,cat   3  1
313    1    Steven              Jones      1974-04-04  Allie     cat       4  2
314    0    Jane                Doe        1978-05-24                      5  3
315    1    James               Smith      1980-10-20  Spot                6  4
316
317    Filter 5 - normalize list columns[5,6]:
318
319    2    Paula               Brown      1978-05-24  Rex       dog       3  1
320    2    Paula               Brown      1978-05-24  Fluff     cat       3  1
321    1    Steven              Jones      1974-04-04  Allie     cat       4  2
322    0    Jane                Doe        1978-05-24                      5  3
323    1    James               Smith      1980-10-20  Spot                6  4
324
325    Filter 6 - select columns by indices/slices: '1:6'
326
327    Paula               Brown      1978-05-24  Rex       dog
328    Paula               Brown      1978-05-24  Fluff     cat
329    Steven              Jones      1974-04-04  Allie     cat
330    Jane                Doe        1978-05-24               
331    James               Smith      1980-10-20  Spot         
332
333]]>
334  </token>
335
336  <token name="@QUERY_HELP@">
337<![CDATA[
338
339For help in using SQLite_ see:  http://www.sqlite.org/docs.html
340
341**NOTE:** input for SQLite dates input field must be in the format: *YYYY-MM-DD* for example: 2015-09-30
342
343See: http://www.sqlite.org/lang_datefunc.html
344
345**Example** 
346
347  Given 2 tabular datasets: *customers* and *sales*
348  
349   Dataset *customers*
350  
351    Table name: "customers"
352  
353    Column names: "CustomerID,FirstName,LastName,Email,DOB,Phone"
354  
355    =========== ========== ========== ===================== ========== ============
356    #CustomerID FirstName  LastName   Email                 DOB        Phone
357    =========== ========== ========== ===================== ========== ============
358    1           John       Smith      John.Smith@yahoo.com  1968-02-04 626 222-2222
359    2           Steven     Goldfish   goldfish@fishhere.net 1974-04-04 323 455-4545
360    3           Paula      Brown      pb@herowndomain.org   1978-05-24 416 323-3232
361    4           James      Smith      jim@supergig.co.uk    1980-10-20 416 323-8888
362    =========== ========== ========== ===================== ========== ============
363  
364   Dataset *sales*
365  
366    Table name: "sales"
367  
368    Column names: "CustomerID,Date,SaleAmount"
369  
370    =============  ============  ============
371      #CustomerID    Date          SaleAmount
372    =============  ============  ============
373               2    2004-05-06         100.22
374               1    2004-05-07          99.95
375               3    2004-05-07         122.95
376               3    2004-05-13         100.00
377               4    2004-05-22         555.55
378    =============  ============  ============
379  
380  The query
381  
382  ::
383  
384    SELECT FirstName,LastName,sum(SaleAmount) as "TotalSales" 
385    FROM customers join sales on customers.CustomerID = sales.CustomerID 
386    GROUP BY customers.CustomerID ORDER BY TotalSales DESC;
387  
388  Produces this tabular output:
389  
390    ========== ======== ==========
391    #FirstName LastName TotalSales
392    ========== ======== ==========
393    James      Smith    555.55
394    Paula      Brown    222.95
395    Steven     Goldfish 100.22
396    John       Smith    99.95
397    ========== ======== ==========
398  
399  
400  If the optional Table name and Column names inputs are not used, the query would be:
401  
402  ::
403  
404    SELECT t1.c2 as "FirstName", t1.c3 as "LastName", sum(t2.c3) as "TotalSales" 
405    FROM t1 join t2 on t1.c1 = t2.c1 
406    GROUP BY t1.c1 ORDER BY TotalSales DESC;
407  
408  You can selectively name columns, e.g. on the customers input you could just name columns 2,3, and 5: 
409  
410    Column names: ,FirstName,LastName,,BirthDate
411  
412    Results in the following data base table
413  
414    =========== ========== ========== ===================== ========== ============
415    #c1         FirstName  LastName   c4                    BirthDate  c6
416    =========== ========== ========== ===================== ========== ============
417    1           John       Smith      John.Smith@yahoo.com  1968-02-04 626 222-2222
418    2           Steven     Goldfish   goldfish@fishhere.net 1974-04-04 323 455-4545
419    3           Paula      Brown      pb@herowndomain.org   1978-05-24 416 323-3232
420    4           James      Smith      jim@supergig.co.uk    1980-10-20 416 323-8888
421    =========== ========== ========== ===================== ========== ============
422
423
424   
425  Regular_expression_ functions are included for: 
426
427  ::
428
429    matching:      re_match('pattern',column) 
430
431    SELECT t1.FirstName, t1.LastName
432    FROM t1
433    WHERE re_match('^.*\.(net|org)$',c4)
434
435  Results:
436
437    =========== ==========
438    #FirstName  LastName
439    =========== ==========
440    Steven      Goldfish
441    Paula       Brown
442    =========== ==========
443
444
445  ::
446
447    searching:     re_search('pattern',column)
448    substituting:  re_sub('pattern','replacement,column)
449
450    SELECT t1.FirstName, t1.LastName, re_sub('^\d{2}(\d{2})-(\d\d)-(\d\d)','\3/\2/\1',BirthDate) as "DOB"
451    FROM t1
452    WHERE re_search('[hp]er',c4)
453
454  Results:
455
456
457    =========== ========== ==========
458    #FirstName  LastName   DOB
459    =========== ========== ==========
460    Steven      Goldfish   04/04/74
461    Paula       Brown      24/05/78
462    James       Smith      20/10/80
463    =========== ========== ==========
464
465  Math_ functions *( python math library: https://docs.python.org/3.6/library/math.html )*:
466
467  acos(x), acosh(x), asin(x), asinh(x), atan(x), atanh(x), atan2(x, y), ceil(x), cos(x), cosh(x), degrees(x), exp(x), expm1(x), fabs(x), floor(x), fmod(x, y), log(b,x), log(x), log10(x), log1p(x), log2(x), mod(x, y), pow(x, y), radians(x), sin(x), sinh(x), sqrt(x), tan(x), tanh(x), trunc(x)
468
469
470  ::
471
472    Query:
473
474    SELECT SaleAmount, floor(SaleAmount) as "dollars"
475    FROM sales
476
477  Results:
478    
479    ============  ========
480      SaleAmount   dollars
481    ============  ========
482          100.22       100
483           99.95        99
484          122.95       122
485          100.00       100
486          555.55       555
487    ============  ========
488  
489
490.. _Regular_expression: https://docs.python.org/3.9/library/re.html
491.. _Math: https://docs.python.org/3.9/library/math.html
492.. _SQLite: http://www.sqlite.org/index.html
493.. _SQLite_functions: http://www.sqlite.org/docs.html
494
495
496]]>
497  </token>
498  <xml name="citations">
499    <citations>
500      <citation type="doi">10.12688/f1000research.16450.1</citation>
501      <yield />
502    </citations>
503  </xml>
504
505
506</macros>
507