upstream/tools/query_tabular/macros.xml
1<macros>
2 <token name="@LINEFILTERS@">
3<![CDATA[
4 ## set linefilters to the
5 #set $input_filters = []
6 #for $fi in $linefilters:
7 #if $fi.filter.filter_type == 'skip':
8 #set $skip_lines = None
9 #if str($fi.filter.skip_lines) != '':
10 #set $skip_lines = int($fi.filter.skip_lines)
11 #elif $table.metadata.comment_lines and int($table.metadata.comment_lines) > 0:
12 #set $skip_lines = int($table.metadata.comment_lines)
13 #end if
14 #if $skip_lines is not None:
15 #set $filter_dict = dict()
16 #set $filter_dict['filter'] = str($fi.filter.filter_type)
17 #set $filter_dict['count'] = $skip_lines
18 #silent $input_filters.append($filter_dict)
19 #end if
20 #elif $fi.filter.filter_type == 'comment':
21 #set $filter_dict = dict()
22 #set $filter_dict['filter'] = 'regex'
23 #set $filter_dict['pattern'] = '^(%s).*$' % '|'.join([chr(int(x)).replace('|','[|]') for x in (str($fi.filter.comment_char)).split(',')])
24 #set $filter_dict['action'] = 'exclude_match'
25 #silent $input_filters.append($filter_dict)
26 #elif $fi.filter.filter_type == 'regex':
27 #set $filter_dict = dict()
28 #set $filter_dict['filter'] = str($fi.filter.filter_type)
29 #set $filter_dict['pattern'] = str($fi.filter.regex_pattern)
30 #set $filter_dict['action'] = str($fi.filter.regex_action)
31 #silent $input_filters.append($filter_dict)
32 #elif $fi.filter.filter_type == 'select_columns':
33 #set $filter_dict = dict()
34 #set $filter_dict['filter'] = str($fi.filter.filter_type)
35 #set $filter_dict['columns'] = [int($c) for $c in str($fi.filter.columns).replace('c','').split(',')]
36 #silent $input_filters.append($filter_dict)
37 #elif $fi.filter.filter_type == 'select_column_slices':
38 #set $filter_dict = dict()
39 #set $filter_dict['filter'] = str($fi.filter.filter_type)
40 #set $filter_dict['columns'] = [$c for $c in str($fi.filter.columns).split(',')]
41 #silent $input_filters.append($filter_dict)
42 #elif $fi.filter.filter_type == 'replace':
43 #set $filter_dict = dict()
44 #set $filter_dict['filter'] = str($fi.filter.filter_type)
45 #set $filter_dict['column'] = int(str($fi.filter.column).replace('c',''))
46 #set $filter_dict['pattern'] = str($fi.filter.regex_pattern)
47 #set $filter_dict['replace'] = str($fi.filter.regex_replace)
48 #if $fi.filter.add:
49 #set $filter_dict['add'] = str($fi.filter.add)
50 #end if
51 #silent $input_filters.append($filter_dict)
52 #elif str($fi.filter.filter_type).endswith('pend_line_num'):
53 #set $filter_dict = dict()
54 #set $filter_dict['filter'] = str($fi.filter.filter_type)
55 #silent $input_filters.append($filter_dict)
56 #elif str($fi.filter.filter_type).endswith('pend_text'):
57 #set $filter_dict = dict()
58 #set $filter_dict['filter'] = str($fi.filter.filter_type)
59 #set $filter_dict['column_text'] = str($fi.filter.column_text)
60 #silent $input_filters.append($filter_dict)
61 #elif str($fi.filter.filter_type).endswith('pend_dataset_name'):
62 #set $filter_dict = dict()
63 #set $filter_dict['filter'] = str($fi.filter.filter_type).replace('dataset_name', 'text')
64 #set $filter_dict['column_text'] = $dataset_name
65 #silent $input_filters.append($filter_dict)
66 #elif $fi.filter.filter_type == 'normalize':
67 #set $filter_dict = dict()
68 #set $filter_dict['filter'] = str($fi.filter.filter_type)
69 #set $filter_dict['columns'] = [int(str($ci).replace('c','')) for $ci in str($fi.filter.columns).split(',')]
70 #set $filter_dict['separator'] = str($fi.filter.separator)
71 #silent $input_filters.append($filter_dict)
72 #end if
73 #end for
74]]>
75 </token>
76 <token name="@RESULT_HEADER@">
77<![CDATA[
78 #if $query_result.header == 'yes':
79 #if $query_result.header_prefix:
80 #set $header_prefix = chr(int(str($query_result.header_prefix)))
81 --comment_char='$header_prefix'
82 #end if
83 #else
84 --no_header
85 #end if
86]]>
87 </token>
88 <xml name="requirements">
89 <requirements>
90 <requirement type="package" version="3.7">python</requirement>
91 </requirements>
92 </xml>
93 <xml name="comment_char_options">
94 <option value="62">></option>
95 <option value="64">@</option>
96 <option value="43">+</option>
97 <option value="60"><</option>
98 <option value="42">*</option>
99 <option value="45">-</option>
100 <option value="61">=</option>
101 <option value="124">|</option>
102 <option value="63">?</option>
103 <option value="36">$</option>
104 <option value="46">.</option>
105 <option value="58">:</option>
106 <option value="38">&</option>
107 <option value="37">%</option>
108 <option value="94">^</option>
109 <option value="35" selected="true">#</option>
110 <option value="33">!</option>
111 </xml>
112 <xml name="result_results_header_line">
113 <conditional name="query_result">
114 <param name="header" type="select" label="include query result column headers">
115 <option value="yes">Yes</option>
116 <option value="no">No</option>
117 </param>
118 <when value="yes">
119 <param name="header_prefix" type="select" optional="true" label="Prefix character for column_header line">
120 <option value="">no comment character prefix</option>
121 <expand macro="comment_char_options" />
122 </param>
123 </when>
124 <when value="no"/>
125 </conditional>
126 </xml>
127 <xml name="sql_query_input">
128 <param name="sqlquery" type="text" area="true" size="20x80" value="" optional="true" label="SQL Query to generate tabular output">
129 <help>By default: tables are named: t1,t2,...,tn and columns in each table: c1,c2,...,cn</help>
130 <sanitizer sanitize="False"/>
131 <validator type="regex" message="">(?ims)^\s*select\s+.*\s+from\s+.*$</validator>
132 </param>
133 </xml>
134 <xml name="macro_line_filters">
135 <repeat name="linefilters" title="Filter Tabular Input Lines">
136 <conditional name="filter">
137 <param name="filter_type" type="select" label="Filter By">
138 <option value="skip">skip leading lines</option>
139 <option value="comment">comment char</option>
140 <option value="regex">by regex expression matching</option>
141 <option value="select_columns">select columns</option>
142 <option value="select_column_slices">select columns by indices/slices</option>
143 <option value="replace">regex replace value in column</option>
144 <option value="prepend_line_num">prepend a line number column</option>
145 <option value="append_line_num">append a line number column</option>
146 <option value="prepend_dataset_name">prepend a column with the dataset name</option>
147 <option value="append_dataset_name">append a column with the dataset name</option>
148 <option value="prepend_text">prepend a column with the given text</option>
149 <option value="append_text">append a column with the given text</option>
150 <option value="normalize">normalize list columns, replicates row for each item in list</option>
151 </param>
152 <when value="skip">
153 <param name="skip_lines" type="integer" value="" min="0" optional="true" label="Skip lines"
154 help="Leave blank to use the comment lines metadata for this dataset" />
155 </when>
156 <when value="comment">
157 <param name="comment_char" type="select" display="checkboxes" multiple="True" label="Ignore lines beginning with these characters" help="lines beginning with these are skipped">
158 <expand macro="comment_char_options" />
159 </param>
160 </when>
161 <when value="prepend_line_num"/>
162 <when value="append_line_num"/>
163 <when value="prepend_dataset_name"/>
164 <when value="append_dataset_name"/>
165 <when value="prepend_text">
166 <param name="column_text" type="text" value="" label="text for column">
167 </param>
168 </when>
169 <when value="append_text">
170 <param name="column_text" type="text" value="" label="text for column">
171 </param>
172 </when>
173 <when value="regex">
174 <param name="regex_pattern" type="text" value="" label="regex pattern">
175 <sanitizer sanitize="False"/>
176 </param>
177 <param name="regex_action" type="select" label="action for regex match">
178 <option value="exclude_match">exclude line on pattern match</option>
179 <option value="include_match">include line on pattern match</option>
180 <option value="exclude_find">exclude line if pattern found</option>
181 <option value="include_find">include line if pattern found</option>
182 </param>
183 </when>
184 <when value="select_columns">
185 <param name="columns" type="text" value="" label="enter column numbers to keep"
186 help="example: 1,4,2 or c1,c4,c2(selects the first,fourth, and second columns)">
187 <validator type="regex" message="Column ordinal positions separated by commas">^(c?[1-9]\d*)(,c?[1-9]\d*)*$</validator>
188 </param>
189 </when>
190 <when value="select_column_slices">
191 <param name="columns" type="text" value="" label="enter indices or slices of the columns to keep">
192 <help><![CDATA[ Python offset indexes or slices. Examples:
193 <ul>
194 <li>Column offset indexes: 0,3,1 (selects the first,fourth, and second columns)</li>
195 <li>Negative column numbers: -1,-2 (selects the last, and second last columns)</li>
196 <li>python slices ( slice(start, stop[, step]) select a range of columns): <li>
197 <ul>
198 <li>0:3 or :3 (selects the first 3 columns)</li>
199 <li>3:5 (selects the fourth and fifth columns)</li>
200 <li>2: (selects all columns after the second)</li>
201 <li>-2: (selects the last 2 columns)</li>
202 <li>2::-1 (selects the first 3 columns n reverse order: third,second,first)</li>
203 </ul>
204 </ul>
205 ]]></help>
206 <validator type="regex" message="Column ordinal positions separated by commas">^(-?[1-9]\d*|((-?\d+)?:(-?\d*(:-?\d*)?)))(,(-?[1-9]\d*|((-?\d+)?:(-?\d*(:-?\d*)?))))*$</validator>
207 </param>
208 </when>
209
210 <when value="replace">
211 <param name="column" type="text" value="" label="enter column number to replace"
212 help="example: 1 or c1 (selects the first column)">
213 <validator type="regex" message="Column ordinal position">^(c?[1-9]\d*)$</validator>
214 </param>
215 <param name="regex_pattern" type="text" value="" label="regex pattern">
216 <sanitizer sanitize="False"/>
217 </param>
218 <param name="regex_replace" type="text" value="" label="replacement expression">
219 <sanitizer sanitize="False"/>
220 </param>
221 <param name="add" type="select" optional="true" label="Instead of replacing, Add as new column:">
222 <option value="prepend">prepend to beginning of row</option>
223 <option value="append">append to the end of row</option>
224 <option value="before">insert before column field</option>
225 <option value="after">insert after column field</option>
226 </param>
227 </when>
228 <when value="normalize">
229 <param name="columns" type="text" value="" label="enter column numbers to normalize">
230 <help><![CDATA[
231 example: 2,4 or c2,c4 (selects the second, and fourth columns)
232 If multiple columns are selected, they should have the same length and separator on each line
233 ]]></help>
234 <validator type="regex" message="Column ordinal positions separated by commas">^(c?[1-9]\d*)(,c?[1-9]\d*)*$</validator>
235 </param>
236 <param name="separator" type="text" value="," label="List item delimiter in column">
237 <sanitizer sanitize="False"/>
238 <validator type="regex" message="Anything but TAB or Newline">^[^\t\n\r\f\v]+$</validator>
239 </param>
240 </when>
241 </conditional>
242 </repeat>
243 </xml>
244
245 <token name="@LINEFILTERS_HELP@">
246<![CDATA[
247**Input Line Filters**
248
249 As a tabular file is being read, line filters may be applied:
250
251 - skip leading lines - skip the first *number* of lines
252 - comment char - omit any lines that start with the specified comment character
253 - by regex expression matching - *include/exclude* lines that match the regex expression
254 - select columns - choose to include only selected columns in the order specified
255 - select columns by indices/slices - *indices or slices* of the columns to keep (python_list_ indexing)
256 - regex replace value in column - replace a field in a column using a regex substitution (good for date reformatting)
257 - regex replace value in column - add a new column using a regex substitution of a column value
258 - prepend a line number column - each line has the *ordinal* value of the line read by this filter as the first column
259 - append a line number column - each line has the *ordinal* value of the line read by this filter as the last column
260 - prepend a text column - each line has the text string as the first column
261 - append a text column - each line has the text string as the last column
262 - prepend the dataset name - each line has the *dataset name* as the first column
263 - append the dataset name - each line has the *dataset name* as the last column
264 - normalize list columns - replicates the line for each item in the specified list *columns*
265
266.. _python_list: https://docs.python.org/3/library/stdtypes.html#common-sequence-operations
267]]>
268 </token>
269
270 <token name="@LINEFILTERS_HELP_EXAMPLE@">
271<![CDATA[
272**Line Filtering Example**
273 *(Six filters are applied as the following file is read)*
274
275 ::
276
277 Input Tabular File:
278
279 #People with pets
280 Pets FirstName LastName DOB PetNames PetType
281 2 Paula Brown 24/05/78 Rex,Fluff dog,cat
282 1 Steven Jones 04/04/74 Allie cat
283 0 Jane Doe 24/05/78
284 1 James Smith 20/10/80 Spot
285
286
287 Filter 1 - append a line number column:
288
289 #People with pets 1
290 Pets FirstName LastName DOB PetNames PetType 2
291 2 Paula Brown 24/05/78 Rex,Fluff dog,cat 3
292 1 Steven Jones 04/04/74 Allie cat 4
293 0 Jane Doe 24/05/78 5
294 1 James Smith 20/10/80 Spot 6
295
296 Filter 2 - by regex expression matching [include]: '^\d+' (include lines that start with a number)
297
298 2 Paula Brown 24/05/78 Rex,Fluff dog,cat 3
299 1 Steven Jones 04/04/74 Allie cat 4
300 0 Jane Doe 24/05/78 5
301 1 James Smith 20/10/80 Spot 6
302
303 Filter 3 - append a line number column:
304
305 2 Paula Brown 24/05/78 Rex,Fluff dog,cat 3 1
306 1 Steven Jones 04/04/74 Allie cat 4 2
307 0 Jane Doe 24/05/78 5 3
308 1 James Smith 20/10/80 Spot 6 4
309
310 Filter 4 - regex replace value in column[4]: '(\d+)/(\d+)/(\d+)' '19\3-\2-\1' (convert dates to sqlite format)
311
312 2 Paula Brown 1978-05-24 Rex,Fluff dog,cat 3 1
313 1 Steven Jones 1974-04-04 Allie cat 4 2
314 0 Jane Doe 1978-05-24 5 3
315 1 James Smith 1980-10-20 Spot 6 4
316
317 Filter 5 - normalize list columns[5,6]:
318
319 2 Paula Brown 1978-05-24 Rex dog 3 1
320 2 Paula Brown 1978-05-24 Fluff cat 3 1
321 1 Steven Jones 1974-04-04 Allie cat 4 2
322 0 Jane Doe 1978-05-24 5 3
323 1 James Smith 1980-10-20 Spot 6 4
324
325 Filter 6 - select columns by indices/slices: '1:6'
326
327 Paula Brown 1978-05-24 Rex dog
328 Paula Brown 1978-05-24 Fluff cat
329 Steven Jones 1974-04-04 Allie cat
330 Jane Doe 1978-05-24
331 James Smith 1980-10-20 Spot
332
333]]>
334 </token>
335
336 <token name="@QUERY_HELP@">
337<![CDATA[
338
339For help in using SQLite_ see: http://www.sqlite.org/docs.html
340
341**NOTE:** input for SQLite dates input field must be in the format: *YYYY-MM-DD* for example: 2015-09-30
342
343See: http://www.sqlite.org/lang_datefunc.html
344
345**Example**
346
347 Given 2 tabular datasets: *customers* and *sales*
348
349 Dataset *customers*
350
351 Table name: "customers"
352
353 Column names: "CustomerID,FirstName,LastName,Email,DOB,Phone"
354
355 =========== ========== ========== ===================== ========== ============
356 #CustomerID FirstName LastName Email DOB Phone
357 =========== ========== ========== ===================== ========== ============
358 1 John Smith John.Smith@yahoo.com 1968-02-04 626 222-2222
359 2 Steven Goldfish goldfish@fishhere.net 1974-04-04 323 455-4545
360 3 Paula Brown pb@herowndomain.org 1978-05-24 416 323-3232
361 4 James Smith jim@supergig.co.uk 1980-10-20 416 323-8888
362 =========== ========== ========== ===================== ========== ============
363
364 Dataset *sales*
365
366 Table name: "sales"
367
368 Column names: "CustomerID,Date,SaleAmount"
369
370 ============= ============ ============
371 #CustomerID Date SaleAmount
372 ============= ============ ============
373 2 2004-05-06 100.22
374 1 2004-05-07 99.95
375 3 2004-05-07 122.95
376 3 2004-05-13 100.00
377 4 2004-05-22 555.55
378 ============= ============ ============
379
380 The query
381
382 ::
383
384 SELECT FirstName,LastName,sum(SaleAmount) as "TotalSales"
385 FROM customers join sales on customers.CustomerID = sales.CustomerID
386 GROUP BY customers.CustomerID ORDER BY TotalSales DESC;
387
388 Produces this tabular output:
389
390 ========== ======== ==========
391 #FirstName LastName TotalSales
392 ========== ======== ==========
393 James Smith 555.55
394 Paula Brown 222.95
395 Steven Goldfish 100.22
396 John Smith 99.95
397 ========== ======== ==========
398
399
400 If the optional Table name and Column names inputs are not used, the query would be:
401
402 ::
403
404 SELECT t1.c2 as "FirstName", t1.c3 as "LastName", sum(t2.c3) as "TotalSales"
405 FROM t1 join t2 on t1.c1 = t2.c1
406 GROUP BY t1.c1 ORDER BY TotalSales DESC;
407
408 You can selectively name columns, e.g. on the customers input you could just name columns 2,3, and 5:
409
410 Column names: ,FirstName,LastName,,BirthDate
411
412 Results in the following data base table
413
414 =========== ========== ========== ===================== ========== ============
415 #c1 FirstName LastName c4 BirthDate c6
416 =========== ========== ========== ===================== ========== ============
417 1 John Smith John.Smith@yahoo.com 1968-02-04 626 222-2222
418 2 Steven Goldfish goldfish@fishhere.net 1974-04-04 323 455-4545
419 3 Paula Brown pb@herowndomain.org 1978-05-24 416 323-3232
420 4 James Smith jim@supergig.co.uk 1980-10-20 416 323-8888
421 =========== ========== ========== ===================== ========== ============
422
423
424
425 Regular_expression_ functions are included for:
426
427 ::
428
429 matching: re_match('pattern',column)
430
431 SELECT t1.FirstName, t1.LastName
432 FROM t1
433 WHERE re_match('^.*\.(net|org)$',c4)
434
435 Results:
436
437 =========== ==========
438 #FirstName LastName
439 =========== ==========
440 Steven Goldfish
441 Paula Brown
442 =========== ==========
443
444
445 ::
446
447 searching: re_search('pattern',column)
448 substituting: re_sub('pattern','replacement,column)
449
450 SELECT t1.FirstName, t1.LastName, re_sub('^\d{2}(\d{2})-(\d\d)-(\d\d)','\3/\2/\1',BirthDate) as "DOB"
451 FROM t1
452 WHERE re_search('[hp]er',c4)
453
454 Results:
455
456
457 =========== ========== ==========
458 #FirstName LastName DOB
459 =========== ========== ==========
460 Steven Goldfish 04/04/74
461 Paula Brown 24/05/78
462 James Smith 20/10/80
463 =========== ========== ==========
464
465 Math_ functions *( python math library: https://docs.python.org/3.6/library/math.html )*:
466
467 acos(x), acosh(x), asin(x), asinh(x), atan(x), atanh(x), atan2(x, y), ceil(x), cos(x), cosh(x), degrees(x), exp(x), expm1(x), fabs(x), floor(x), fmod(x, y), log(b,x), log(x), log10(x), log1p(x), log2(x), mod(x, y), pow(x, y), radians(x), sin(x), sinh(x), sqrt(x), tan(x), tanh(x), trunc(x)
468
469
470 ::
471
472 Query:
473
474 SELECT SaleAmount, floor(SaleAmount) as "dollars"
475 FROM sales
476
477 Results:
478
479 ============ ========
480 SaleAmount dollars
481 ============ ========
482 100.22 100
483 99.95 99
484 122.95 122
485 100.00 100
486 555.55 555
487 ============ ========
488
489
490.. _Regular_expression: https://docs.python.org/3.9/library/re.html
491.. _Math: https://docs.python.org/3.9/library/math.html
492.. _SQLite: http://www.sqlite.org/index.html
493.. _SQLite_functions: http://www.sqlite.org/docs.html
494
495
496]]>
497 </token>
498 <xml name="citations">
499 <citations>
500 <citation type="doi">10.12688/f1000research.16450.1</citation>
501 <yield />
502 </citations>
503 </xml>
504
505
506</macros>
507