upstream-extended/iuc/tools/fixed2tab/fixed2tab.xml
1<tool id="fixed2tab" name="fixed2tab" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="@PROFILE@">
2 <description>convert fixed-width text to tabular</description>
3 <macros>
4 <import>macros.xml</import>
5 </macros>
6 <expand macro="edam"/>
7 <expand macro="requirements"/>
8 <version_command>fixed2tab --version</version_command>
9 <command detect_errors="exit_code"><![CDATA[
10fixed2tab
11 --input '$input'
12 --table '$table'
13 --report '$report'
14 --rejected '$rejected'
15 #if str($geometry.mode) == 'columns':
16 --columns '$geometry.columns'
17 #elif str($geometry.mode) == 'widths':
18 --widths '$geometry.widths'
19 #end if
20 --min-gutter $detection.min_gutter
21 #if str($detection.record_width):
22 --record-width $detection.record_width
23 #end if
24 #if str($detection.header_line):
25 --header-line $detection.header_line
26 #end if
27 --encoding '$reading.encoding'
28 --short-lines '$reading.short_lines'
29 $output_shape.collapse_spaces
30 --header '$output_shape.header'
31 #if str($strictness.mode) == 'strict':
32 --strict
33 #if $strictness.allow_rejects:
34 --allow-rejects
35 #for $reason in $strictness.allow_rejects:
36 '$reason'
37 #end for
38 #end if
39 #end if
40 ]]></command>
41 <inputs>
42 <param argument="--input" type="data" format="txt" label="Fixed-width text file" help="Do not upload it with 'convert spaces to tabs': that destroys the column positions."/>
43 <conditional name="geometry">
44 <param name="mode" type="select" label="Column boundaries">
45 <option value="auto" selected="true">Detect automatically</option>
46 <option value="columns">Give the column ranges (--columns)</option>
47 <option value="widths">Give the column widths (--widths)</option>
48 </param>
49 <when value="auto"/>
50 <when value="columns">
51 <param argument="--columns" type="text" value="" label="Column ranges" help="START-END[:NAME] separated by commas, for example 1-12:DATE,18-27:BEST_TIME. Positions start at 1, and both ends are included. The report of an earlier run prints this string: copy the part between the quotes.">
52 <validator type="regex" message="Write START-END or START-END:NAME, separated by commas. Names use letters, digits and underscores only.">^\s*\d+-\d+(:[A-Za-z0-9_]+)?(\s*,\s*\d+-\d+(:[A-Za-z0-9_]+)?)*\s*$</validator>
53 </param>
54 </when>
55 <when value="widths">
56 <param argument="--widths" type="text" value="" label="Column widths" help="Read from position 1: N takes the next N positions as a column, N:NAME does the same and names it, and Nx skips N positions. For example 12:DATE,5x,10:BEST_TIME. A FORTRAN FORMAT such as 2X,I5,2X,F8.3 transcribes directly to 2x,5,2x,8.">
57 <validator type="regex" message="Write N, N:NAME or Nx, separated by commas. Names use letters, digits and underscores only.">^\s*(\d*[xX]|\d+(:[A-Za-z0-9_]+)?)(\s*,\s*(\d*[xX]|\d+(:[A-Za-z0-9_]+)?))*\s*$</validator>
58 </param>
59 </when>
60 </conditional>
61 <section name="detection" title="Detection" expanded="false">
62 <param argument="--min-gutter" type="integer" min="1" value="2" label="Blank positions that separate two columns" help="Lower values give more columns, because a smaller blank run separates two columns. Higher values merge narrow columns."/>
63 <param argument="--record-width" type="integer" min="1" optional="true" label="Record width" help="Leave empty to detect it. Set it for a file whose trailing blanks were stripped."/>
64 <param argument="--header-line" type="integer" min="0" optional="true" label="Line with the column headings" help="Leave empty to detect it. Lines 1 to N are preamble and are not converted. 0 means that the file has no preamble. Required when short lines are padded."/>
65 </section>
66 <section name="reading" title="Reading" expanded="false">
67 <param argument="--encoding" type="select" label="Input encoding" help="Try latin-1 for byte-padded legacy output.">
68 <option value="utf-8" selected="true">UTF-8</option>
69 <option value="latin-1">Latin-1 (ISO-8859-1)</option>
70 <option value="cp1252">Windows-1252</option>
71 </param>
72 <param argument="--short-lines" type="select" label="Lines shorter than the record width" help="Padding can make page numbers and footers into data rows, so rejecting is the default.">
73 <option value="reject" selected="true">Reject them</option>
74 <option value="pad">Pad them with spaces</option>
75 </param>
76 </section>
77 <section name="output_shape" title="Output shape" expanded="false">
78 <param argument="--collapse-spaces" type="boolean" truevalue="--collapse-spaces" falsevalue="" checked="false" label="Reduce runs of spaces inside each cell to one space" help="Applies to every cell."/>
79 <param argument="--header" type="select" label="Header row in the table" help="The column names are always in the report. A header row shifts the row positions by one for later steps.">
80 <option value="none" selected="true">No header row</option>
81 <option value="plain">Column names as the first row</option>
82 </param>
83 </section>
84 <conditional name="strictness">
85 <param name="mode" type="select" label="Fail the job when lines are rejected?">
86 <option value="off" selected="true">No</option>
87 <option value="strict">Yes (--strict)</option>
88 </param>
89 <when value="off"/>
90 <when value="strict">
91 <param argument="--allow-rejects" type="select" multiple="true" optional="true" label="Rejection reasons to tolerate">
92 <option value="length">length</option>
93 <option value="control-char">control-char</option>
94 <option value="preamble-identity">preamble-identity</option>
95 <option value="rule-line">rule-line</option>
96 </param>
97 </when>
98 </conditional>
99 </inputs>
100 <outputs>
101 <data name="table" format="tabular" label="${tool.name} on ${on_string}: table"/>
102 <data name="report" format="txt" label="${tool.name} on ${on_string}: report"/>
103 <data name="rejected" format="tabular" label="${tool.name} on ${on_string}: rejected lines"/>
104 </outputs>
105 <tests>
106 <!-- Automatic detection gives the expected table and a reconciled report. -->
107 <test expect_num_outputs="3">
108 <param name="input" value="fixedwidth_input.txt"/>
109 <output name="table" file="fixedwidth_expected.tsv"/>
110 <output name="report">
111 <assert_contents>
112 <has_text text="rows + rejected 8 [OK]"/>
113 <has_text text="--columns '1-12:DATE,18-27:BEST_TIME,"/>
114 </assert_contents>
115 </output>
116 <output name="rejected">
117 <assert_contents>
118 <has_n_lines n="2"/>
119 </assert_contents>
120 </output>
121 </test>
122 <!-- A file with a title block and rule lines: rejected lines and a warning. -->
123 <test expect_num_outputs="3">
124 <param name="input" value="fixedwidth_messy.txt"/>
125 <output name="table">
126 <assert_contents>
127 <has_n_lines n="6"/>
128 </assert_contents>
129 </output>
130 <output name="report">
131 <assert_contents>
132 <has_text text="rows + rejected 16 [OK]"/>
133 <has_text text="[width-share]"/>
134 </assert_contents>
135 </output>
136 <output name="rejected">
137 <assert_contents>
138 <has_n_lines n="10"/>
139 <has_text text="rule-line"/>
140 </assert_contents>
141 </output>
142 </test>
143 <!-- Explicit ranges, as the report prints them, give the same table. -->
144 <test expect_num_outputs="3">
145 <param name="input" value="fixedwidth_input.txt"/>
146 <conditional name="geometry">
147 <param name="mode" value="columns"/>
148 <param name="columns" value="1-12:DATE,18-27:BEST_TIME,31-37:DELTAT_s,42-47:sigma,53-64:Egyptian_Date,70-73:TSun,77-80:TMoon,85-89:q,92-93:code,100-110:NEW_MOON_DATE,115-125:TIME"/>
149 </conditional>
150 <section name="reading">
151 <param name="encoding" value="latin-1"/>
152 </section>
153 <output name="table" file="fixedwidth_expected.tsv"/>
154 </test>
155 <!-- Widths that describe the same columns give the same table. -->
156 <test expect_num_outputs="3">
157 <param name="input" value="fixedwidth_input.txt"/>
158 <conditional name="geometry">
159 <param name="mode" value="widths"/>
160 <param name="widths" value="12:DATE,5x,10:BEST_TIME,3x,7:DELTAT_s,4x,6:sigma,5x,12:Egyptian_Date,5x,4:TSun,3x,4:TMoon,4x,5:q,2x,2:code,6x,11:NEW_MOON_DATE,4x,11:TIME"/>
161 </conditional>
162 <output name="table" file="fixedwidth_expected.tsv"/>
163 </test>
164 <!-- Detection overrides, a header row and collapsed spaces. -->
165 <test expect_num_outputs="3">
166 <param name="input" value="fixedwidth_input.txt"/>
167 <section name="detection">
168 <param name="min_gutter" value="2"/>
169 <param name="record_width" value="128"/>
170 <param name="header_line" value="2"/>
171 </section>
172 <section name="output_shape">
173 <param name="collapse_spaces" value="true"/>
174 <param name="header" value="plain"/>
175 </section>
176 <output name="table">
177 <assert_contents>
178 <has_n_lines n="7"/>
179 <has_line line="DATE	BEST_TIME	DELTAT_s	sigma	Egyptian_Date	TSun	TMoon	q	code	NEW_MOON_DATE	TIME"/>
180 <has_text text="3h 5m 1s"/>
181 </assert_contents>
182 </output>
183 </test>
184 <!-- Strict mode passes when every rejection reason is tolerated. -->
185 <test expect_num_outputs="3">
186 <param name="input" value="fixedwidth_messy.txt"/>
187 <conditional name="strictness">
188 <param name="mode" value="strict"/>
189 <param name="allow_rejects" value="length,rule-line"/>
190 </conditional>
191 <output name="rejected">
192 <assert_contents>
193 <has_n_lines n="10"/>
194 </assert_contents>
195 </output>
196 </test>
197 <!-- Strict mode fails the job with exit code 4 when lines are rejected. -->
198 <test expect_exit_code="4" expect_failure="true">
199 <param name="input" value="fixedwidth_messy.txt"/>
200 <conditional name="strictness">
201 <param name="mode" value="strict"/>
202 </conditional>
203 <assert_stderr>
204 <has_text text="--strict"/>
205 </assert_stderr>
206 </test>
207 <!-- Padding short lines without a header line is a parameter error (exit code 2). -->
208 <test expect_exit_code="2" expect_failure="true">
209 <param name="input" value="fixedwidth_input.txt"/>
210 <section name="reading">
211 <param name="short_lines" value="pad"/>
212 </section>
213 <assert_stderr>
214 <has_text text="--short-lines pad needs --header-line"/>
215 </assert_stderr>
216 </test>
217 </tests>
218 <help format="markdown"><![CDATA[
219**What it does**
220
221Converts a fixed-width text file into a tabular dataset. Legacy programs
222often write their results as aligned columns. Tools that split on a delimiter
223cannot read such files: a cell that contains spaces breaks into several cells,
224and a blank cell shifts every later field to the left.
225
226The tool finds the column boundaries itself. A boundary is a run of character
227positions that are blank in every record. Usually, no settings are necessary.
228
229-----
230
231**Outputs**
232
233Every successful run gives three datasets:
234
235- **Table**: one row per record, cells separated by tabs. It has no header row
236 unless you select one.
237- **Report**: what the tool detected, and how to correct it.
238- **Rejected lines**: every input line that the tool did not convert, as three
239 columns: line number, reason and the line itself. It usually holds only the
240 title and heading lines.
241
242The table rows plus the rejected lines always equal the input lines, so no
243line disappears without a record.
244
245-----
246
247**Check and correct the result**
248
249Read the report in this order:
250
2511. **Reconciliation** must end in `[OK]`.
2522. **Input** shows the record width and the line used as the header.
2533. **Columns** lists each column with its range, name and sample values.
2544. **Warnings** names what to change.
255
256To correct a boundary or a name, find the section "Reproducing or correcting
257this geometry" in the report. Copy the text between the quotes, for example
258`1-12:DATE,18-27:BEST_TIME`. Select "Give the column ranges", paste the text,
259change only the wrong range or name, and run the tool again.
260
261-----
262
263**Upload hazard**
264
265Do not upload the file with "convert spaces to tabs". That option destroys
266the column positions before the tool sees the data. A tab inside a record
267makes the job fail.
268
269-----
270
271**Exit codes**
272
273A failed job shows its reason in the error message:
274
275- 2: the parameters are not valid.
276- 3: the input cannot be used. It cannot be decoded, has a tab inside a
277 record, is empty, or has no records.
278- 4: strict mode is on, and lines were rejected.
279
280More information: [Running fixed2tab on a file, step by step](https://github.com/dasch-swiss/rdu-fixed2tab/blob/main/docs/running-fixed2tab.md)
281 ]]></help>
282 <expand macro="citations"/>
283</tool>