upstream/tools/text_processing/text_processing/multijoin
1#!/usr/bin/env perl
2use strict;
3use warnings;
4use Getopt::Long qw(:config no_ignore_case);
5use Data::Dumper;
6use Carp;
7use File::Basename;
8
9my $version = "0.1.1";
10my $field_sep = "\t";
11my $key_column;
12my @values_columns;
13my $max_value_column;
14my @input_files;
15my $input_headers ;
16my $output_headers;
17my $filler = "0";
18my $filler_string ;
19my $ignore_duplicates;
20my $debug = 0 ;
21my %input_headers;
22my $have_file_labels;
23my %file_labels;
24
25sub parse_command_line_parameters();
26sub show_help();
27sub read_input_file($);
28sub print_combined_data();
29sub sanitize_filename($);
30sub print_output_header();
31sub show_examples();
32
33##
34## Program Start
35##
36
37parse_command_line_parameters();
38
39my %data;
40foreach my $file (@input_files) {
41 read_input_file($file);
42}
43#print STDERR Dumper(\%input_headers),"\n";
44#print STDERR Dumper(\%data) if $debug;
45print_output_header() if $output_headers;
46print_combined_data();
47
48
49##
50## Program End
51##
52sub print_output_header()
53{
54 my @output = ("key");
55 foreach my $file ( @input_files ) {
56 foreach my $column ( @values_columns ) {
57 my $column_name = ( exists $input_headers{$file}->{$column} ) ?
58 $input_headers{$file}->{$column} :
59 "V$column" ;
60
61 push @output, $file_labels{$file} . "_" . $column_name;
62 }
63 }
64 print join($field_sep,@output),"\n"
65 or die "Output error: can't write output line: $!\n";
66}
67
68sub print_combined_data()
69{
70 my @keys = sort keys %data ;
71
72 foreach my $key ( @keys ) {
73 my @outputs;
74
75 foreach my $file (@input_files) {
76 push @outputs,
77 (exists $data{$key}->{$file}) ? $data{$key}->{$file} : $filler_string;
78 }
79
80 print join($field_sep,$key,@outputs),"\n"
81 or die "Output error: can't write output line: $!\n";
82 }
83}
84
85sub sanitize_filename($)
86{
87 my ($filename) = shift or croak "missing file name";
88 my $file_ID = basename($filename);
89 $file_ID =~ s/\.\w+$//; # remove extension
90 $file_ID =~ s/^[^\w\.\-]+//;
91 $file_ID =~ s/[^\w\.\-]+$//;
92 $file_ID =~ s/[^\w\.\-]+/_/g; # sanitize bad characters
93 return $file_ID;
94}
95
96sub read_input_file($)
97{
98 my ($filename) = shift or croak "Missing input file name";
99
100 my @value_indexes = map { $_-1 } @values_columns; #zero-based indexes for value columns
101
102 open FILE, "<", $filename
103 or die "Error: can't open file '$filename': $!\n";
104
105 ## Read file's header
106 if ($input_headers) {
107 my $line = <FILE>;
108 chomp $line;
109 my @fields = split $field_sep, $line;
110
111 my $num_input_fields = scalar(@fields);
112 die "Input error: file '$filename' line $. doesn't have enough columns (value column = $max_value_column, line has only $num_input_fields columns)\n" if $num_input_fields < $max_value_column ;
113
114 foreach my $col (@values_columns) {
115 $input_headers{$filename}->{$col} = $fields[$col-1] ;
116 }
117 }
118
119
120 ## Read file's data
121 while ( my $line = <FILE> ) {
122 chomp $line;
123 my @fields = split $field_sep, $line;
124
125 my $num_input_fields = scalar(@fields);
126 die "Input error: file '$filename' line $. doesn't have enough columns (key column = $key_column, line has only $num_input_fields columns)\n" if $num_input_fields < $key_column ;
127 die "Input error: file '$filename' line $. doesn't have enough columns (value column = $max_value_column, line has only $num_input_fields columns)\n" if $num_input_fields < $max_value_column ;
128
129
130 my $key = $fields[$key_column-1];
131 my $value = join($field_sep, @fields[@value_indexes]);
132
133 die "Input error: file '$filename' line $. have duplicated key '$key'.\n"
134 if (exists $data{$key}->{$filename} && !$ignore_duplicates) ;
135 $data{$key}->{$filename} = $value;
136 }
137 close FILE
138 or die "Error: can't write and close file '$filename': $!\n";
139}
140
141sub parse_command_line_parameters()
142{
143 my $values_columns_string;
144
145 my $rc = GetOptions("help" => \&show_help,
146 "key|k=i" => \$key_column,
147 "values|v=s" => \$values_columns_string,
148 "t=s" => \$field_sep,
149 "in-header" => \$input_headers,
150 "out-header|h" => \$output_headers,
151 "H" => sub { $input_headers = 1 ; $output_headers = 1 ; },
152 "ignore-dups" => \$ignore_duplicates,
153 "filler|f=s" => \$filler,
154 "examples" => \&show_examples,
155 "labels" => \$have_file_labels,
156 );
157 die "Error: inalid command-line parameters.\n" unless $rc;
158
159 die "Error: missing key column. use --key N. see --help for more details.\n" unless defined $key_column;
160 die "Error: Invalid key column ($key_column). Must be bigger than zero. see --help for more details.\n" if $key_column <= 0 ;
161
162 die "Error: missing values column. use --values V1,V2,Vn. See --help for more details.\n" unless defined $values_columns_string;
163 @values_columns = split(/\s*,\s*/, $values_columns_string);
164
165 die "Error: missing values column. use --values N,N,N. see --help for more details.\n" unless scalar(@values_columns)>0;
166 foreach my $v (@values_columns) {
167 die "Error: invalid value column ($v), please use only numbers>=1. see --help for more details.\n"
168 unless $v =~ /^\d+$/ && $v>=1;
169
170 $max_value_column = $v unless defined $max_value_column && $max_value_column>$v;
171 }
172
173 $filler_string = join($field_sep, map { $filler } @values_columns);
174
175
176 if ($have_file_labels) {
177 ## have file labels - each pair of parameters is a file/label pair.
178 die "Error: missing input files and labels\n" if scalar(@ARGV)==0;
179 die "Error: when using --labels, a pair of file names + labels is required (got odd number of argiments)\n" unless scalar(@ARGV)%2==0;
180
181 while (@ARGV) {
182 my $filename = shift @ARGV;
183 my $label = shift @ARGV;
184 $label =~ s/^[^\.\w\-]+//;
185 $label =~ s/[^\.\w\-]+$//g;
186 $label =~ s/[^\.\w\-]+/_/g;
187
188 my $file_ID = sanitize_filename($filename);
189 $file_labels{$filename} = $label;
190 push @input_files, $filename;
191 }
192 } else {
193 ## no file labels - the rest of the arguments are just file names;
194 @input_files = @ARGV;
195 die "Error: missing input files\n" if scalar(@input_files)==0;
196 die "Error: need more than one input file to join.\n" if scalar(@input_files)==1;
197
198 foreach my $file (@input_files) {
199 my $file_ID = sanitize_filename($file);
200 $file_labels{$file} = $file_ID;
201 }
202 }
203
204}
205
206sub show_help()
207{
208 print<<EOF;
209Multi-File join, version $version
210Copyright (C) 2012 - A. Gordon (gordon at cshl dot edu)
211License AGPLv3+: Affero GPL version 3 or later (http://www.gnu.org/licenses/agpl.html)
212
213Usage:
214 multijoin [OPTIONS] -k N -v V1,V2,Vn,.. FILE1 FILE2 ... FILEn
215
216Options:
217
218 --help This helpful help screen.
219
220 -k N
221 --key N Use column N as key column.
222
223 -v V1,V2,Vn
224 --values V1,V2,Vn
225 Use columns V1,V2,Vn as value columns - those will be joined
226 According to the Key column.
227 Multiple columns can be specified.
228
229 -t SEP Use SEP as field separator character (default: tab).
230
231 -h
232 --out-header Add a header line to the output file.
233
234 --in-header The input files have a header line.
235 The first line will not be joined.
236 if '--out-header' is also used, the output column headers will
237 be constructed based on the input header column names.
238
239 -H
240 --headers Same as '--in-header --out-header' combined.
241
242 --ignore-dups Ignore duplicated keys (within a file).
243 By default, duplicated keys cause an error.
244
245 -f X
246 --filler X Fill missing values with X.
247 (Default: '$filler').
248
249 --labels When printing output headers with '-h', instead of using the file name,
250 use specific labels.
251 Each file name must be followed by a name.
252
253 example (without labels):
254 \$ multijoin -h -k 1 -v 2 A.TXT B.TXT C.TXT
255
256 example (with labels):
257 \$ multijoin -h --labels -k 1 -v 2 A.TXT Sample1 B.TXT SampleB C.TXT SampleC
258
259 --examples Show detailed examples.
260
261EOF
262 exit(0);
263}
264
265sub show_examples()
266{
267 print<<EOF;
268
269To join three files, based on the 4th column, and keeping the 7th,8th,9th columns:
270
271\$ head *.txt
272==> AAA.txt <==
273chr4 888449 890171 FBtr0308778 0 + 266 1527 1722
274chr4 972167 979017 FBtr0310651 0 - 3944 6428 6850
275chr4 972186 979017 FBtr0089229 0 - 3944 6428 6831
276chr4 972186 979017 FBtr0089231 0 - 3944 6428 6831
277chr4 972186 979017 FBtr0089233 0 - 3944 6428 6831
278chr4 995793 996435 FBtr0111046 0 + 7 166 642
279chr4 995793 997931 FBtr0111044 0 + 28 683 2138
280chr4 995793 997931 FBtr0111045 0 + 28 683 2138
281chr4 1034029 1047719 FBtr0089223 0 - 5293 13394 13690
282
283==> BBB.txt <==
284chr4 90286 134453 FBtr0309803 0 + 657 29084 44167
285chr4 251355 266499 FBtr0089116 0 + 56 1296 15144
286chr4 252050 266506 FBtr0308086 0 + 56 1296 14456
287chr4 252050 266506 FBtr0308087 0 + 56 1296 14456
288chr4 252053 266528 FBtr0300796 0 + 56 1296 14475
289chr4 252053 266528 FBtr0300800 0 + 56 1296 14475
290chr4 252055 266528 FBtr0300798 0 + 56 1296 14473
291chr4 252055 266528 FBtr0300799 0 + 56 1296 14473
292chr4 252541 266528 FBtr0300797 0 + 56 1296 13987
293
294==> CCC.txt <==
295chr4 972167 979017 FBtr0310651 0 - 9927 6738 6850
296chr4 972186 979017 FBtr0089229 0 - 9927 6738 6831
297chr4 972186 979017 FBtr0089231 0 - 9927 6738 6831
298chr4 972186 979017 FBtr0089233 0 - 9927 6738 6831
299chr4 995793 996435 FBtr0111046 0 + 5 304 642
300chr4 995793 997931 FBtr0111044 0 + 17 714 2138
301chr4 995793 997931 FBtr0111045 0 + 17 714 2138
302chr4 1034029 1047719 FBtr0089223 0 - 17646 13536 13690
303
304\$ multijoin -h --key 4 --values 7,8,9 *.txt | head -n 10
305key AAA__V7 AAA__V8 AAA__V9 BBB__V7 BBB__V8 BBB__V9 CCC__V7 CCC__V8 CCC__V9
306FBtr0089116 0 0 0 56 1296 15144 0 0 0
307FBtr0089223 5293 13394 13690 0 0 0 17646 13536 13690
308FBtr0089229 3944 6428 6831 0 0 0 9927 6738 6831
309FBtr0089231 3944 6428 6831 0 0 0 9927 6738 6831
310FBtr0089233 3944 6428 6831 0 0 0 9927 6738 6831
311FBtr0111044 28 683 2138 0 0 0 17 714 2138
312FBtr0111045 28 683 2138 0 0 0 17 714 2138
313FBtr0111046 7 166 642 0 0 0 5 304 642
314FBtr0300796 0 0 0 56 1296 14475 0 0 0
315
316
317
318EOF
319 exit(0);
320}