upstream-extended/community/tools/text_processing/text_processing/easyjoin
1#!/usr/bin/env perl
2## EASY Join -
3## Join with automatic pre-sorting of both files
4## Copyright (C) 2010 A. Gordon (gordon@cshl.edu)
5## license: AGPLv3+
6use strict;
7use warnings;
8use Data::Dumper;
9use Getopt::Long qw(:config bundling no_ignore_case_always);
10use File::Temp qw/tempfile/;
11use POSIX qw(locale_h);
12
13sub show_help();
14sub show_version();
15sub show_examples();
16sub parse_commandline_options();
17sub sort_file($$$);
18sub join_files($$);
19sub cleanup_files(@);
20
21
22my $PROGRAM="easyjoin";
23my $VERSION="0.6.1";
24
25my $debug=undef;
26my $HEADER=undef;
27my $IGNORE_CASE=undef;
28my $FIELD_SEP=undef;
29my $FILE1_KEY_COLUMN=1;
30my $FILE2_KEY_COLUMN=1;
31my @OUTPUT_SPECIFIERS=();
32my $OUTPUT_FORMAT=undef;
33my $EMPTY_FILLER=undef;
34my $SORT_BUFFER_SIZE=undef;
35my $SORT_TEMP_DIR=undef;
36my $input_filename1;
37my $input_filename2;
38
39##
40## Program Start
41##
42$ENV{'LANG'}="C";## "C" locale is critical for sorting and joining correctly
43parse_commandline_options();
44my (undef, $tmp_filename1) = tempfile(OPEN=>0);
45my (undef, $tmp_filename2) = tempfile(OPEN=>0);
46sort_file($input_filename1, $tmp_filename1, $FILE1_KEY_COLUMN);
47sort_file($input_filename2, $tmp_filename2, $FILE2_KEY_COLUMN);
48my $join_exit_code = join_files($tmp_filename1, $tmp_filename2);
49cleanup_files($tmp_filename1, $tmp_filename2);
50exit($join_exit_code);
51
52##
53## Program end
54##
55
56
57sub show_help()
58{
59print<<EOF;
60${PROGRAM}: Wrapper for GNU join+sort, automaticalyl sorts files before joining them.
61
62Usage: $PROGRAM [OPTIONS] [JOIN-OPTIONS] [SORT-OPTIONS] FILE1 FILE2
63
64OPTIONS: Options specific to this program:
65
66 --header = Both input files have a header line as the first line.
67 The header line will be joined properly, without being sorted.
68
69 --version = Print ${PROGRAM}'s version.
70
71 --debug = Print debug messages (relating to ${PROGRAM}'s operation).
72
73 --help = Show this help screen.
74
75 --example = Show usage examples.
76
77 --all = Short-cut for:
78 -a 1 -a 2 -o auto -e . -t <TAB>
79 This will show all values (paired and unpared) from both files,
80 Automatically formatting the columns, and using TAB as field separator.
81 You can override the empty filler (-e X) on the command line.
82
83 --allh = Short-cut for:
84 -a 1 -a 2 -o auto -e . -t <TAB> --header
85 Same as above, but will also respect the header line from both input files.
86
87JOIN-OPTIONS:
88 All of GNU join options are supported.
89 Run:
90 join --help
91 To see all possible joining options.
92
93SORT-OPTIONS:
94 The following options are supported for the intermediate sorting step:
95
96 -S SIZE
97 --buffer-size SIZE = GNU sort's --buffer-size option.
98
99 -T DIR
100 --temporary-directory DIR = GNU sort's --temporary-directory option.
101
102 Run:
103 sort --help
104 To learn about these options. They might improve sorting performances for big files.
105
106FILE1 FILE2:
107 The two input files to be sorted, joined.
108 Unlike GNU join, joining STDIN is not supported. Both files must be real files.
109
110
111NOTE About "--header" and "--auto-format":
112 The "--header" feature requires GNU coreutils version 8.6 or later.
113 The "-o auto" feature requires GNU coreutils version 8.10 or later.
114
115EOF
116 exit(0);
117}
118
119sub show_version()
120{
121print<<EOF;
122$PROGRAM $VERSION
123Copyright (C) 2010 A. Gordon (gordon\@cshl.edu)
124License AGPLv3+: Affero GPL version 3 or later (http://www.gnu.org/licenses/agpl.html)
125
126To see the GNU's join version, run:
127 join --version
128EOF
129 exit(0);
130}
131
132sub show_examples()
133{
134print<<EOF;
135Example of joining two unsorted files (each file having a header line):
136
137\$ cat input1.txt
138Fruit Color
139Apple red
140Banana yellow
141Orange orange
142Melon green
143
144\$ cat input2.txt
145Fruit Price
146Orange 7
147Avocado 8
148Apple 4
149Banana 3
150
151\$ easyjoin -j 1 -a 1 -a 2 --header -e . -o auto input1.txt input2.txt
152Fruit Color Price
153Apple red 4
154Avocado . 8
155Banana yellow 3
156Melon green .
157Orange orange 7
158
159## A short-cut for all the options above:
160\$ easyjoin --allh input1.txt input2.txt
161Fruit Color Price
162Apple red 4
163Avocado . 8
164Banana yellow 3
165Melon green .
166Orange orange 7
167
168EOF
169 exit(0);
170}
171
172sub parse_commandline_options()
173{
174 ##
175 ## Parse command line
176 ##
177 my $rc = GetOptions(
178 "a=i" => sub { push @OUTPUT_SPECIFIERS, '-a', $_[1] },
179 "e=s" => \$EMPTY_FILLER,
180 "ignore-case|i" => \$IGNORE_CASE,
181 "j=i" => sub { $FILE1_KEY_COLUMN = $_[1] ; $FILE2_KEY_COLUMN = $_[1] ; },
182 "o=s" => \$OUTPUT_FORMAT,
183 "t=s" => \$FIELD_SEP,
184 "v=i" => sub { push @OUTPUT_SPECIFIERS, '-v', $_[1] },
185 "1=i" => \$FILE1_KEY_COLUMN,
186 "2=i" => \$FILE2_KEY_COLUMN,
187 "debug" => \$debug,
188 "header" => \$HEADER,
189 "help" => \&show_help,
190 "version" => \&show_version,
191 "examples" => \&show_examples,
192 "buffer-size|S=s" => \$SORT_BUFFER_SIZE,
193 "temporary-directory|T=s" => \$SORT_TEMP_DIR,
194 "all" => sub {
195 push @OUTPUT_SPECIFIERS, "-a", 1, "-a", 2;
196 $FIELD_SEP = "\t";
197 $OUTPUT_FORMAT = "auto";
198 $EMPTY_FILLER = "." unless defined $EMPTY_FILLER;
199 },
200 "allh" => sub {
201 push @OUTPUT_SPECIFIERS, "-a", 1, "-a", 2;
202 $FIELD_SEP = "\t";
203 $OUTPUT_FORMAT = "auto";
204 $HEADER=1;
205 $EMPTY_FILLER = "." unless defined $EMPTY_FILLER;
206 },
207 );
208 die "$PROGRAM: invalid command-line arguments.\n" unless $rc;
209
210 ## We need two file names to join
211 my @INPUT_FILES = @ARGV;
212 die "$PROGRAM: missing operand: two file names to join\n" if (scalar(@INPUT_FILES)<2);
213 die "$PROGRAM: error: too many files specified (can only join two files)\n" if (scalar(@INPUT_FILES)>2);
214 die "$PROGRAM: error: input file can't be STDIN, please use a real file name.\n" if $INPUT_FILES[0] eq "-" || $INPUT_FILES[1] eq "-";
215 die "$PROGRAM: error: input file 1 '" . $INPUT_FILES[0] . "' not found!" unless -e $INPUT_FILES[0];
216 die "$PROGRAM: error: input file 2 '" . $INPUT_FILES[1] . "' not found!" unless -e $INPUT_FILES[1];
217
218 $input_filename1 = $INPUT_FILES[0];
219 $input_filename2 = $INPUT_FILES[1];
220}
221
222sub sort_file($$$)
223{
224 my ($input_filename, $output_filename, $key_column) = @_;
225
226 my @SORT_COMMAND;
227 push @SORT_COMMAND, $HEADER ? "./sort-header" : "sort" ;
228 push @SORT_COMMAND, "-f" if $IGNORE_CASE;
229 push @SORT_COMMAND, "-k${key_column},${key_column}" ;
230 push @SORT_COMMAND, "--buffer-size", $SORT_BUFFER_SIZE if $SORT_BUFFER_SIZE;
231 push @SORT_COMMAND, "--temporary-directory", $SORT_TEMP_DIR if $SORT_TEMP_DIR;
232 push @SORT_COMMAND, "--output", $output_filename;
233 push @SORT_COMMAND, "--debugheader" if $debug && $HEADER;
234 push @SORT_COMMAND, "-t", $FIELD_SEP if $FIELD_SEP;
235 push @SORT_COMMAND, $input_filename;
236
237 if ($debug) {
238 warn "$PROGRAM: Running sort on '$input_filename' => '$output_filename'\n";
239 warn "$PROGRAM: Sort command line:\n";
240 print STDERR Dumper(\@SORT_COMMAND), "\n";
241 }
242
243 my $sort_exit_code=1;
244 system(@SORT_COMMAND);
245 if ($? == -1) {
246 die "$PROGRAM: Error: failed to execute 'sort': $!\n";
247 }
248 elsif ($? & 127) {
249 my $signal = ($? & 127);
250 kill 2, $$ if $signal == 2; ##if sort was interrupted (CTRL-C) - just pass it on and commit suicide
251 die "$PROGRAM: Error: 'sort' child-process died with signal $signal\n";
252 }
253 else {
254 $sort_exit_code = ($? >> 8);
255 }
256 die "$PROGRAM: Error: 'sort' process failed, exit code $sort_exit_code\n" if $sort_exit_code!=0;
257}
258
259sub join_files($$)
260{
261 my ($file1, $file2) = @_;
262
263 my @join_command = qw/join/;
264 push @join_command, "--header" if $HEADER;
265 push @join_command, "--ignore-case" if $IGNORE_CASE;
266 push @join_command, "-t", $FIELD_SEP if $FIELD_SEP;
267 push @join_command, "-1", $FILE1_KEY_COLUMN if $FILE1_KEY_COLUMN;
268 push @join_command, "-2", $FILE2_KEY_COLUMN if $FILE2_KEY_COLUMN;
269 push @join_command, "-e", $EMPTY_FILLER if defined $EMPTY_FILLER;
270 push @join_command, "-o", $OUTPUT_FORMAT if $OUTPUT_FORMAT;
271 push @join_command, @OUTPUT_SPECIFIERS;
272 push @join_command, $file1, $file2;
273
274 if ($debug) {
275 warn "$PROGRAM: Running join on '$file1' and '$file2'\n";
276 warn "$PROGRAM: join command line:\n";
277 print STDERR Dumper(\@join_command), "\n";
278 }
279
280 my $join_exit_code=1;
281 system(@join_command);
282 if ($? == -1) {
283 die "$PROGRAM: Error: failed to execute 'join': $!\n";
284 }
285 elsif ($? & 127) {
286 my $signal = ($? & 127);
287 kill 2, $$ if $signal == 2; ##if join was interrupted (CTRL-C) - just pass it on and commit suicide
288 die "$PROGRAM: Error: 'join' child-process died with signal $signal\n";
289 }
290 else {
291 $join_exit_code = ($? >> 8);
292 }
293 return $join_exit_code;
294}
295
296sub cleanup_files(@)
297{
298 my (@files) = @_;
299
300 foreach my $file (@files) {
301 if ($debug) {
302 warn "$PROGRAM: debug mode, not deleting temporary file '$file'\n";
303 } else {
304 my $count = unlink $file;
305 warn "$PROGRAM: Error: failed to delete temporary file '$file': $!\n" if ($count != 1);
306 }
307 }
308}