upstream-dependencies/bcftools/vcfquery.c
1/* vcfquery.c -- Extracts fields from VCF/BCF file.
2
3 Copyright (C) 2013-2023 Genome Research Ltd.
4
5 Author: Petr Danecek <pd3@sanger.ac.uk>
6
7Permission is hereby granted, free of charge, to any person obtaining a copy
8of this software and associated documentation files (the "Software"), to deal
9in the Software without restriction, including without limitation the rights
10to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11copies of the Software, and to permit persons to whom the Software is
12furnished to do so, subject to the following conditions:
13
14The above copyright notice and this permission notice shall be included in
15all copies or substantial portions of the Software.
16
17THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23THE SOFTWARE. */
24
25#include <stdio.h>
26#include <unistd.h>
27#include <getopt.h>
28#include <ctype.h>
29#include <string.h>
30#include <errno.h>
31#include <sys/stat.h>
32#include <sys/types.h>
33#include <htslib/vcf.h>
34#include <htslib/synced_bcf_reader.h>
35#include <htslib/khash_str2int.h>
36#include <htslib/vcfutils.h>
37#include "bcftools.h"
38#include "filter.h"
39#include "convert.h"
40#include "smpl_ilist.h"
41
42
43// Logic of the filters: include or exclude sites which match the filters?
44#define FLT_INCLUDE 1
45#define FLT_EXCLUDE 2
46
47typedef struct
48{
49 filter_t *filter;
50 char *filter_str;
51 int filter_logic; // include or exclude sites which match the filters? One of FLT_INCLUDE/FLT_EXCLUDE
52 uint8_t *smpl_pass;
53 convert_t *convert;
54 bcf_srs_t *files;
55 bcf_hdr_t *header;
56 int sample_is_file;
57 char **argv, *format_str, *sample_list, *targets_list, *regions_list, *vcf_list, *fn_out;
58 char *print_filtered;
59 int argc, list_columns, print_header, allow_undef_tags, force_samples, force_newline;
60 FILE *out;
61}
62args_t;
63
64static void destroy_list(char **list, int n)
65{
66 int i;
67 for (i=0; i<n; i++)
68 free(list[i]);
69 free(list);
70}
71
72static void init_data(args_t *args)
73{
74 args->header = args->files->readers[0].header;
75
76 int i, nsamples = 0, *samples = NULL;
77 if ( args->sample_list && strcmp("-",args->sample_list) )
78 {
79 for (i=0; i<args->files->nreaders; i++)
80 {
81 // This tells htslib to subset samples directly when reading. Also the header is modified to
82 // include only the requested samples
83 int ret = bcf_hdr_set_samples(args->files->readers[i].header,args->sample_list,args->sample_is_file);
84 if ( ret<0 ) error("Error parsing the sample list\n");
85 else if ( ret>0 && !args->force_samples )
86 error("Error: sample #%d not found in the header, user --force-samples to proceed anyway\n", ret);
87 }
88
89 int flags = SMPL_REORDER;
90 smpl_ilist_t *ilist = smpl_ilist_init(args->files->readers[0].header, args->sample_list, args->sample_is_file, flags);
91 nsamples = ilist->n;
92 samples = (int*) malloc(sizeof(int)*nsamples);
93 for (i=0; i<ilist->n; i++)
94 samples[i] = ilist->idx[i];
95 smpl_ilist_destroy(ilist);
96 }
97 args->convert = convert_init(args->header, samples, nsamples, args->format_str);
98 if ( args->force_newline ) convert_set_option(args->convert, force_newline, 1);
99 convert_set_option(args->convert, subset_samples, &args->smpl_pass);
100 if ( args->allow_undef_tags ) convert_set_option(args->convert, allow_undef_tags, 1);
101 if ( args->print_header>1 ) convert_set_option(args->convert, no_hdr_indices, 1);
102 free(samples);
103
104 int max_unpack = convert_max_unpack(args->convert);
105 if ( args->filter_str )
106 {
107 args->filter = filter_init(args->header, args->filter_str);
108 max_unpack |= filter_max_unpack(args->filter);
109 }
110 args->files->max_unpack = max_unpack;
111 if ( !args->filter || args->print_filtered || !(filter_max_unpack(args->filter) & BCF_UN_FMT) )
112 convert_set_option(args->convert, header_samples, 1);
113 if ( args->print_filtered ) convert_set_option(args->convert, print_filtered, args->print_filtered);
114}
115
116static void destroy_data(args_t *args)
117{
118 convert_destroy(args->convert);
119 if ( args->filter )
120 filter_destroy(args->filter);
121}
122
123static void query_vcf(args_t *args)
124{
125 kstring_t str = {0,0,0};
126
127 if ( args->print_header )
128 {
129 convert_header(args->convert,&str);
130 if ( fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
131 }
132
133 int i,max_convert_unpack = convert_max_unpack(args->convert);
134 int max_filter_unpack = args->filter ? filter_max_unpack(args->filter) : 0;
135 while ( bcf_sr_next_line(args->files) )
136 {
137 if ( !bcf_sr_has_line(args->files,0) ) continue;
138 bcf1_t *line = args->files->readers[0].buffer[0];
139 bcf_unpack(line, args->files->max_unpack);
140
141 if ( args->filter )
142 {
143 int pass = filter_test(args->filter, line, (const uint8_t**) &args->smpl_pass);
144 if ( args->filter_logic & FLT_EXCLUDE )
145 {
146 // This code addresses this problem:
147 // -i can include a site but exclude a sample
148 // -e exclude a site but include a sample
149
150 if ( pass )
151 {
152 if ( !args->smpl_pass ) continue;
153 if ( !(max_convert_unpack & BCF_UN_FMT) && !(max_filter_unpack & BCF_UN_FMT) ) continue;
154
155 pass = 0;
156 for (i=0; i<line->n_sample; i++)
157 {
158 if ( args->smpl_pass[i] ) args->smpl_pass[i] = 0;
159 else { args->smpl_pass[i] = 1; pass = 1; }
160 }
161 if ( !pass ) continue;
162 }
163 else if ( args->smpl_pass )
164 for (i=0; i<line->n_sample; i++) args->smpl_pass[i] = 1;
165 }
166 else if ( !pass ) continue;
167 }
168
169 str.l = 0;
170 convert_line(args->convert, line, &str);
171 if ( str.l && fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
172 }
173 if ( str.m ) free(str.s);
174}
175
176static void list_columns(args_t *args)
177{
178 int negate = 0;
179 int i;
180 bcf_sr_t *reader = &args->files->readers[0];
181 void *has_sample = NULL;
182 if ( args->sample_list )
183 {
184 if ( args->sample_list[0]=='^' ) negate = 1;
185 has_sample = khash_str2int_init();
186 int i, nsmpl;
187 char **smpl = hts_readlist(negate ? args->sample_list+1 : args->sample_list, args->sample_is_file, &nsmpl);
188 if ( !smpl ) error("Error: failed to read %s\n", negate ? args->sample_list+1 : args->sample_list);
189 for (i=0; i<nsmpl; i++)
190 {
191 if ( bcf_hdr_id2int(reader->header,BCF_DT_SAMPLE,smpl[i])<0 && !args->force_samples )
192 error("Error: sample #%d not found in the header, user --force-samples to proceed anyway\n", i+1);
193 khash_str2int_inc(has_sample, smpl[i]);
194 }
195 free(smpl);
196 }
197
198 for (i=0; i<bcf_hdr_nsamples(reader->header); i++)
199 {
200 int skip = 0;
201 if ( negate )
202 {
203 if ( khash_str2int_has_key(has_sample, reader->header->samples[i]) ) skip = 1;
204 }
205 else if ( has_sample && !khash_str2int_has_key(has_sample, reader->header->samples[i]) ) skip = 1;
206 if ( skip ) continue;
207 printf("%s\n", reader->header->samples[i]);
208 }
209
210 if ( has_sample )
211 khash_str2int_destroy_free(has_sample);
212}
213
214static char **copy_header(bcf_hdr_t *hdr, char **src, int nsrc)
215{
216 char **dst = (char**) malloc(sizeof(char*)*nsrc);
217 int i;
218 for (i=0; i<nsrc; i++) dst[i] = strdup(src[i]);
219 return dst;
220}
221static int compare_header(bcf_hdr_t *hdr, char **a, int na, char **b, int nb)
222{
223 if ( na!=nb ) return na-nb;
224 int i;
225 for (i=0; i<na; i++)
226 if ( strcmp(a[i],b[i]) ) return 1;
227 return 0;
228}
229
230
231static void usage(void)
232{
233 fprintf(stderr, "\n");
234 fprintf(stderr, "About: Extracts fields from VCF/BCF file and prints them in user-defined format\n");
235 fprintf(stderr, "Usage: bcftools query [options] <A.vcf.gz> [<B.vcf.gz> [...]]\n");
236 fprintf(stderr, "\n");
237 fprintf(stderr, "Options:\n");
238 fprintf(stderr, " -e, --exclude EXPR Exclude sites for which the expression is true (see man page for details)\n");
239 fprintf(stderr, " --force-samples Only warn about unknown subset samples\n");
240 fprintf(stderr, " -F, --print-filtered STR Output STR for samples failing the -i/-e filtering expression\n");
241 fprintf(stderr, " -f, --format STRING See man page for details\n");
242 fprintf(stderr, " -H, --print-header Print header, -HH to omit column indices\n");
243 fprintf(stderr, " -i, --include EXPR Select sites for which the expression is true (see man page for details)\n");
244 fprintf(stderr, " -l, --list-samples Print the list of samples and exit\n");
245 fprintf(stderr, " -N, --disable-automatic-newline Disable automatic addition of newline character when not present\n");
246 fprintf(stderr, " -o, --output FILE Output file name [stdout]\n");
247 fprintf(stderr, " -r, --regions REGION Restrict to comma-separated list of regions\n");
248 fprintf(stderr, " -R, --regions-file FILE Restrict to regions listed in a file\n");
249 fprintf(stderr, " --regions-overlap 0|1|2 Include if POS in the region (0), record overlaps (1), variant overlaps (2) [1]\n");
250 fprintf(stderr, " -s, --samples LIST List of samples to include\n");
251 fprintf(stderr, " -S, --samples-file FILE File of samples to include\n");
252 fprintf(stderr, " -t, --targets REGION Similar to -r but streams rather than index-jumps\n");
253 fprintf(stderr, " -T, --targets-file FILE Similar to -R but streams rather than index-jumps\n");
254 fprintf(stderr, " --targets-overlap 0|1|2 Include if POS in the region (0), record overlaps (1), variant overlaps (2) [0]\n");
255 fprintf(stderr, " -u, --allow-undef-tags Print \".\" for undefined tags\n");
256 fprintf(stderr, " -v, --vcf-list FILE Process multiple VCFs listed in the file\n");
257 fprintf(stderr, "\n");
258 fprintf(stderr, "Examples:\n");
259 fprintf(stderr, "\tbcftools query -f '%%CHROM\\t%%POS\\t%%REF\\t%%ALT[\\t%%SAMPLE=%%GT]\\n' file.vcf.gz\n");
260 fprintf(stderr, "\t# For more examples see http://samtools.github.io/bcftools/bcftools.html#query\n");
261 fprintf(stderr, "\n");
262 exit(1);
263}
264
265int main_vcfquery(int argc, char *argv[])
266{
267 int c, collapse = 0;
268 args_t *args = (args_t*) calloc(1,sizeof(args_t));
269 args->argc = argc; args->argv = argv;
270 args->force_newline = 1;
271 int regions_is_file = 0, targets_is_file = 0;
272 int regions_overlap = 1;
273 int targets_overlap = 0;
274
275 static struct option loptions[] =
276 {
277 {"help",0,0,'h'},
278 {"list-samples",0,0,'l'},
279 {"disable-automatic-newline",required_argument,NULL,'N'},
280 {"include",1,0,'i'},
281 {"exclude",1,0,'e'},
282 {"print-filtered",1,0,'F'},
283 {"format",1,0,'f'},
284 {"force-samples",0,0,3},
285 {"output-file",1,0,'o'},
286 {"output",1,0,'o'},
287 {"regions",1,0,'r'},
288 {"regions-file",1,0,'R'},
289 {"regions-overlap",required_argument,NULL,1},
290 {"targets",1,0,'t'},
291 {"targets-file",1,0,'T'},
292 {"targets-overlap",required_argument,NULL,2},
293 {"annots",1,0,'a'},
294 {"samples",1,0,'s'},
295 {"samples-file",1,0,'S'},
296 {"print-header",0,0,'H'},
297 {"collapse",1,0,'c'},
298 {"vcf-list",1,0,'v'},
299 {"allow-undef-tags",0,0,'u'},
300 {0,0,0,0}
301 };
302 while ((c = getopt_long(argc, argv, "hlr:R:F:f:a:s:S:Ht:T:c:v:i:e:o:uN",loptions,NULL)) >= 0) {
303 switch (c) {
304 case 'o': args->fn_out = optarg; break;
305 case 'F': args->print_filtered = optarg; break;
306 case 'f': args->format_str = strdup(optarg); break;
307 case 'N': args->force_newline = 0; break;
308 case 'H': args->print_header++; break;
309 case 'v': args->vcf_list = optarg; break;
310 case 'c':
311 error("The --collapse option is obsolete, pipe through `bcftools norm -c` instead.\n");
312 break;
313 case 'a':
314 {
315 kstring_t str = {0,0,0};
316 kputs("%CHROM\t%POS\t%MASK\t%REF\t%ALT\t%", &str);
317 char *p = optarg;
318 while ( *p )
319 {
320 if ( *p==',' )
321 kputs("\t%", &str);
322 else
323 kputc(*p, &str);
324 p++;
325 }
326 kputc('\n', &str);
327 args->format_str = str.s;
328 break;
329 }
330 case 'e':
331 if ( args->filter_str ) error("Error: only one -i or -e expression can be given, and they cannot be combined\n");
332 args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
333 case 'i':
334 if ( args->filter_str ) error("Error: only one -i or -e expression can be given, and they cannot be combined\n");
335 args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
336 case 'r': args->regions_list = optarg; break;
337 case 'R': args->regions_list = optarg; regions_is_file = 1; break;
338 case 't': args->targets_list = optarg; break;
339 case 'T': args->targets_list = optarg; targets_is_file = 1; break;
340 case 'l': args->list_columns = 1; break;
341 case 'u': args->allow_undef_tags = 1; break;
342 case 's': args->sample_list = optarg; break;
343 case 'S': args->sample_list = optarg; args->sample_is_file = 1; break;
344 case 1 :
345 regions_overlap = parse_overlap_option(optarg);
346 if ( regions_overlap < 0 ) error("Could not parse: --regions-overlap %s\n",optarg);
347 break;
348 case 2 :
349 targets_overlap = parse_overlap_option(optarg);
350 if ( targets_overlap < 0 ) error("Could not parse: --targets-overlap %s\n",optarg);
351 break;
352 case 3 : args->force_samples = 1; break;
353 case 'h':
354 case '?': usage(); break;
355 default: error("Unknown argument: %s\n", optarg);
356 }
357 }
358
359 char *fname = NULL;
360 if ( optind>=argc )
361 {
362 if ( !isatty(fileno((FILE *)stdin)) ) fname = "-";
363 }
364 else fname = argv[optind];
365
366 if ( args->list_columns )
367 {
368 if ( !fname ) error("Missing the VCF file name\n");
369 args->files = bcf_sr_init();
370 if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
371 list_columns(args);
372 bcf_sr_destroy(args->files);
373 free(args);
374 return 0;
375 }
376
377 if ( !args->format_str )
378 {
379 if ( argc==1 && !fname ) usage();
380 error("Error: Missing the --format option\n");
381 }
382 args->out = args->fn_out ? fopen(args->fn_out, "w") : stdout;
383 if ( !args->out ) error("%s: %s\n", args->fn_out,strerror(errno));
384
385 if ( !args->vcf_list )
386 {
387 if ( !fname ) usage();
388 args->files = bcf_sr_init();
389 if ( optind+1 < argc ) args->files->require_index = 1;
390 if ( args->regions_list )
391 {
392 bcf_sr_set_opt(args->files,BCF_SR_REGIONS_OVERLAP,regions_overlap);
393 if ( bcf_sr_set_regions(args->files, args->regions_list, regions_is_file)<0 )
394 error("Failed to read the regions: %s\n", args->regions_list);
395 }
396 if ( args->targets_list )
397 {
398 bcf_sr_set_opt(args->files,BCF_SR_TARGETS_OVERLAP,targets_overlap);
399 if ( bcf_sr_set_targets(args->files, args->targets_list, targets_is_file, 0)<0 )
400 error("Failed to read the targets: %s\n", args->targets_list);
401 }
402 while ( fname )
403 {
404 if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
405 fname = ++optind < argc ? argv[optind] : NULL;
406 }
407 init_data(args);
408 query_vcf(args);
409 free(args->format_str);
410 destroy_data(args);
411 bcf_sr_destroy(args->files);
412 if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);
413 free(args);
414 return 0;
415 }
416
417 // multiple VCFs
418 int i, k, nfiles, prev_nsamples = 0;
419 char **fnames, **prev_samples = NULL;
420 fnames = hts_readlist(args->vcf_list, 1, &nfiles);
421 if ( !fnames ) error("Error: failed to read %s\n", args->vcf_list);
422 if ( !nfiles ) error("No files in %s?\n", args->vcf_list);
423 for (i=0; i<nfiles; i++)
424 {
425 args->files = bcf_sr_init();
426 args->files->collapse = collapse;
427 if ( args->regions_list && bcf_sr_set_regions(args->files, args->regions_list, regions_is_file)<0 )
428 error("Failed to read the regions: %s\n", args->regions_list);
429 if ( optind < argc ) args->files->require_index = 1;
430 if ( args->targets_list )
431 {
432 if ( bcf_sr_set_targets(args->files, args->targets_list,targets_is_file, 0)<0 )
433 error("Failed to read the targets: %s\n", args->targets_list);
434 }
435 if ( !bcf_sr_add_reader(args->files, fnames[i]) ) error("Failed to open %s: %s\n", fnames[i],bcf_sr_strerror(args->files->errnum));
436 for (k=optind; k<argc; k++)
437 if ( !bcf_sr_add_reader(args->files, argv[k]) ) error("Failed to open %s: %s\n", argv[k],bcf_sr_strerror(args->files->errnum));
438 init_data(args);
439 if ( i==0 )
440 {
441 prev_samples = copy_header(args->header, args->files->readers[0].header->samples, bcf_hdr_nsamples(args->files->readers[0].header));
442 prev_nsamples = bcf_hdr_nsamples(args->files->readers[0].header);
443 }
444 else
445 {
446 args->print_header = 0;
447 if ( compare_header(args->header, args->files->readers[0].header->samples, bcf_hdr_nsamples(args->files->readers[0].header), prev_samples, prev_nsamples) )
448 error("Different samples in %s and %s\n", fnames[i-1],fnames[i]);
449 }
450 query_vcf(args);
451 destroy_data(args);
452 bcf_sr_destroy(args->files);
453 }
454 if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);;
455 destroy_list(fnames, nfiles);
456 destroy_list(prev_samples, prev_nsamples);
457 free(args->format_str);
458 free(args);
459 return 0;
460}
461
462