Source code

Download source archive ↓

upstream-dependencies/bcftools/vcfquery.c

1/*  vcfquery.c -- Extracts fields from VCF/BCF file.
2
3    Copyright (C) 2013-2023 Genome Research Ltd.
4
5    Author: Petr Danecek <pd3@sanger.ac.uk>
6
7Permission is hereby granted, free of charge, to any person obtaining a copy
8of this software and associated documentation files (the "Software"), to deal
9in the Software without restriction, including without limitation the rights
10to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11copies of the Software, and to permit persons to whom the Software is
12furnished to do so, subject to the following conditions:
13
14The above copyright notice and this permission notice shall be included in
15all copies or substantial portions of the Software.
16
17THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23THE SOFTWARE.  */
24
25#include <stdio.h>
26#include <unistd.h>
27#include <getopt.h>
28#include <ctype.h>
29#include <string.h>
30#include <errno.h>
31#include <sys/stat.h>
32#include <sys/types.h>
33#include <htslib/vcf.h>
34#include <htslib/synced_bcf_reader.h>
35#include <htslib/khash_str2int.h>
36#include <htslib/vcfutils.h>
37#include "bcftools.h"
38#include "filter.h"
39#include "convert.h"
40#include "smpl_ilist.h"
41
42
43// Logic of the filters: include or exclude sites which match the filters?
44#define FLT_INCLUDE 1
45#define FLT_EXCLUDE 2
46
47typedef struct
48{
49    filter_t *filter;
50    char *filter_str;
51    int filter_logic;   // include or exclude sites which match the filters? One of FLT_INCLUDE/FLT_EXCLUDE
52    uint8_t *smpl_pass;
53    convert_t *convert;
54    bcf_srs_t *files;
55    bcf_hdr_t *header;
56    int sample_is_file;
57    char **argv, *format_str, *sample_list, *targets_list, *regions_list, *vcf_list, *fn_out;
58    char *print_filtered;
59    int argc, list_columns, print_header, allow_undef_tags, force_samples, force_newline;
60    FILE *out;
61}
62args_t;
63
64static void destroy_list(char **list, int n)
65{
66    int i;
67    for (i=0; i<n; i++)
68        free(list[i]);
69    free(list);
70}
71
72static void init_data(args_t *args)
73{
74    args->header = args->files->readers[0].header;
75
76    int i, nsamples = 0, *samples = NULL;
77    if ( args->sample_list && strcmp("-",args->sample_list) )
78    {
79        for (i=0; i<args->files->nreaders; i++)
80        {
81            // This tells htslib to subset samples directly when reading. Also the header is modified to
82            // include only the requested samples
83            int ret = bcf_hdr_set_samples(args->files->readers[i].header,args->sample_list,args->sample_is_file);
84            if ( ret<0 ) error("Error parsing the sample list\n");
85            else if ( ret>0 && !args->force_samples )
86                error("Error: sample #%d not found in the header, user --force-samples to proceed anyway\n", ret);
87        }
88
89        int flags = SMPL_REORDER;
90        smpl_ilist_t *ilist = smpl_ilist_init(args->files->readers[0].header, args->sample_list, args->sample_is_file, flags);
91        nsamples = ilist->n;
92        samples = (int*) malloc(sizeof(int)*nsamples);
93        for (i=0; i<ilist->n; i++)
94            samples[i] = ilist->idx[i];
95        smpl_ilist_destroy(ilist);
96    }
97    args->convert = convert_init(args->header, samples, nsamples, args->format_str);
98    if ( args->force_newline ) convert_set_option(args->convert, force_newline, 1);
99    convert_set_option(args->convert, subset_samples, &args->smpl_pass);
100    if ( args->allow_undef_tags ) convert_set_option(args->convert, allow_undef_tags, 1);
101    if ( args->print_header>1 ) convert_set_option(args->convert, no_hdr_indices, 1);
102    free(samples);
103
104    int max_unpack = convert_max_unpack(args->convert);
105    if ( args->filter_str )
106    {
107        args->filter = filter_init(args->header, args->filter_str);
108        max_unpack |= filter_max_unpack(args->filter);
109    }
110    args->files->max_unpack = max_unpack;
111    if ( !args->filter || args->print_filtered || !(filter_max_unpack(args->filter) & BCF_UN_FMT) )
112        convert_set_option(args->convert, header_samples, 1);
113    if ( args->print_filtered ) convert_set_option(args->convert, print_filtered, args->print_filtered);
114}
115
116static void destroy_data(args_t *args)
117{
118    convert_destroy(args->convert);
119    if ( args->filter )
120        filter_destroy(args->filter);
121}
122
123static void query_vcf(args_t *args)
124{
125    kstring_t str = {0,0,0};
126
127    if ( args->print_header )
128    {
129        convert_header(args->convert,&str);
130        if ( fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
131    }
132
133    int i,max_convert_unpack = convert_max_unpack(args->convert);
134    int max_filter_unpack = args->filter ? filter_max_unpack(args->filter) : 0;
135    while ( bcf_sr_next_line(args->files) )
136    {
137        if ( !bcf_sr_has_line(args->files,0) ) continue;
138        bcf1_t *line = args->files->readers[0].buffer[0];
139        bcf_unpack(line, args->files->max_unpack);
140
141        if ( args->filter )
142        {
143            int pass = filter_test(args->filter, line, (const uint8_t**) &args->smpl_pass);
144            if ( args->filter_logic & FLT_EXCLUDE )
145            {
146                // This code addresses this problem:
147                //  -i can include a site but exclude a sample
148                //  -e exclude a site but include a sample
149
150                if ( pass )
151                {
152                    if ( !args->smpl_pass ) continue;
153                    if ( !(max_convert_unpack & BCF_UN_FMT) && !(max_filter_unpack & BCF_UN_FMT) ) continue;
154
155                    pass = 0;
156                    for (i=0; i<line->n_sample; i++)
157                    {
158                        if ( args->smpl_pass[i] ) args->smpl_pass[i] = 0;
159                        else { args->smpl_pass[i] = 1; pass = 1; }
160                    }
161                    if ( !pass ) continue;
162                }
163                else if ( args->smpl_pass )
164                    for (i=0; i<line->n_sample; i++) args->smpl_pass[i] = 1;
165            }
166            else if ( !pass ) continue;
167        }
168
169        str.l = 0;
170        convert_line(args->convert, line, &str);
171        if ( str.l && fwrite(str.s, str.l, 1, args->out)!=1 ) error("[%s] Error: cannot write to %s\n", __func__,args->fn_out?args->fn_out:"standard output");
172    }
173    if ( str.m ) free(str.s);
174}
175
176static void list_columns(args_t *args)
177{
178    int negate = 0;
179    int i;
180    bcf_sr_t *reader = &args->files->readers[0];
181    void *has_sample = NULL;
182    if ( args->sample_list )
183    {
184        if ( args->sample_list[0]=='^' ) negate = 1;
185        has_sample = khash_str2int_init();
186        int i, nsmpl;
187        char **smpl = hts_readlist(negate ? args->sample_list+1 : args->sample_list, args->sample_is_file, &nsmpl);
188        if ( !smpl ) error("Error: failed to read %s\n", negate ? args->sample_list+1 : args->sample_list);
189        for (i=0; i<nsmpl; i++)
190        {
191            if ( bcf_hdr_id2int(reader->header,BCF_DT_SAMPLE,smpl[i])<0 && !args->force_samples )
192                error("Error: sample #%d not found in the header, user --force-samples to proceed anyway\n", i+1);
193            khash_str2int_inc(has_sample, smpl[i]);
194        }
195        free(smpl);
196    }
197
198    for (i=0; i<bcf_hdr_nsamples(reader->header); i++)
199    {
200        int skip = 0;
201        if ( negate )
202        {
203            if ( khash_str2int_has_key(has_sample, reader->header->samples[i]) ) skip = 1;
204        }
205        else if ( has_sample && !khash_str2int_has_key(has_sample, reader->header->samples[i]) ) skip = 1;
206        if ( skip ) continue;
207        printf("%s\n", reader->header->samples[i]);
208    }
209
210    if ( has_sample )
211        khash_str2int_destroy_free(has_sample);
212}
213
214static char **copy_header(bcf_hdr_t *hdr, char **src, int nsrc)
215{
216    char **dst = (char**) malloc(sizeof(char*)*nsrc);
217    int i;
218    for (i=0; i<nsrc; i++) dst[i] = strdup(src[i]);
219    return dst;
220}
221static int compare_header(bcf_hdr_t *hdr, char **a, int na, char **b, int nb)
222{
223    if ( na!=nb ) return na-nb;
224    int i;
225    for (i=0; i<na; i++)
226        if ( strcmp(a[i],b[i]) ) return 1;
227    return 0;
228}
229
230
231static void usage(void)
232{
233    fprintf(stderr, "\n");
234    fprintf(stderr, "About:   Extracts fields from VCF/BCF file and prints them in user-defined format\n");
235    fprintf(stderr, "Usage:   bcftools query [options] <A.vcf.gz> [<B.vcf.gz> [...]]\n");
236    fprintf(stderr, "\n");
237    fprintf(stderr, "Options:\n");
238    fprintf(stderr, "    -e, --exclude EXPR                Exclude sites for which the expression is true (see man page for details)\n");
239    fprintf(stderr, "        --force-samples               Only warn about unknown subset samples\n");
240    fprintf(stderr, "    -F, --print-filtered STR          Output STR for samples failing the -i/-e filtering expression\n");
241    fprintf(stderr, "    -f, --format STRING               See man page for details\n");
242    fprintf(stderr, "    -H, --print-header                Print header, -HH to omit column indices\n");
243    fprintf(stderr, "    -i, --include EXPR                Select sites for which the expression is true (see man page for details)\n");
244    fprintf(stderr, "    -l, --list-samples                Print the list of samples and exit\n");
245    fprintf(stderr, "    -N, --disable-automatic-newline   Disable automatic addition of newline character when not present\n");
246    fprintf(stderr, "    -o, --output FILE                 Output file name [stdout]\n");
247    fprintf(stderr, "    -r, --regions REGION              Restrict to comma-separated list of regions\n");
248    fprintf(stderr, "    -R, --regions-file FILE           Restrict to regions listed in a file\n");
249    fprintf(stderr, "        --regions-overlap 0|1|2       Include if POS in the region (0), record overlaps (1), variant overlaps (2) [1]\n");
250    fprintf(stderr, "    -s, --samples LIST                List of samples to include\n");
251    fprintf(stderr, "    -S, --samples-file FILE           File of samples to include\n");
252    fprintf(stderr, "    -t, --targets REGION              Similar to -r but streams rather than index-jumps\n");
253    fprintf(stderr, "    -T, --targets-file FILE           Similar to -R but streams rather than index-jumps\n");
254    fprintf(stderr, "        --targets-overlap 0|1|2       Include if POS in the region (0), record overlaps (1), variant overlaps (2) [0]\n");
255    fprintf(stderr, "    -u, --allow-undef-tags            Print \".\" for undefined tags\n");
256    fprintf(stderr, "    -v, --vcf-list FILE               Process multiple VCFs listed in the file\n");
257    fprintf(stderr, "\n");
258    fprintf(stderr, "Examples:\n");
259    fprintf(stderr, "\tbcftools query -f '%%CHROM\\t%%POS\\t%%REF\\t%%ALT[\\t%%SAMPLE=%%GT]\\n' file.vcf.gz\n");
260    fprintf(stderr, "\t# For more examples see http://samtools.github.io/bcftools/bcftools.html#query\n");
261    fprintf(stderr, "\n");
262    exit(1);
263}
264
265int main_vcfquery(int argc, char *argv[])
266{
267    int c, collapse = 0;
268    args_t *args = (args_t*) calloc(1,sizeof(args_t));
269    args->argc   = argc; args->argv = argv;
270    args->force_newline = 1;
271    int regions_is_file = 0, targets_is_file = 0;
272    int regions_overlap = 1;
273    int targets_overlap = 0;
274
275    static struct option loptions[] =
276    {
277        {"help",0,0,'h'},
278        {"list-samples",0,0,'l'},
279        {"disable-automatic-newline",required_argument,NULL,'N'},
280        {"include",1,0,'i'},
281        {"exclude",1,0,'e'},
282        {"print-filtered",1,0,'F'},
283        {"format",1,0,'f'},
284        {"force-samples",0,0,3},
285        {"output-file",1,0,'o'},
286        {"output",1,0,'o'},
287        {"regions",1,0,'r'},
288        {"regions-file",1,0,'R'},
289        {"regions-overlap",required_argument,NULL,1},
290        {"targets",1,0,'t'},
291        {"targets-file",1,0,'T'},
292        {"targets-overlap",required_argument,NULL,2},
293        {"annots",1,0,'a'},
294        {"samples",1,0,'s'},
295        {"samples-file",1,0,'S'},
296        {"print-header",0,0,'H'},
297        {"collapse",1,0,'c'},
298        {"vcf-list",1,0,'v'},
299        {"allow-undef-tags",0,0,'u'},
300        {0,0,0,0}
301    };
302    while ((c = getopt_long(argc, argv, "hlr:R:F:f:a:s:S:Ht:T:c:v:i:e:o:uN",loptions,NULL)) >= 0) {
303        switch (c) {
304            case 'o': args->fn_out = optarg; break;
305            case 'F': args->print_filtered = optarg; break;
306            case 'f': args->format_str = strdup(optarg); break;
307            case 'N': args->force_newline = 0; break;
308            case 'H': args->print_header++; break;
309            case 'v': args->vcf_list = optarg; break;
310            case 'c':
311                error("The --collapse option is obsolete, pipe through `bcftools norm -c` instead.\n");
312                break;
313            case 'a':
314                {
315                    kstring_t str = {0,0,0};
316                    kputs("%CHROM\t%POS\t%MASK\t%REF\t%ALT\t%", &str);
317                    char *p = optarg;
318                    while ( *p )
319                    {
320                        if ( *p==',' )
321                            kputs("\t%", &str);
322                        else
323                            kputc(*p, &str);
324                        p++;
325                    }
326                    kputc('\n', &str);
327                    args->format_str = str.s;
328                    break;
329                }
330            case 'e':
331                if ( args->filter_str ) error("Error: only one -i or -e expression can be given, and they cannot be combined\n");
332                args->filter_str = optarg; args->filter_logic |= FLT_EXCLUDE; break;
333            case 'i':
334                if ( args->filter_str ) error("Error: only one -i or -e expression can be given, and they cannot be combined\n");
335                args->filter_str = optarg; args->filter_logic |= FLT_INCLUDE; break;
336            case 'r': args->regions_list = optarg; break;
337            case 'R': args->regions_list = optarg; regions_is_file = 1; break;
338            case 't': args->targets_list = optarg; break;
339            case 'T': args->targets_list = optarg; targets_is_file = 1; break;
340            case 'l': args->list_columns = 1; break;
341            case 'u': args->allow_undef_tags = 1; break;
342            case 's': args->sample_list = optarg; break;
343            case 'S': args->sample_list = optarg; args->sample_is_file = 1; break;
344            case  1 :
345                regions_overlap = parse_overlap_option(optarg);
346                if ( regions_overlap < 0 ) error("Could not parse: --regions-overlap %s\n",optarg);
347                break;
348            case  2 :
349                targets_overlap = parse_overlap_option(optarg);
350                if ( targets_overlap < 0 ) error("Could not parse: --targets-overlap %s\n",optarg);
351                break;
352            case  3 : args->force_samples = 1; break;
353            case 'h':
354            case '?': usage(); break;
355            default: error("Unknown argument: %s\n", optarg);
356        }
357    }
358
359    char *fname = NULL;
360    if ( optind>=argc )
361    {
362        if ( !isatty(fileno((FILE *)stdin)) ) fname = "-";
363    }
364    else fname = argv[optind];
365
366    if ( args->list_columns )
367    {
368        if ( !fname ) error("Missing the VCF file name\n");
369        args->files = bcf_sr_init();
370        if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
371        list_columns(args);
372        bcf_sr_destroy(args->files);
373        free(args);
374        return 0;
375    }
376
377    if ( !args->format_str )
378    {
379        if ( argc==1 && !fname ) usage();
380        error("Error: Missing the --format option\n");
381    }
382    args->out = args->fn_out ? fopen(args->fn_out, "w") : stdout;
383    if ( !args->out ) error("%s: %s\n", args->fn_out,strerror(errno));
384
385    if ( !args->vcf_list )
386    {
387        if ( !fname ) usage();
388        args->files = bcf_sr_init();
389        if ( optind+1 < argc ) args->files->require_index = 1;
390        if ( args->regions_list )
391        {
392            bcf_sr_set_opt(args->files,BCF_SR_REGIONS_OVERLAP,regions_overlap);
393            if ( bcf_sr_set_regions(args->files, args->regions_list, regions_is_file)<0 )
394                error("Failed to read the regions: %s\n", args->regions_list);
395        }
396        if ( args->targets_list )
397        {
398            bcf_sr_set_opt(args->files,BCF_SR_TARGETS_OVERLAP,targets_overlap);
399            if ( bcf_sr_set_targets(args->files, args->targets_list, targets_is_file, 0)<0 )
400                error("Failed to read the targets: %s\n", args->targets_list);
401        }
402        while ( fname )
403        {
404            if ( !bcf_sr_add_reader(args->files, fname) ) error("Failed to read from %s: %s\n", !strcmp("-",fname)?"standard input":fname,bcf_sr_strerror(args->files->errnum));
405            fname = ++optind < argc ? argv[optind] : NULL;
406        }
407        init_data(args);
408        query_vcf(args);
409        free(args->format_str);
410        destroy_data(args);
411        bcf_sr_destroy(args->files);
412        if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);
413        free(args);
414        return 0;
415    }
416
417    // multiple VCFs
418    int i, k, nfiles, prev_nsamples = 0;
419    char **fnames, **prev_samples = NULL;
420    fnames = hts_readlist(args->vcf_list, 1, &nfiles);
421    if ( !fnames ) error("Error: failed to read %s\n", args->vcf_list);
422    if ( !nfiles ) error("No files in %s?\n", args->vcf_list);
423    for (i=0; i<nfiles; i++)
424    {
425        args->files = bcf_sr_init();
426        args->files->collapse = collapse;
427        if ( args->regions_list && bcf_sr_set_regions(args->files, args->regions_list, regions_is_file)<0 )
428            error("Failed to read the regions: %s\n", args->regions_list);
429        if ( optind < argc ) args->files->require_index = 1;
430        if ( args->targets_list )
431        {
432            if ( bcf_sr_set_targets(args->files, args->targets_list,targets_is_file, 0)<0 )
433                error("Failed to read the targets: %s\n", args->targets_list);
434        }
435        if ( !bcf_sr_add_reader(args->files, fnames[i]) ) error("Failed to open %s: %s\n", fnames[i],bcf_sr_strerror(args->files->errnum));
436        for (k=optind; k<argc; k++)
437            if ( !bcf_sr_add_reader(args->files, argv[k]) ) error("Failed to open %s: %s\n", argv[k],bcf_sr_strerror(args->files->errnum));
438        init_data(args);
439        if ( i==0 )
440        {
441            prev_samples = copy_header(args->header, args->files->readers[0].header->samples, bcf_hdr_nsamples(args->files->readers[0].header));
442            prev_nsamples = bcf_hdr_nsamples(args->files->readers[0].header);
443        }
444        else
445        {
446            args->print_header = 0;
447            if ( compare_header(args->header, args->files->readers[0].header->samples, bcf_hdr_nsamples(args->files->readers[0].header), prev_samples, prev_nsamples) )
448                error("Different samples in %s and %s\n", fnames[i-1],fnames[i]);
449        }
450        query_vcf(args);
451        destroy_data(args);
452        bcf_sr_destroy(args->files);
453    }
454    if ( fclose(args->out)!=0 ) error("[%s] Error: close failed .. %s\n", __func__,args->fn_out);;
455    destroy_list(fnames, nfiles);
456    destroy_list(prev_samples, prev_nsamples);
457    free(args->format_str);
458    free(args);
459    return 0;
460}
461
462