Source code

Download source archive ↓

upstream-dependencies/samtools/bam_fastq.c

1/*  bam_fastq.c -- FASTA and FASTQ file generation
2
3    Copyright (C) 2009-2017, 2019-2020, 2023-2024 Genome Research Ltd.
4    Portions copyright (C) 2009, 2011, 2012 Broad Institute.
5
6    Author: Heng Li <lh3@sanger.ac.uk>
7
8Permission is hereby granted, free of charge, to any person obtaining a copy
9of this software and associated documentation files (the "Software"), to deal
10in the Software without restriction, including without limitation the rights
11to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12copies of the Software, and to permit persons to whom the Software is
13furnished to do so, subject to the following conditions:
14
15The above copyright notices and this permission notice shall be included in
16all copies or substantial portions of the Software.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE.  */
25
26#include <config.h>
27
28#include <stdlib.h>
29#include <string.h>
30#include <strings.h>
31#include <stdbool.h>
32#include <ctype.h>
33#include <assert.h>
34#include <inttypes.h>
35#include <unistd.h>
36#include <float.h>
37
38#include "htslib/sam.h"
39#include "htslib/klist.h"
40#include "htslib/kstring.h"
41#include "htslib/bgzf.h"
42#include "htslib/thread_pool.h"
43#include "htslib/khash.h"
44#include "samtools.h"
45#include "sam_opts.h"
46
47KHASH_SET_INIT_STR(str)
48typedef khash_t(str) strhash_t;
49
50#define DEFAULT_BARCODE_TAG "BC"
51#define DEFAULT_QUALITY_TAG "QT"
52#define INDEX_SEPARATOR "+"
53
54int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };
55static void bam2fq_usage(FILE *to, const char *command)
56{
57    int fq = strcasecmp("fastq", command) == 0 || strcasecmp("bam2fq", command) == 0;
58    fprintf(to,
59"Usage: samtools %s [options...] <in.bam>\n", command);
60    fprintf(to,
61"\n"
62"Description:\n"
63"Converts a SAM, BAM or CRAM to %s format.\n"
64"\n"
65"Options:\n"
66"  -0 FILE      write reads designated READ_OTHER to FILE\n"
67"  -1 FILE      write reads designated READ1 to FILE\n"
68"  -2 FILE      write reads designated READ2 to FILE\n"
69"  -o FILE      write reads designated READ1 or READ2 to FILE\n"
70"               note: if a singleton file is specified with -s, only\n"
71"               paired reads will be written to the -1 and -2 files.\n"
72"  -d, --tag TAG[:VAL]\n"
73"               only include reads containing TAG, optionally with value VAL\n"
74"  -D, --tag-file STR:FILE\n"
75"               only include reads containing TAG, with a value listed in FILE\n"
76"  -f, --require-flags INT\n"
77"               only include reads with all  of the FLAGs in INT present [0]\n"       //   F&x == x
78"  -F, --excl[ude]-flags INT\n"
79"               only include reads with none of the FLAGs in INT present [0x900]\n"   //   F&x == 0
80"      --rf, --incl[ude]-flags INT\n"
81"               only include reads with any  of the FLAGs in INT present [0]\n"       // !(F&x == 0)
82"  -G INT       only EXCLUDE reads with all  of the FLAGs in INT present [0]\n"       // !(F&x == x)
83"  -n           don't append /1 and /2 to the read name\n"
84"  -N           always append /1 and /2 to the read name\n",
85    fq ? "FASTQ" : "FASTA");
86    if (fq) fprintf(to,
87"  -O           output quality in the OQ tag if present\n");
88    fprintf(to,
89"  -s FILE      write singleton reads designated READ1 or READ2 to FILE\n"
90"  -t           copy RG, BC and QT tags to the %s header line\n",
91    fq ? "FASTQ" : "FASTA");
92    fprintf(to,
93"  -T TAGLIST   copy arbitrary tags to the %s header line, '*' for all\n",
94    fq ? "FASTQ" : "FASTA");
95    if (fq) fprintf(to,
96"  -v INT       default quality score if not given in file [1]\n"
97"  -i           add Illumina Casava 1.8 format entry to header (eg 1:N:0:ATCACG)\n"
98"  -c INT       compression level [0..9] to use when writing bgzf files [1]\n"
99"  --i1 FILE    write first index reads to FILE\n"
100"  --i2 FILE    write second index reads to FILE\n"
101"  --barcode-tag TAG\n"
102"               Barcode tag [" DEFAULT_BARCODE_TAG "]\n"
103"  --quality-tag TAG\n"
104"               Quality tag [" DEFAULT_QUALITY_TAG "]\n"
105"  --index-format STR\n"
106"               How to parse barcode and quality tags\n\n");
107    sam_global_opt_help(to, "-.--.@-.");
108    fprintf(to,
109"\n"
110"The files will be automatically compressed if the file names have a .gz\n"
111"or .bgzf extension.  The input to this program must be collated by name.\n"
112"Run 'samtools collate' or 'samtools sort -n' to achieve this.\n"
113"\n"
114"Reads are designated READ1 if FLAG READ1 is set and READ2 is not set.\n"
115"Reads are designated READ2 if FLAG READ1 is not set and READ2 is set.\n"
116"Otherwise reads are designated READ_OTHER (both flags set or both flags unset).\n"
117"Run 'samtools flags' for more information on flag codes and meanings.\n");
118    fprintf(to,
119"\n"
120"The index-format string describes how to parse the barcode and quality tags.\n"
121"It is made up of 'i' or 'n' followed by a length or '*'.  For example:\n"
122"   i14i8       The first 14 characters are index 1, the next 8 are index 2\n"
123"   n8i14       Ignore the first 8 characters, and use the next 14 for index 1\n\n"
124"If the tag contains a separator, then the numeric part can be replaced with\n"
125"'*' to mean 'read until the separator or end of tag', for example:\n"
126"   i*i*        Break the tag at the separator into index 1 and index 2\n"
127"   n*i*        Ignore the left part of the tag until the separator,\n"
128"               then use the second part of the tag as index 1\n");
129    fprintf(to,
130"\n"
131"Examples:\n"
132"To get just the paired reads in separate files, use:\n"
133"   samtools %s -1 pair1.%s -2 pair2.%s -0 /dev/null -s /dev/null -n in.bam\n"
134"\nTo get all non-supplementary/secondary reads in a single file, redirect\n"
135"the output:\n"
136"   samtools %s in.bam > all_reads.%s\n",
137            command, fq ? "fq" : "fa", fq ? "fq" : "fa",
138            command, fq ? "fq" : "fa");
139}
140
141typedef enum { READ_UNKNOWN = 0, READ_1 = 1, READ_2 = 2 } readpart;
142typedef enum { FASTA, FASTQ } fastfile;
143typedef struct bam2fq_opts {
144    char *fnse;
145    char *fnr[3];
146    char *fn_input; // pointer to input filename in argv do not free
147    bool has12, has12always, use_oq, copy_tags, illumina_tag;
148    int flag_on, flag_off, flag_alloff, flag_anyon;
149    sam_global_args ga;
150    fastfile filetype;
151    int def_qual;
152    char *barcode_tag;
153    char *quality_tag;
154    char *index_file[2];
155    char *index_format;
156    char *extra_tags;
157    char compression_level;
158    const char *filter_tag;       // -d opt
159    strhash_t *filter_tag_vals;
160} bam2fq_opts_t;
161
162typedef struct bam2fq_state {
163    samFile *fp;
164    samFile *fpse;
165    samFile *fpr[3];
166    samFile *fpi[3];
167    samFile *hstdout;
168    sam_hdr_t *h;
169    bool has12, use_oq, copy_tags, illumina_tag;
170    int flag_on, flag_off, flag_alloff, flag_anyon;
171    fastfile filetype;
172    int def_qual;
173    char *index_sequence;
174    char compression_level;
175    htsThreadPool p;
176} bam2fq_state_t;
177
178// Adds a single tag value to the filter tag value hash
179static int add_tag_value(bam2fq_opts_t *opts, char *val) {
180    if (!opts->filter_tag_vals) {
181        if (!(opts->filter_tag_vals = kh_init(str)))
182            return -1;
183    }
184
185    if (!(val = strdup(val)))
186        return -1;
187
188    int ret = 0;
189    kh_put(str, opts->filter_tag_vals, val, &ret);
190    if (ret <= 0)
191        free(val);
192
193    return ret < 0 ? -1 : 0;
194}
195
196// Adds multiple values, listed in a file
197static int add_tag_file(bam2fq_opts_t *opts, char *fn) {
198    FILE *fp;
199    kstring_t ks = {0,0};
200
201    if (!(fp = fopen(fn, "r"))) {
202        print_error_errno("fastq", "failed to open \"%s\" for reading", fn);
203        return -1;
204    }
205
206    while (ks.l = 0, kgetline(&ks, (kgets_func *)fgets, fp) >= 0) {
207        if (add_tag_value(opts, ks.s) < 0) {
208            ks_free(&ks);
209            return -1;
210        }
211    }
212
213    ks_free(&ks);
214    return 0;
215}
216
217static readpart which_readpart(const bam1_t *b)
218{
219    if ((b->core.flag & BAM_FREAD1) && !(b->core.flag & BAM_FREAD2)) {
220        return READ_1;
221    } else if ((b->core.flag & BAM_FREAD2) && !(b->core.flag & BAM_FREAD1)) {
222        return READ_2;
223    } else {
224        return READ_UNKNOWN;
225    }
226}
227
228static void free_opts(bam2fq_opts_t *opts)
229{
230    if (opts->filter_tag_vals) {
231        khint_t k;
232        for (k = 0; k < kh_end(opts->filter_tag_vals); k++)
233            if (kh_exist(opts->filter_tag_vals, k))
234                free((char *)kh_key(opts->filter_tag_vals, k));
235
236        kh_destroy(str, opts->filter_tag_vals);
237    }
238    free(opts);
239}
240
241// Make mnemonic distinct values for longoption-only options
242#define LONGOPT(c)  ((c) + 128)
243
244// return true if valid
245static bool parse_opts(int argc, char *argv[], bam2fq_opts_t** opts_out)
246{
247    // Parse args
248    bam2fq_opts_t* opts = calloc(1, sizeof(bam2fq_opts_t));
249    opts->has12 = true;
250    opts->has12always = false;
251    opts->filetype = FASTQ;
252    opts->def_qual = 1;
253    opts->barcode_tag = NULL;
254    opts->quality_tag = NULL;
255    opts->index_format = NULL;
256    opts->index_file[0] = NULL;
257    opts->index_file[1] = NULL;
258    opts->extra_tags = NULL;
259    opts->compression_level = 1;
260    opts->flag_off = BAM_FSECONDARY|BAM_FSUPPLEMENTARY;
261
262    int c;
263    sam_global_args_init(&opts->ga);
264    static const struct option lopts[] = {
265        SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
266        {"require-flags", required_argument, NULL, 'f'},
267        {"excl-flags", required_argument, NULL, 'F'},
268        {"exclude-flags", required_argument, NULL, 'F'},
269        // following the same convention as view: g exists as a longoption_only
270        // argument, accessible from the command line as --rf/--incl[ude]-flags
271        {"rf", required_argument, NULL, LONGOPT('g')},
272        {"incl-flags", required_argument, NULL, LONGOPT('g')},
273        {"include-flags", required_argument, NULL, LONGOPT('g')},
274        {"i1", required_argument, NULL, 1},
275        {"I1", required_argument, NULL, 1},
276        {"i2", required_argument, NULL, 2},
277        {"I2", required_argument, NULL, 2},
278        {"if", required_argument, NULL, 3},
279        {"IF", required_argument, NULL, 3},
280        {"index-format", required_argument, NULL, 3},
281        {"barcode-tag", required_argument, NULL, 'b'},
282        {"quality-tag", required_argument, NULL, 'q'},
283        {"tag", required_argument, NULL, 'd'},
284        {"tag-file", required_argument, NULL, 'D'},
285        { NULL, 0, NULL, 0 }
286    };
287    while ((c = getopt_long(argc, argv, "0:1:2:o:f:F:G:niNOs:c:tT:v:@:d:D:",
288                            lopts, NULL)) > 0) {
289        switch (c) {
290            case 'b': opts->barcode_tag = optarg; break;
291            case 'q': opts->quality_tag = optarg; break;
292            case  1 : opts->index_file[0] = optarg; break;
293            case  2 : opts->index_file[1] = optarg; break;
294            case  3 : opts->index_format = optarg; break;
295            case '0': opts->fnr[0] = optarg; break;
296            case '1': opts->fnr[1] = optarg; break;
297            case '2': opts->fnr[2] = optarg; break;
298            case 'o': opts->fnr[1] = optarg; opts->fnr[2] = optarg; break;
299            case 'f': opts->flag_on |= strtol(optarg, 0, 0); break;
300            // note that flag_off does not have |= because it has a default
301            // value of 0x900 which needs to be replaced by the optarg
302            case 'F': opts->flag_off = strtol(optarg, 0, 0); break;
303            case 'G': opts->flag_alloff |= strtol(optarg, 0, 0); break;
304            case LONGOPT('g'): opts->flag_anyon |= strtol(optarg, 0, 0); break;
305            case 'n': opts->has12 = false; break;
306            case 'N': opts->has12always = true; break;
307            case 'O': opts->use_oq = true; break;
308            case 's': opts->fnse = optarg; break;
309            case 't': opts->copy_tags = true; break;
310            case 'i': opts->illumina_tag = true; break;
311            case 'c':
312                opts->compression_level = atoi(optarg);
313                if (opts->compression_level < 0)
314                    opts->compression_level = 0;
315                if (opts->compression_level > 9)
316                    opts->compression_level = 9;
317                break;
318            case 'T': opts->extra_tags = optarg; break;
319            case 'v': opts->def_qual = atoi(optarg); break;
320
321            case 'd':
322                if (strlen(optarg) < 2 ||
323                    (strlen(optarg) > 2 && optarg[2] != ':')) {
324                    print_error("fastq",
325                                "Invalid \"tag:value\" option: \"%s\"",
326                                optarg);
327                    free_opts(opts);
328                    return false;
329                }
330
331                if (opts->filter_tag && memcmp(opts->filter_tag, optarg, 2)) {
332                    print_error("fastq", "Different tag type specified "
333                                "to before");
334                    free_opts(opts);
335                    return false;
336                }
337
338                if (strlen(optarg) >= 3)
339                    add_tag_value(opts, optarg+3);
340                opts->filter_tag = optarg;
341                break;
342
343            case 'D':
344                // Allow ";" as delimiter besides ":" to support MinGW CLI POSIX
345                // path translation as described at:
346                // http://www.mingw.org/wiki/Posix_path_conversion
347                if (strlen(optarg) < 4
348                    || (optarg[2] != ':' && optarg[2] != ';')) {
349                    print_error("view", "Invalid \"tag:file\" option: \"%s\"",
350                                optarg);
351                    free_opts(opts);
352                    return false;
353                }
354
355                if (opts->filter_tag && memcmp(opts->filter_tag, optarg, 2)) {
356                    print_error("fastq", "Different tag type specified "
357                                "to before");
358                    free_opts(opts);
359                    return false;
360                }
361
362                if (strlen(optarg) >= 3) {
363                    if (add_tag_file(opts, optarg+3) < 0) {
364                        free_opts(opts);
365                        return false;
366                    }
367                }
368                opts->filter_tag = optarg;
369                break;
370
371            case '?':
372                bam2fq_usage(stderr, argv[0]);
373                free_opts(opts);
374                return false;
375            default:
376                if (parse_sam_global_opt(c, optarg, lopts, &opts->ga) != 0) {
377                    bam2fq_usage(stderr, argv[0]);
378                    free_opts(opts);
379                    return false;
380                }
381                break;
382        }
383    }
384
385    if (opts->fnr[1] || opts->fnr[2]) opts->has12 = false;
386    if (opts->has12always) opts->has12 = true;
387
388    if (!opts->barcode_tag) opts->barcode_tag = DEFAULT_BARCODE_TAG;
389    if (!opts->quality_tag) opts->quality_tag = DEFAULT_QUALITY_TAG;
390
391    int nIndex = 0;
392    if (opts->index_format) {
393        char *s;
394        for (s = opts->index_format; *s; s++) {
395            if (*s == 'i') nIndex++;
396        }
397    }
398    if (nIndex>2) {
399        fprintf(stderr,"Invalid index format: more than 2 indexes\n");
400        bam2fq_usage(stderr, argv[0]);
401        free_opts(opts);
402        return false;
403    }
404
405    if (opts->index_file[1] && !opts->index_file[0]) {
406        fprintf(stderr, "Index one specified, but index two not given\n");
407        bam2fq_usage(stderr, argv[0]);
408        free_opts(opts);
409        return false;
410    }
411
412    if (opts->illumina_tag && !nIndex) {
413        fprintf(stderr, "You must specify an index format (--index-format) with the Illumina Casava (-i) option\n");
414        bam2fq_usage(stderr, argv[0]);
415        free_opts(opts);
416        return false;
417    }
418
419    if (nIndex==0 && opts->index_file[0]) {
420        fprintf(stderr, "index_format not specified, but index file given\n");
421        bam2fq_usage(stderr, argv[0]);
422        free_opts(opts);
423        return false;
424    }
425
426    if (opts->def_qual < 0 || 93 < opts->def_qual) {
427        fprintf(stderr, "Invalid -v default quality %i, allowed range 0 to 93\n", opts->def_qual);
428        bam2fq_usage(stderr, argv[0]);
429        free_opts(opts);
430        return false;
431    }
432
433    const char* type_str = argv[0];
434    if (strcasecmp("fastq", type_str) == 0 ||
435        strcasecmp("bam2fq", type_str) == 0) {
436        opts->filetype = FASTQ;
437    } else if (strcasecmp("fasta", type_str) == 0) {
438        opts->filetype = FASTA;
439    } else {
440        print_error("bam2fq", "Unrecognised type call \"%s\", this should be impossible... but you managed it!", type_str);
441        bam2fq_usage(stderr, argv[0]);
442        free_opts(opts);
443        return false;
444    }
445
446    if (argc == optind && isatty(STDIN_FILENO)) {
447        bam2fq_usage(stdout, argv[0]);
448        free_opts(opts);
449        return true;
450    }
451
452    if (argc - optind > 1) {
453        fprintf(stderr, "Too many arguments.\n");
454        bam2fq_usage(stderr, argv[0]);
455        free_opts(opts);
456        return false;
457    }
458    opts->fn_input = argc > optind ? argv[optind] : "-";
459    *opts_out = opts;
460    return true;
461}
462
463void set_sam_opts(samFile *fp, bam2fq_state_t *state,
464                  const bam2fq_opts_t *opts) {
465    if (state->has12)
466        hts_set_opt(fp, FASTQ_OPT_RNUM, 1);
467
468    if (state->illumina_tag)
469        hts_set_opt(fp, FASTQ_OPT_CASAVA, 1);
470
471    hts_set_opt(fp, FASTQ_OPT_BARCODE, opts->barcode_tag);
472
473    if (opts->extra_tags && (*opts->extra_tags == '*' || *opts->extra_tags == '\0'))
474        hts_set_opt(fp, FASTQ_OPT_AUX, NULL);
475    else {
476        kstring_t tag_list = {0,0};
477        if (state->copy_tags)
478            kputs("RG,BC,QT", &tag_list);
479        if (opts->extra_tags) {
480            if (tag_list.l)
481                kputc(',', &tag_list);
482            kputs(opts->extra_tags, &tag_list);
483        }
484        if (tag_list.l)
485            hts_set_opt(fp, FASTQ_OPT_AUX, tag_list.s);
486        ks_free(&tag_list);
487    }
488}
489
490// Open a file as normal or gzipped based on filename.
491// Note we always use bgzf and don't bother to attempt non-blocked
492// gzip streams.  This is a departure from the old fastq code.
493static samFile *sam_open_z(char *fn, char *mode, bam2fq_state_t *state) {
494    char modez[6];
495    strcpy(modez, mode);
496
497    size_t l = strlen(fn);
498    if ((l > 3 && strcmp(fn+l-3, ".gz") == 0) ||
499        (l > 4 && strcmp(fn+l-4, ".bgz") == 0) ||
500        (l > 5 && strcmp(fn+l-5, ".bgzf") == 0)) {
501        char m[3] = {'z', state->compression_level+'0', '\0'};
502        strcat(modez, m);
503    }
504
505    samFile *fp = sam_open(fn, modez);
506    if (!fp)
507        return NULL;
508
509    if (state->p.pool)
510        hts_set_thread_pool(fp, &state->p);
511
512    return fp;
513}
514
515static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
516{
517    char *mode = opts->filetype == FASTA ? "wF" : "wf";
518
519    bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
520    if (!state)
521        return false;
522    state->flag_on = opts->flag_on;
523    state->flag_off = opts->flag_off;
524    state->flag_alloff = opts->flag_alloff;
525    state->flag_anyon = opts->flag_anyon;
526    state->has12 = opts->has12;
527    state->use_oq = opts->use_oq;
528    state->illumina_tag = opts->illumina_tag;
529    state->copy_tags = opts->copy_tags;
530    state->filetype = opts->filetype;
531    state->def_qual = opts->def_qual;
532    state->index_sequence = NULL;
533    state->hstdout = NULL;
534    state->compression_level = opts->compression_level;
535
536    state->fp = sam_open_format(opts->fn_input, "r", &opts->ga.in);
537    if (state->fp == NULL) {
538        print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
539        free(state);
540        return false;
541    }
542
543    state->p.pool = NULL;
544    if (opts->ga.nthreads > 0) {
545        if (!(state->p.pool = hts_tpool_init(opts->ga.nthreads))) {
546            fprintf(stderr, "Failed to create thread pool\n");
547            free(state);
548            return false;
549        }
550        state->p.qsize = opts->ga.nthreads*2;
551        hts_set_thread_pool(state->fp, &state->p);
552    }
553
554    uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
555    if (opts->use_oq || opts->extra_tags || opts->index_file[0])
556        rf |= SAM_AUX;
557    if (opts->filter_tag) {
558        if (memcmp(opts->filter_tag, "NM", 2) == 0 ||
559            memcmp(opts->filter_tag, "MD", 2) == 0)
560            rf |= SAM_AUX | SAM_SEQ;
561        else if (memcmp(opts->filter_tag, "RG", 2) == 0)
562            rf |= SAM_RGAUX;
563        else
564            rf |= SAM_AUX;
565    }
566    if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
567        fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
568        free(state);
569        return false;
570    }
571    if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
572        fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
573        free(state);
574        return false;
575    }
576    if (opts->fnse) {
577        if (!(state->fpse = sam_open_z(opts->fnse, mode, state))) {
578            print_error_errno("bam2fq", "Cannot open singleton file \"%s\"", opts->fnse);
579            free(state);
580            return false;
581        }
582        set_sam_opts(state->fpse, state, opts);
583    }
584
585    if (opts->ga.reference) {
586        if (hts_set_fai_filename(state->fp, opts->ga.reference) != 0) {
587            print_error_errno("bam2fq", "cannot load reference \"%s\"", opts->ga.reference);
588            free(state);
589            return false;
590        }
591    }
592
593    // single, read1, read2
594    int i, j;
595    for (i = 0; i < 3; ++i) {
596        if (opts->fnr[i]) {
597            for (j = 0; j < i; j++)
598                if (opts->fnr[j] && strcmp(opts->fnr[j], opts->fnr[i]) == 0)
599                    break;
600            if (j == i) {
601                if (!(state->fpr[i] = sam_open_z(opts->fnr[i], mode, state))) {
602                    print_error_errno("bam2fq", "Cannot open r%d file \"%s\"",
603                                      i, opts->fnr[i]);
604                    free(state);
605                    return false;
606                }
607                set_sam_opts(state->fpr[i], state, opts);
608            } else {
609                state->fpr[i] = state->fpr[j];
610            }
611        } else {
612            if (!state->hstdout) {
613                if (!(state->hstdout = sam_open_z("-", mode, state))) {
614                    print_error_errno("bam2fq", "Cannot open STDOUT");
615                    free(state);
616                    return false;
617                }
618                set_sam_opts(state->hstdout, state, opts);
619                autoflush_if_stdout(state->hstdout, "-");
620            }
621            state->fpr[i] = state->hstdout;
622        }
623    }
624
625    // index 1, index 2
626    for (i = 0; i < 2; i++) {
627        state->fpi[i] = NULL;
628        if (opts->index_file[i]) {
629            for (j = 0; j < 3; j++)
630                if (opts->fnr[j] && strcmp(opts->fnr[j], opts->index_file[i]) == 0)
631                    break;
632            for (j -= 3; j >= 0 && j < i; j++)
633                if (opts->index_file[j] && strcmp(opts->index_file[j], opts->index_file[i]) == 0)
634                    break;
635            if (i == j) {
636                if (!(state->fpi[i] = sam_open_z(opts->index_file[i], mode,
637                                                 state))) {
638                    print_error_errno("bam2fq", "Cannot open i%d file \"%s\"",
639                                      i+1, opts->index_file[i]);
640                    free(state);
641                    return false;
642                }
643                set_sam_opts(state->fpi[i], state, opts);
644            } else if (j < 0) {
645                state->fpi[i] = state->fpr[j+3];
646            } else {
647                state->fpi[i] = state->fpi[j];
648            }
649        }
650    }
651
652    state->h = sam_hdr_read(state->fp);
653    if (state->h == NULL) {
654        fprintf(stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
655        free(state);
656        return false;
657    }
658
659    *state_out = state;
660    return true;
661}
662
663static bool destroy_state(const bam2fq_opts_t *opts, bam2fq_state_t *state, int* status)
664{
665    bool valid = true;
666    sam_hdr_destroy(state->h);
667    check_sam_close("bam2fq", state->fp, opts->fn_input, "file", status);
668    if (state->fpse && sam_close(state->fpse) < 0) {
669        print_error_errno("bam2fq", "Error closing singleton file \"%s\"", opts->fnse);
670        valid = false;
671    }
672
673    int i, j;
674    for (i = 0; i < 3; ++i) {
675        if (state->fpr[i] != state->hstdout) {
676            for (j = 0; j < i; j++)
677                if (state->fpr[i] == state->fpr[j])
678                    break;
679            if (j == i && sam_close(state->fpr[i])) {
680                print_error_errno("bam2fq", "Error closing r%d file \"%s\"", i, opts->fnr[i]);
681                valid = false;
682            }
683        }
684    }
685    if (state->hstdout) {
686        release_autoflush(state->hstdout);
687        if (sam_close(state->hstdout) < 0) {
688            print_error_errno("bam2fq", "Error closing STDOUT");
689            valid = false;
690        }
691    }
692    for (i = 0; i < 2; i++) {
693        for (j = 0; j < 3; j++)
694            if (state->fpi[i] == state->fpr[j])
695                break;
696        for (j -= 3; j >= 0 && j < i; j++)
697            if (state->fpi[i] == state->fpi[j])
698                break;
699        if (j == i && state->fpi[i] && sam_close(state->fpi[i]) < 0) {
700            print_error_errno("bam2fq", "Error closing i%d file \"%s\"", i+1, opts->index_file[i]);
701            valid = false;
702        }
703    }
704    free(state->index_sequence);
705    if (state->p.pool)
706        hts_tpool_destroy(state->p.pool);
707    free(state);
708    return valid;
709}
710
711static inline bool filter_it_out(const bam1_t *b, const bam2fq_state_t *state,
712                                 bam2fq_opts_t *opts)
713{
714    if (opts->filter_tag) {
715        uint8_t *s = bam_aux_get(b, opts->filter_tag);
716        if (!s)
717            return true;
718
719        if (opts->filter_tag_vals) {
720            char t[32], *val = t;
721            switch (*s) {
722            case 'i': case 'I':
723            case 's': case 'S':
724            case 'c': case 'C':
725                if (snprintf(t, 32, "%"PRId64, bam_aux2i(s)) <= 0)
726                    return true;
727                break;
728
729            case 'f':
730                // Comparing floats is hard.
731                // Eg (double)0.1 - (double)0.1f is -1.5e-9.
732                // Given BAM binary encoding is float however, just keep it.
733                // This means rounding errors will (hopefully) always be the
734                // same and basic equality still works.
735                if (snprintf(t, 32, "%f", (float)bam_aux2f(s)) <= 0)
736                    return true;
737                break;
738
739            case 'A':
740                t[0] = s[1];
741                t[1] = 0;
742                break;
743
744            case 'Z': case 'H':
745                val = (char *)s+1;
746                break;
747
748            default:
749                // Anything unsupported fails the filter match too.
750                return true;
751            }
752
753            khint_t k = kh_get(str, opts->filter_tag_vals, val);
754            if (k == kh_end(opts->filter_tag_vals))
755                return 1; // tag value not found
756        }
757    }
758
759    return ((b->core.flag&(state->flag_on)) != state->flag_on // or reads indicated by filter flags
760        ||  (b->core.flag&(state->flag_off)) != 0
761        ||  (((b->core.flag&(state->flag_anyon)) == 0) && (state->flag_anyon != 0))
762        ||  (b->core.flag&(state->flag_alloff) && (b->core.flag&(state->flag_alloff)) == state->flag_alloff));
763
764}
765
766int write_index_rec(samFile *fp, bam1_t *b, bam2fq_state_t *state,
767                    bam2fq_opts_t* opts, char *seq, int seq_len,
768                    char *qual, int qual_len) {
769    if (!fp || !b || !seq_len)
770        return 0;
771
772    int ret = -1;
773    bam1_t *b2 = bam_init1(); // FIXME: reuse
774    if (!b2)
775        return -1;
776
777    size_t aux_len = b->data + b->l_data - bam_get_aux(b);
778    if (bam_set1(b2, b->core.l_qname, bam_get_qname(b),
779                 (b->core.flag | BAM_FUNMAP) & ~BAM_FREVERSE,
780                 -1, -1, 0,    // refid, pos, mapq
781                 0, NULL,      // cigar
782                 -1, -1, 0,    // rnext, pnext, tlen
783                 seq_len, seq, qual,
784                 aux_len) < 0)
785        goto err;
786
787    uint8_t *q = bam_get_qual(b2);
788    if (qual) {
789        int i;
790        for (i = 0; i < seq_len; i++)
791            q[i] -= '!';
792    } else {
793        memset(q, opts->def_qual, seq_len);
794    }
795
796    memcpy(bam_get_aux(b2), bam_get_aux(b), aux_len);
797    b2->l_data += aux_len;
798    if (sam_write1(fp, state->h, b2) < 0)
799        goto err;
800
801    ret = 0;
802 err:
803    if (b2)
804        bam_destroy1(b2);
805    return ret;
806}
807
808int output_index(bam1_t *b1, bam1_t *b2, bam2fq_state_t *state,
809                 bam2fq_opts_t* opts) {
810    bam1_t *b = b1 ? b1 : b2;
811
812    char *ifmt = opts->index_format;
813    if (!ifmt)
814        ifmt = "i*i*";
815
816    // Get seq / qual elements
817    char *bc = NULL, *qt = NULL;
818    if (b1)
819        bc = (char *)bam_aux_get(b1, opts->barcode_tag);
820    if (b2 && !bc)
821        bc = (char *)bam_aux_get(b2, opts->barcode_tag);
822    if (!bc)
823        return 0;
824    else
825        bc++; // skip Z
826
827    if (b1)
828        qt = (char *)bam_aux_get(b1, opts->quality_tag);
829    if (b2 && !qt)
830        qt = (char *)bam_aux_get(b2, opts->quality_tag);
831    if (qt && strlen(bc) != strlen(qt)-1)
832        qt = NULL;
833    else if (qt)
834        qt++;
835
836    int inum = 0;
837    while (inum < 2) {
838        char fc = *ifmt++;
839        if (!fc)
840            break; // ran out of index-format
841
842        long len, rem = 0;
843        if (isdigit(*ifmt)) {
844            rem = len = strtol(ifmt, &ifmt, 10);
845        } else {
846            ifmt++;
847            len = 0;
848        }
849
850        char *bc_end = bc, *qt_end = qt;
851        if (qt) {
852            while (len ? *bc_end && rem-- : isalpha(*bc_end))
853                bc_end++, qt_end++;
854        } else {
855            while (len ? *bc_end && rem-- : isalpha(*bc_end))
856                bc_end++;
857        }
858
859        switch (fc) {
860        case 'n':
861            // skip
862            bc = bc_end + (len==0);
863            if (qt)
864                qt = qt_end + (len==0);
865            break;
866
867        case 'i':
868            if (write_index_rec(state->fpi[inum], b, state, opts,
869                                bc, bc_end-bc, qt, qt_end-qt) < 0)
870                return -1;
871            bc = bc_end + (len==0);
872            if (qt)
873                qt = qt_end + (len==0);
874            inum++;
875            break;
876
877        default:
878            fprintf(stderr, "Unknown index-format code\n");
879            return -1;
880        }
881    }
882
883    return 0;
884}
885
886static int flush_rec(bam2fq_state_t *state, bam2fq_opts_t* opts,
887                     bam1_t *b[4], int score[3], int best[3],
888                     int64_t *n_singletons) {
889    // Paired data, with 1 or 2 ends present.
890    if (score[1] > 0 && score[2] > 0) {
891        // If CASAVA tag is required and barcode is only on R1,
892        // copy it to R2
893        if (state->illumina_tag) {
894            char *tag;
895            if ((tag = (char *)bam_aux_get(b[best[1]],
896                                           opts->barcode_tag)))
897                if (bam_aux_update_str(b[best[2]],
898                                       opts->barcode_tag,
899                                       strlen(tag), tag+1) < 0)
900                    goto err;
901            if ((tag = (char *)bam_aux_get(b[best[1]],
902                                           opts->quality_tag)))
903                if (bam_aux_update_str(b[best[2]],
904                                       opts->quality_tag,
905                                       strlen(tag), tag+1) < 0)
906                    goto err;
907
908        }
909        if (sam_write1(state->fpr[1], state->h, b[best[1]]) < 0)
910            goto err;
911        if (sam_write1(state->fpr[2], state->h, b[best[2]]) < 0)
912            goto err;
913
914        if (output_index(b[best[1]], b[best[2]], state, opts) < 0)
915            goto err;
916    } else if (score[1] > 0 || score[2] > 0) {
917        if (state->fpse) {
918            // print whichever one exists to fpse
919            if (score[1] > 0) {
920                if (sam_write1(state->fpse, state->h, b[best[1]]) < 0)
921                    goto err;
922            } else {
923                if (sam_write1(state->fpse, state->h, b[best[2]]) < 0)
924                    goto err;
925            }
926            ++(*n_singletons);
927        } else {
928            if (score[1] > 0) {
929                if (sam_write1(state->fpr[1], state->h, b[best[1]]) < 0)
930                    goto err;
931            } else {
932                if (sam_write1(state->fpr[2], state->h, b[best[2]]) < 0)
933                    goto err;
934            }
935        }
936
937        if (output_index(score[1] > 0 ? b[best[1]] : NULL,
938                         score[2] > 0 ? b[best[2]] : NULL,
939                         state, opts) < 0)
940            goto err;
941    }
942
943    if (score[0]) { // single ended data (neither READ1 nor READ2)
944        if (sam_write1(state->fpr[0], state->h, b[best[0]]) < 0)
945            goto err;
946
947        if (output_index(b[best[0]], NULL, state, opts) < 0)
948            goto err;
949    }
950
951    return 0;
952
953 err:
954    return -1;
955}
956
957static bool bam2fq_mainloop(bam2fq_state_t *state, bam2fq_opts_t* opts)
958{
959    int n;
960    char *current_qname = NULL;
961    int64_t n_reads = 0, n_singletons = 0; // Statistics
962    int score[3];
963    int at_eof;
964    bool valid = false;
965    int best[3] = {-1, -1, -1}; // map R0, R1, single to b[] indices;
966                                // indexed by [readpart]
967    bam1_t *b[4];               // 3 readparts, plus current record
968
969    for (n = 0; n < 4; n++) {
970        if (!(b[n] = bam_init1())) {
971            perror("[bam2fq_mainloop] Malloc error for bam record buffer.");
972            return false;
973        }
974    }
975
976    n = 0;
977    while (true) {
978        int res = sam_read1(state->fp, state->h, b[n]);
979        if (res < -1) {
980            print_error("bam2fq", "Failed to read bam record");
981            goto err;
982        }
983        at_eof = res < 0;
984
985        if (!at_eof && filter_it_out(b[n], state, opts))
986            continue;
987        if (!at_eof) {
988            ++n_reads;
989
990            // Handle -O option: use OQ for qual
991            uint8_t *oq;
992            if (state->use_oq && (oq = bam_aux_get(b[n],"OQ")) && *oq == 'Z') {
993                int i, l = strlen((char *)++oq);
994                uint8_t *qual = bam_get_qual(b[n]);
995                for (i = 0; i < l && i < b[n]->core.l_qseq; i++)
996                    qual[i] = oq[i] - '!';
997            }
998        }
999
1000        if (at_eof
1001            || !current_qname
1002            || (strcmp(current_qname, bam_get_qname(b[n])) != 0)) {
1003            // New name, so flush best examples of previous name.
1004            if (current_qname)
1005                if (flush_rec(state, opts, b, score, best, &n_singletons) < 0)
1006                    goto err;
1007
1008            current_qname = bam_get_qname(b[n]);
1009            score[0] = score[1] = score[2] = 0;
1010
1011            if (at_eof) { break; }
1012        }
1013
1014        // Prefer a copy of the read that has base qualities
1015        int b_score = bam_get_qual(b[n])[0] != 0xff? 2 : 1;
1016        readpart rp = which_readpart(b[n]);
1017        if (score[rp] < b_score) {
1018            score[rp] = b_score;
1019            // Record b[n] slot for best copy of readpair and find a new
1020            // slot for next bam read
1021            best[rp] = n;
1022            int used_slot[4] = {0}, i;
1023            for (i = 0; i < 3; i++)
1024                if (best[i] >= 0)
1025                    used_slot[best[i]] = 1;
1026            for (i = 0; i < 4 && used_slot[i]; i++)
1027                ;
1028            n = i;
1029        }
1030    }
1031
1032    valid = true;
1033 err:
1034    if (!valid)
1035        print_error_errno("bam2fq", "Error writing to FASTx files.");
1036
1037    for (n = 0; n < 4; n++)
1038        bam_destroy1(b[n]);
1039
1040    fprintf(stderr, "[M::%s] discarded %" PRId64 " singletons\n",
1041            __func__, n_singletons);
1042    fprintf(stderr, "[M::%s] processed %" PRId64 " reads\n",
1043            __func__, n_reads);
1044
1045    return valid;
1046}
1047
1048int main_bam2fq(int argc, char *argv[])
1049{
1050    int status = EXIT_FAILURE;
1051    bam2fq_opts_t* opts = NULL;
1052    bam2fq_state_t* state = NULL;
1053
1054    bool valid = parse_opts(argc, argv, &opts);
1055    if (!valid || opts == NULL) return valid ? EXIT_SUCCESS : EXIT_FAILURE;
1056
1057    if (!init_state(opts, &state)) goto err;
1058
1059    if (!bam2fq_mainloop(state,opts)) goto err;
1060
1061    if (!destroy_state(opts, state, &status)) goto err;
1062
1063    status = EXIT_SUCCESS;
1064 err:
1065    sam_global_args_free(&opts->ga);
1066    free_opts(opts);
1067
1068    return status;
1069}