upstream-dependencies/samtools/bam_fastq.c
1/* bam_fastq.c -- FASTA and FASTQ file generation
2
3 Copyright (C) 2009-2017, 2019-2020, 2023-2024 Genome Research Ltd.
4 Portions copyright (C) 2009, 2011, 2012 Broad Institute.
5
6 Author: Heng Li <lh3@sanger.ac.uk>
7
8Permission is hereby granted, free of charge, to any person obtaining a copy
9of this software and associated documentation files (the "Software"), to deal
10in the Software without restriction, including without limitation the rights
11to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12copies of the Software, and to permit persons to whom the Software is
13furnished to do so, subject to the following conditions:
14
15The above copyright notices and this permission notice shall be included in
16all copies or substantial portions of the Software.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE. */
25
26#include <config.h>
27
28#include <stdlib.h>
29#include <string.h>
30#include <strings.h>
31#include <stdbool.h>
32#include <ctype.h>
33#include <assert.h>
34#include <inttypes.h>
35#include <unistd.h>
36#include <float.h>
37
38#include "htslib/sam.h"
39#include "htslib/klist.h"
40#include "htslib/kstring.h"
41#include "htslib/bgzf.h"
42#include "htslib/thread_pool.h"
43#include "htslib/khash.h"
44#include "samtools.h"
45#include "sam_opts.h"
46
47KHASH_SET_INIT_STR(str)
48typedef khash_t(str) strhash_t;
49
50#define DEFAULT_BARCODE_TAG "BC"
51#define DEFAULT_QUALITY_TAG "QT"
52#define INDEX_SEPARATOR "+"
53
54int8_t seq_comp_table[16] = { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 };
55static void bam2fq_usage(FILE *to, const char *command)
56{
57 int fq = strcasecmp("fastq", command) == 0 || strcasecmp("bam2fq", command) == 0;
58 fprintf(to,
59"Usage: samtools %s [options...] <in.bam>\n", command);
60 fprintf(to,
61"\n"
62"Description:\n"
63"Converts a SAM, BAM or CRAM to %s format.\n"
64"\n"
65"Options:\n"
66" -0 FILE write reads designated READ_OTHER to FILE\n"
67" -1 FILE write reads designated READ1 to FILE\n"
68" -2 FILE write reads designated READ2 to FILE\n"
69" -o FILE write reads designated READ1 or READ2 to FILE\n"
70" note: if a singleton file is specified with -s, only\n"
71" paired reads will be written to the -1 and -2 files.\n"
72" -d, --tag TAG[:VAL]\n"
73" only include reads containing TAG, optionally with value VAL\n"
74" -D, --tag-file STR:FILE\n"
75" only include reads containing TAG, with a value listed in FILE\n"
76" -f, --require-flags INT\n"
77" only include reads with all of the FLAGs in INT present [0]\n" // F&x == x
78" -F, --excl[ude]-flags INT\n"
79" only include reads with none of the FLAGs in INT present [0x900]\n" // F&x == 0
80" --rf, --incl[ude]-flags INT\n"
81" only include reads with any of the FLAGs in INT present [0]\n" // !(F&x == 0)
82" -G INT only EXCLUDE reads with all of the FLAGs in INT present [0]\n" // !(F&x == x)
83" -n don't append /1 and /2 to the read name\n"
84" -N always append /1 and /2 to the read name\n",
85 fq ? "FASTQ" : "FASTA");
86 if (fq) fprintf(to,
87" -O output quality in the OQ tag if present\n");
88 fprintf(to,
89" -s FILE write singleton reads designated READ1 or READ2 to FILE\n"
90" -t copy RG, BC and QT tags to the %s header line\n",
91 fq ? "FASTQ" : "FASTA");
92 fprintf(to,
93" -T TAGLIST copy arbitrary tags to the %s header line, '*' for all\n",
94 fq ? "FASTQ" : "FASTA");
95 if (fq) fprintf(to,
96" -v INT default quality score if not given in file [1]\n"
97" -i add Illumina Casava 1.8 format entry to header (eg 1:N:0:ATCACG)\n"
98" -c INT compression level [0..9] to use when writing bgzf files [1]\n"
99" --i1 FILE write first index reads to FILE\n"
100" --i2 FILE write second index reads to FILE\n"
101" --barcode-tag TAG\n"
102" Barcode tag [" DEFAULT_BARCODE_TAG "]\n"
103" --quality-tag TAG\n"
104" Quality tag [" DEFAULT_QUALITY_TAG "]\n"
105" --index-format STR\n"
106" How to parse barcode and quality tags\n\n");
107 sam_global_opt_help(to, "-.--.@-.");
108 fprintf(to,
109"\n"
110"The files will be automatically compressed if the file names have a .gz\n"
111"or .bgzf extension. The input to this program must be collated by name.\n"
112"Run 'samtools collate' or 'samtools sort -n' to achieve this.\n"
113"\n"
114"Reads are designated READ1 if FLAG READ1 is set and READ2 is not set.\n"
115"Reads are designated READ2 if FLAG READ1 is not set and READ2 is set.\n"
116"Otherwise reads are designated READ_OTHER (both flags set or both flags unset).\n"
117"Run 'samtools flags' for more information on flag codes and meanings.\n");
118 fprintf(to,
119"\n"
120"The index-format string describes how to parse the barcode and quality tags.\n"
121"It is made up of 'i' or 'n' followed by a length or '*'. For example:\n"
122" i14i8 The first 14 characters are index 1, the next 8 are index 2\n"
123" n8i14 Ignore the first 8 characters, and use the next 14 for index 1\n\n"
124"If the tag contains a separator, then the numeric part can be replaced with\n"
125"'*' to mean 'read until the separator or end of tag', for example:\n"
126" i*i* Break the tag at the separator into index 1 and index 2\n"
127" n*i* Ignore the left part of the tag until the separator,\n"
128" then use the second part of the tag as index 1\n");
129 fprintf(to,
130"\n"
131"Examples:\n"
132"To get just the paired reads in separate files, use:\n"
133" samtools %s -1 pair1.%s -2 pair2.%s -0 /dev/null -s /dev/null -n in.bam\n"
134"\nTo get all non-supplementary/secondary reads in a single file, redirect\n"
135"the output:\n"
136" samtools %s in.bam > all_reads.%s\n",
137 command, fq ? "fq" : "fa", fq ? "fq" : "fa",
138 command, fq ? "fq" : "fa");
139}
140
141typedef enum { READ_UNKNOWN = 0, READ_1 = 1, READ_2 = 2 } readpart;
142typedef enum { FASTA, FASTQ } fastfile;
143typedef struct bam2fq_opts {
144 char *fnse;
145 char *fnr[3];
146 char *fn_input; // pointer to input filename in argv do not free
147 bool has12, has12always, use_oq, copy_tags, illumina_tag;
148 int flag_on, flag_off, flag_alloff, flag_anyon;
149 sam_global_args ga;
150 fastfile filetype;
151 int def_qual;
152 char *barcode_tag;
153 char *quality_tag;
154 char *index_file[2];
155 char *index_format;
156 char *extra_tags;
157 char compression_level;
158 const char *filter_tag; // -d opt
159 strhash_t *filter_tag_vals;
160} bam2fq_opts_t;
161
162typedef struct bam2fq_state {
163 samFile *fp;
164 samFile *fpse;
165 samFile *fpr[3];
166 samFile *fpi[3];
167 samFile *hstdout;
168 sam_hdr_t *h;
169 bool has12, use_oq, copy_tags, illumina_tag;
170 int flag_on, flag_off, flag_alloff, flag_anyon;
171 fastfile filetype;
172 int def_qual;
173 char *index_sequence;
174 char compression_level;
175 htsThreadPool p;
176} bam2fq_state_t;
177
178// Adds a single tag value to the filter tag value hash
179static int add_tag_value(bam2fq_opts_t *opts, char *val) {
180 if (!opts->filter_tag_vals) {
181 if (!(opts->filter_tag_vals = kh_init(str)))
182 return -1;
183 }
184
185 if (!(val = strdup(val)))
186 return -1;
187
188 int ret = 0;
189 kh_put(str, opts->filter_tag_vals, val, &ret);
190 if (ret <= 0)
191 free(val);
192
193 return ret < 0 ? -1 : 0;
194}
195
196// Adds multiple values, listed in a file
197static int add_tag_file(bam2fq_opts_t *opts, char *fn) {
198 FILE *fp;
199 kstring_t ks = {0,0};
200
201 if (!(fp = fopen(fn, "r"))) {
202 print_error_errno("fastq", "failed to open \"%s\" for reading", fn);
203 return -1;
204 }
205
206 while (ks.l = 0, kgetline(&ks, (kgets_func *)fgets, fp) >= 0) {
207 if (add_tag_value(opts, ks.s) < 0) {
208 ks_free(&ks);
209 return -1;
210 }
211 }
212
213 ks_free(&ks);
214 return 0;
215}
216
217static readpart which_readpart(const bam1_t *b)
218{
219 if ((b->core.flag & BAM_FREAD1) && !(b->core.flag & BAM_FREAD2)) {
220 return READ_1;
221 } else if ((b->core.flag & BAM_FREAD2) && !(b->core.flag & BAM_FREAD1)) {
222 return READ_2;
223 } else {
224 return READ_UNKNOWN;
225 }
226}
227
228static void free_opts(bam2fq_opts_t *opts)
229{
230 if (opts->filter_tag_vals) {
231 khint_t k;
232 for (k = 0; k < kh_end(opts->filter_tag_vals); k++)
233 if (kh_exist(opts->filter_tag_vals, k))
234 free((char *)kh_key(opts->filter_tag_vals, k));
235
236 kh_destroy(str, opts->filter_tag_vals);
237 }
238 free(opts);
239}
240
241// Make mnemonic distinct values for longoption-only options
242#define LONGOPT(c) ((c) + 128)
243
244// return true if valid
245static bool parse_opts(int argc, char *argv[], bam2fq_opts_t** opts_out)
246{
247 // Parse args
248 bam2fq_opts_t* opts = calloc(1, sizeof(bam2fq_opts_t));
249 opts->has12 = true;
250 opts->has12always = false;
251 opts->filetype = FASTQ;
252 opts->def_qual = 1;
253 opts->barcode_tag = NULL;
254 opts->quality_tag = NULL;
255 opts->index_format = NULL;
256 opts->index_file[0] = NULL;
257 opts->index_file[1] = NULL;
258 opts->extra_tags = NULL;
259 opts->compression_level = 1;
260 opts->flag_off = BAM_FSECONDARY|BAM_FSUPPLEMENTARY;
261
262 int c;
263 sam_global_args_init(&opts->ga);
264 static const struct option lopts[] = {
265 SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
266 {"require-flags", required_argument, NULL, 'f'},
267 {"excl-flags", required_argument, NULL, 'F'},
268 {"exclude-flags", required_argument, NULL, 'F'},
269 // following the same convention as view: g exists as a longoption_only
270 // argument, accessible from the command line as --rf/--incl[ude]-flags
271 {"rf", required_argument, NULL, LONGOPT('g')},
272 {"incl-flags", required_argument, NULL, LONGOPT('g')},
273 {"include-flags", required_argument, NULL, LONGOPT('g')},
274 {"i1", required_argument, NULL, 1},
275 {"I1", required_argument, NULL, 1},
276 {"i2", required_argument, NULL, 2},
277 {"I2", required_argument, NULL, 2},
278 {"if", required_argument, NULL, 3},
279 {"IF", required_argument, NULL, 3},
280 {"index-format", required_argument, NULL, 3},
281 {"barcode-tag", required_argument, NULL, 'b'},
282 {"quality-tag", required_argument, NULL, 'q'},
283 {"tag", required_argument, NULL, 'd'},
284 {"tag-file", required_argument, NULL, 'D'},
285 { NULL, 0, NULL, 0 }
286 };
287 while ((c = getopt_long(argc, argv, "0:1:2:o:f:F:G:niNOs:c:tT:v:@:d:D:",
288 lopts, NULL)) > 0) {
289 switch (c) {
290 case 'b': opts->barcode_tag = optarg; break;
291 case 'q': opts->quality_tag = optarg; break;
292 case 1 : opts->index_file[0] = optarg; break;
293 case 2 : opts->index_file[1] = optarg; break;
294 case 3 : opts->index_format = optarg; break;
295 case '0': opts->fnr[0] = optarg; break;
296 case '1': opts->fnr[1] = optarg; break;
297 case '2': opts->fnr[2] = optarg; break;
298 case 'o': opts->fnr[1] = optarg; opts->fnr[2] = optarg; break;
299 case 'f': opts->flag_on |= strtol(optarg, 0, 0); break;
300 // note that flag_off does not have |= because it has a default
301 // value of 0x900 which needs to be replaced by the optarg
302 case 'F': opts->flag_off = strtol(optarg, 0, 0); break;
303 case 'G': opts->flag_alloff |= strtol(optarg, 0, 0); break;
304 case LONGOPT('g'): opts->flag_anyon |= strtol(optarg, 0, 0); break;
305 case 'n': opts->has12 = false; break;
306 case 'N': opts->has12always = true; break;
307 case 'O': opts->use_oq = true; break;
308 case 's': opts->fnse = optarg; break;
309 case 't': opts->copy_tags = true; break;
310 case 'i': opts->illumina_tag = true; break;
311 case 'c':
312 opts->compression_level = atoi(optarg);
313 if (opts->compression_level < 0)
314 opts->compression_level = 0;
315 if (opts->compression_level > 9)
316 opts->compression_level = 9;
317 break;
318 case 'T': opts->extra_tags = optarg; break;
319 case 'v': opts->def_qual = atoi(optarg); break;
320
321 case 'd':
322 if (strlen(optarg) < 2 ||
323 (strlen(optarg) > 2 && optarg[2] != ':')) {
324 print_error("fastq",
325 "Invalid \"tag:value\" option: \"%s\"",
326 optarg);
327 free_opts(opts);
328 return false;
329 }
330
331 if (opts->filter_tag && memcmp(opts->filter_tag, optarg, 2)) {
332 print_error("fastq", "Different tag type specified "
333 "to before");
334 free_opts(opts);
335 return false;
336 }
337
338 if (strlen(optarg) >= 3)
339 add_tag_value(opts, optarg+3);
340 opts->filter_tag = optarg;
341 break;
342
343 case 'D':
344 // Allow ";" as delimiter besides ":" to support MinGW CLI POSIX
345 // path translation as described at:
346 // http://www.mingw.org/wiki/Posix_path_conversion
347 if (strlen(optarg) < 4
348 || (optarg[2] != ':' && optarg[2] != ';')) {
349 print_error("view", "Invalid \"tag:file\" option: \"%s\"",
350 optarg);
351 free_opts(opts);
352 return false;
353 }
354
355 if (opts->filter_tag && memcmp(opts->filter_tag, optarg, 2)) {
356 print_error("fastq", "Different tag type specified "
357 "to before");
358 free_opts(opts);
359 return false;
360 }
361
362 if (strlen(optarg) >= 3) {
363 if (add_tag_file(opts, optarg+3) < 0) {
364 free_opts(opts);
365 return false;
366 }
367 }
368 opts->filter_tag = optarg;
369 break;
370
371 case '?':
372 bam2fq_usage(stderr, argv[0]);
373 free_opts(opts);
374 return false;
375 default:
376 if (parse_sam_global_opt(c, optarg, lopts, &opts->ga) != 0) {
377 bam2fq_usage(stderr, argv[0]);
378 free_opts(opts);
379 return false;
380 }
381 break;
382 }
383 }
384
385 if (opts->fnr[1] || opts->fnr[2]) opts->has12 = false;
386 if (opts->has12always) opts->has12 = true;
387
388 if (!opts->barcode_tag) opts->barcode_tag = DEFAULT_BARCODE_TAG;
389 if (!opts->quality_tag) opts->quality_tag = DEFAULT_QUALITY_TAG;
390
391 int nIndex = 0;
392 if (opts->index_format) {
393 char *s;
394 for (s = opts->index_format; *s; s++) {
395 if (*s == 'i') nIndex++;
396 }
397 }
398 if (nIndex>2) {
399 fprintf(stderr,"Invalid index format: more than 2 indexes\n");
400 bam2fq_usage(stderr, argv[0]);
401 free_opts(opts);
402 return false;
403 }
404
405 if (opts->index_file[1] && !opts->index_file[0]) {
406 fprintf(stderr, "Index one specified, but index two not given\n");
407 bam2fq_usage(stderr, argv[0]);
408 free_opts(opts);
409 return false;
410 }
411
412 if (opts->illumina_tag && !nIndex) {
413 fprintf(stderr, "You must specify an index format (--index-format) with the Illumina Casava (-i) option\n");
414 bam2fq_usage(stderr, argv[0]);
415 free_opts(opts);
416 return false;
417 }
418
419 if (nIndex==0 && opts->index_file[0]) {
420 fprintf(stderr, "index_format not specified, but index file given\n");
421 bam2fq_usage(stderr, argv[0]);
422 free_opts(opts);
423 return false;
424 }
425
426 if (opts->def_qual < 0 || 93 < opts->def_qual) {
427 fprintf(stderr, "Invalid -v default quality %i, allowed range 0 to 93\n", opts->def_qual);
428 bam2fq_usage(stderr, argv[0]);
429 free_opts(opts);
430 return false;
431 }
432
433 const char* type_str = argv[0];
434 if (strcasecmp("fastq", type_str) == 0 ||
435 strcasecmp("bam2fq", type_str) == 0) {
436 opts->filetype = FASTQ;
437 } else if (strcasecmp("fasta", type_str) == 0) {
438 opts->filetype = FASTA;
439 } else {
440 print_error("bam2fq", "Unrecognised type call \"%s\", this should be impossible... but you managed it!", type_str);
441 bam2fq_usage(stderr, argv[0]);
442 free_opts(opts);
443 return false;
444 }
445
446 if (argc == optind && isatty(STDIN_FILENO)) {
447 bam2fq_usage(stdout, argv[0]);
448 free_opts(opts);
449 return true;
450 }
451
452 if (argc - optind > 1) {
453 fprintf(stderr, "Too many arguments.\n");
454 bam2fq_usage(stderr, argv[0]);
455 free_opts(opts);
456 return false;
457 }
458 opts->fn_input = argc > optind ? argv[optind] : "-";
459 *opts_out = opts;
460 return true;
461}
462
463void set_sam_opts(samFile *fp, bam2fq_state_t *state,
464 const bam2fq_opts_t *opts) {
465 if (state->has12)
466 hts_set_opt(fp, FASTQ_OPT_RNUM, 1);
467
468 if (state->illumina_tag)
469 hts_set_opt(fp, FASTQ_OPT_CASAVA, 1);
470
471 hts_set_opt(fp, FASTQ_OPT_BARCODE, opts->barcode_tag);
472
473 if (opts->extra_tags && (*opts->extra_tags == '*' || *opts->extra_tags == '\0'))
474 hts_set_opt(fp, FASTQ_OPT_AUX, NULL);
475 else {
476 kstring_t tag_list = {0,0};
477 if (state->copy_tags)
478 kputs("RG,BC,QT", &tag_list);
479 if (opts->extra_tags) {
480 if (tag_list.l)
481 kputc(',', &tag_list);
482 kputs(opts->extra_tags, &tag_list);
483 }
484 if (tag_list.l)
485 hts_set_opt(fp, FASTQ_OPT_AUX, tag_list.s);
486 ks_free(&tag_list);
487 }
488}
489
490// Open a file as normal or gzipped based on filename.
491// Note we always use bgzf and don't bother to attempt non-blocked
492// gzip streams. This is a departure from the old fastq code.
493static samFile *sam_open_z(char *fn, char *mode, bam2fq_state_t *state) {
494 char modez[6];
495 strcpy(modez, mode);
496
497 size_t l = strlen(fn);
498 if ((l > 3 && strcmp(fn+l-3, ".gz") == 0) ||
499 (l > 4 && strcmp(fn+l-4, ".bgz") == 0) ||
500 (l > 5 && strcmp(fn+l-5, ".bgzf") == 0)) {
501 char m[3] = {'z', state->compression_level+'0', '\0'};
502 strcat(modez, m);
503 }
504
505 samFile *fp = sam_open(fn, modez);
506 if (!fp)
507 return NULL;
508
509 if (state->p.pool)
510 hts_set_thread_pool(fp, &state->p);
511
512 return fp;
513}
514
515static bool init_state(const bam2fq_opts_t* opts, bam2fq_state_t** state_out)
516{
517 char *mode = opts->filetype == FASTA ? "wF" : "wf";
518
519 bam2fq_state_t* state = calloc(1, sizeof(bam2fq_state_t));
520 if (!state)
521 return false;
522 state->flag_on = opts->flag_on;
523 state->flag_off = opts->flag_off;
524 state->flag_alloff = opts->flag_alloff;
525 state->flag_anyon = opts->flag_anyon;
526 state->has12 = opts->has12;
527 state->use_oq = opts->use_oq;
528 state->illumina_tag = opts->illumina_tag;
529 state->copy_tags = opts->copy_tags;
530 state->filetype = opts->filetype;
531 state->def_qual = opts->def_qual;
532 state->index_sequence = NULL;
533 state->hstdout = NULL;
534 state->compression_level = opts->compression_level;
535
536 state->fp = sam_open_format(opts->fn_input, "r", &opts->ga.in);
537 if (state->fp == NULL) {
538 print_error_errno("bam2fq","Cannot read file \"%s\"", opts->fn_input);
539 free(state);
540 return false;
541 }
542
543 state->p.pool = NULL;
544 if (opts->ga.nthreads > 0) {
545 if (!(state->p.pool = hts_tpool_init(opts->ga.nthreads))) {
546 fprintf(stderr, "Failed to create thread pool\n");
547 free(state);
548 return false;
549 }
550 state->p.qsize = opts->ga.nthreads*2;
551 hts_set_thread_pool(state->fp, &state->p);
552 }
553
554 uint32_t rf = SAM_QNAME | SAM_FLAG | SAM_SEQ | SAM_QUAL;
555 if (opts->use_oq || opts->extra_tags || opts->index_file[0])
556 rf |= SAM_AUX;
557 if (opts->filter_tag) {
558 if (memcmp(opts->filter_tag, "NM", 2) == 0 ||
559 memcmp(opts->filter_tag, "MD", 2) == 0)
560 rf |= SAM_AUX | SAM_SEQ;
561 else if (memcmp(opts->filter_tag, "RG", 2) == 0)
562 rf |= SAM_RGAUX;
563 else
564 rf |= SAM_AUX;
565 }
566 if (hts_set_opt(state->fp, CRAM_OPT_REQUIRED_FIELDS, rf)) {
567 fprintf(stderr, "Failed to set CRAM_OPT_REQUIRED_FIELDS value\n");
568 free(state);
569 return false;
570 }
571 if (hts_set_opt(state->fp, CRAM_OPT_DECODE_MD, 0)) {
572 fprintf(stderr, "Failed to set CRAM_OPT_DECODE_MD value\n");
573 free(state);
574 return false;
575 }
576 if (opts->fnse) {
577 if (!(state->fpse = sam_open_z(opts->fnse, mode, state))) {
578 print_error_errno("bam2fq", "Cannot open singleton file \"%s\"", opts->fnse);
579 free(state);
580 return false;
581 }
582 set_sam_opts(state->fpse, state, opts);
583 }
584
585 if (opts->ga.reference) {
586 if (hts_set_fai_filename(state->fp, opts->ga.reference) != 0) {
587 print_error_errno("bam2fq", "cannot load reference \"%s\"", opts->ga.reference);
588 free(state);
589 return false;
590 }
591 }
592
593 // single, read1, read2
594 int i, j;
595 for (i = 0; i < 3; ++i) {
596 if (opts->fnr[i]) {
597 for (j = 0; j < i; j++)
598 if (opts->fnr[j] && strcmp(opts->fnr[j], opts->fnr[i]) == 0)
599 break;
600 if (j == i) {
601 if (!(state->fpr[i] = sam_open_z(opts->fnr[i], mode, state))) {
602 print_error_errno("bam2fq", "Cannot open r%d file \"%s\"",
603 i, opts->fnr[i]);
604 free(state);
605 return false;
606 }
607 set_sam_opts(state->fpr[i], state, opts);
608 } else {
609 state->fpr[i] = state->fpr[j];
610 }
611 } else {
612 if (!state->hstdout) {
613 if (!(state->hstdout = sam_open_z("-", mode, state))) {
614 print_error_errno("bam2fq", "Cannot open STDOUT");
615 free(state);
616 return false;
617 }
618 set_sam_opts(state->hstdout, state, opts);
619 autoflush_if_stdout(state->hstdout, "-");
620 }
621 state->fpr[i] = state->hstdout;
622 }
623 }
624
625 // index 1, index 2
626 for (i = 0; i < 2; i++) {
627 state->fpi[i] = NULL;
628 if (opts->index_file[i]) {
629 for (j = 0; j < 3; j++)
630 if (opts->fnr[j] && strcmp(opts->fnr[j], opts->index_file[i]) == 0)
631 break;
632 for (j -= 3; j >= 0 && j < i; j++)
633 if (opts->index_file[j] && strcmp(opts->index_file[j], opts->index_file[i]) == 0)
634 break;
635 if (i == j) {
636 if (!(state->fpi[i] = sam_open_z(opts->index_file[i], mode,
637 state))) {
638 print_error_errno("bam2fq", "Cannot open i%d file \"%s\"",
639 i+1, opts->index_file[i]);
640 free(state);
641 return false;
642 }
643 set_sam_opts(state->fpi[i], state, opts);
644 } else if (j < 0) {
645 state->fpi[i] = state->fpr[j+3];
646 } else {
647 state->fpi[i] = state->fpi[j];
648 }
649 }
650 }
651
652 state->h = sam_hdr_read(state->fp);
653 if (state->h == NULL) {
654 fprintf(stderr, "Failed to read header for \"%s\"\n", opts->fn_input);
655 free(state);
656 return false;
657 }
658
659 *state_out = state;
660 return true;
661}
662
663static bool destroy_state(const bam2fq_opts_t *opts, bam2fq_state_t *state, int* status)
664{
665 bool valid = true;
666 sam_hdr_destroy(state->h);
667 check_sam_close("bam2fq", state->fp, opts->fn_input, "file", status);
668 if (state->fpse && sam_close(state->fpse) < 0) {
669 print_error_errno("bam2fq", "Error closing singleton file \"%s\"", opts->fnse);
670 valid = false;
671 }
672
673 int i, j;
674 for (i = 0; i < 3; ++i) {
675 if (state->fpr[i] != state->hstdout) {
676 for (j = 0; j < i; j++)
677 if (state->fpr[i] == state->fpr[j])
678 break;
679 if (j == i && sam_close(state->fpr[i])) {
680 print_error_errno("bam2fq", "Error closing r%d file \"%s\"", i, opts->fnr[i]);
681 valid = false;
682 }
683 }
684 }
685 if (state->hstdout) {
686 release_autoflush(state->hstdout);
687 if (sam_close(state->hstdout) < 0) {
688 print_error_errno("bam2fq", "Error closing STDOUT");
689 valid = false;
690 }
691 }
692 for (i = 0; i < 2; i++) {
693 for (j = 0; j < 3; j++)
694 if (state->fpi[i] == state->fpr[j])
695 break;
696 for (j -= 3; j >= 0 && j < i; j++)
697 if (state->fpi[i] == state->fpi[j])
698 break;
699 if (j == i && state->fpi[i] && sam_close(state->fpi[i]) < 0) {
700 print_error_errno("bam2fq", "Error closing i%d file \"%s\"", i+1, opts->index_file[i]);
701 valid = false;
702 }
703 }
704 free(state->index_sequence);
705 if (state->p.pool)
706 hts_tpool_destroy(state->p.pool);
707 free(state);
708 return valid;
709}
710
711static inline bool filter_it_out(const bam1_t *b, const bam2fq_state_t *state,
712 bam2fq_opts_t *opts)
713{
714 if (opts->filter_tag) {
715 uint8_t *s = bam_aux_get(b, opts->filter_tag);
716 if (!s)
717 return true;
718
719 if (opts->filter_tag_vals) {
720 char t[32], *val = t;
721 switch (*s) {
722 case 'i': case 'I':
723 case 's': case 'S':
724 case 'c': case 'C':
725 if (snprintf(t, 32, "%"PRId64, bam_aux2i(s)) <= 0)
726 return true;
727 break;
728
729 case 'f':
730 // Comparing floats is hard.
731 // Eg (double)0.1 - (double)0.1f is -1.5e-9.
732 // Given BAM binary encoding is float however, just keep it.
733 // This means rounding errors will (hopefully) always be the
734 // same and basic equality still works.
735 if (snprintf(t, 32, "%f", (float)bam_aux2f(s)) <= 0)
736 return true;
737 break;
738
739 case 'A':
740 t[0] = s[1];
741 t[1] = 0;
742 break;
743
744 case 'Z': case 'H':
745 val = (char *)s+1;
746 break;
747
748 default:
749 // Anything unsupported fails the filter match too.
750 return true;
751 }
752
753 khint_t k = kh_get(str, opts->filter_tag_vals, val);
754 if (k == kh_end(opts->filter_tag_vals))
755 return 1; // tag value not found
756 }
757 }
758
759 return ((b->core.flag&(state->flag_on)) != state->flag_on // or reads indicated by filter flags
760 || (b->core.flag&(state->flag_off)) != 0
761 || (((b->core.flag&(state->flag_anyon)) == 0) && (state->flag_anyon != 0))
762 || (b->core.flag&(state->flag_alloff) && (b->core.flag&(state->flag_alloff)) == state->flag_alloff));
763
764}
765
766int write_index_rec(samFile *fp, bam1_t *b, bam2fq_state_t *state,
767 bam2fq_opts_t* opts, char *seq, int seq_len,
768 char *qual, int qual_len) {
769 if (!fp || !b || !seq_len)
770 return 0;
771
772 int ret = -1;
773 bam1_t *b2 = bam_init1(); // FIXME: reuse
774 if (!b2)
775 return -1;
776
777 size_t aux_len = b->data + b->l_data - bam_get_aux(b);
778 if (bam_set1(b2, b->core.l_qname, bam_get_qname(b),
779 (b->core.flag | BAM_FUNMAP) & ~BAM_FREVERSE,
780 -1, -1, 0, // refid, pos, mapq
781 0, NULL, // cigar
782 -1, -1, 0, // rnext, pnext, tlen
783 seq_len, seq, qual,
784 aux_len) < 0)
785 goto err;
786
787 uint8_t *q = bam_get_qual(b2);
788 if (qual) {
789 int i;
790 for (i = 0; i < seq_len; i++)
791 q[i] -= '!';
792 } else {
793 memset(q, opts->def_qual, seq_len);
794 }
795
796 memcpy(bam_get_aux(b2), bam_get_aux(b), aux_len);
797 b2->l_data += aux_len;
798 if (sam_write1(fp, state->h, b2) < 0)
799 goto err;
800
801 ret = 0;
802 err:
803 if (b2)
804 bam_destroy1(b2);
805 return ret;
806}
807
808int output_index(bam1_t *b1, bam1_t *b2, bam2fq_state_t *state,
809 bam2fq_opts_t* opts) {
810 bam1_t *b = b1 ? b1 : b2;
811
812 char *ifmt = opts->index_format;
813 if (!ifmt)
814 ifmt = "i*i*";
815
816 // Get seq / qual elements
817 char *bc = NULL, *qt = NULL;
818 if (b1)
819 bc = (char *)bam_aux_get(b1, opts->barcode_tag);
820 if (b2 && !bc)
821 bc = (char *)bam_aux_get(b2, opts->barcode_tag);
822 if (!bc)
823 return 0;
824 else
825 bc++; // skip Z
826
827 if (b1)
828 qt = (char *)bam_aux_get(b1, opts->quality_tag);
829 if (b2 && !qt)
830 qt = (char *)bam_aux_get(b2, opts->quality_tag);
831 if (qt && strlen(bc) != strlen(qt)-1)
832 qt = NULL;
833 else if (qt)
834 qt++;
835
836 int inum = 0;
837 while (inum < 2) {
838 char fc = *ifmt++;
839 if (!fc)
840 break; // ran out of index-format
841
842 long len, rem = 0;
843 if (isdigit(*ifmt)) {
844 rem = len = strtol(ifmt, &ifmt, 10);
845 } else {
846 ifmt++;
847 len = 0;
848 }
849
850 char *bc_end = bc, *qt_end = qt;
851 if (qt) {
852 while (len ? *bc_end && rem-- : isalpha(*bc_end))
853 bc_end++, qt_end++;
854 } else {
855 while (len ? *bc_end && rem-- : isalpha(*bc_end))
856 bc_end++;
857 }
858
859 switch (fc) {
860 case 'n':
861 // skip
862 bc = bc_end + (len==0);
863 if (qt)
864 qt = qt_end + (len==0);
865 break;
866
867 case 'i':
868 if (write_index_rec(state->fpi[inum], b, state, opts,
869 bc, bc_end-bc, qt, qt_end-qt) < 0)
870 return -1;
871 bc = bc_end + (len==0);
872 if (qt)
873 qt = qt_end + (len==0);
874 inum++;
875 break;
876
877 default:
878 fprintf(stderr, "Unknown index-format code\n");
879 return -1;
880 }
881 }
882
883 return 0;
884}
885
886static int flush_rec(bam2fq_state_t *state, bam2fq_opts_t* opts,
887 bam1_t *b[4], int score[3], int best[3],
888 int64_t *n_singletons) {
889 // Paired data, with 1 or 2 ends present.
890 if (score[1] > 0 && score[2] > 0) {
891 // If CASAVA tag is required and barcode is only on R1,
892 // copy it to R2
893 if (state->illumina_tag) {
894 char *tag;
895 if ((tag = (char *)bam_aux_get(b[best[1]],
896 opts->barcode_tag)))
897 if (bam_aux_update_str(b[best[2]],
898 opts->barcode_tag,
899 strlen(tag), tag+1) < 0)
900 goto err;
901 if ((tag = (char *)bam_aux_get(b[best[1]],
902 opts->quality_tag)))
903 if (bam_aux_update_str(b[best[2]],
904 opts->quality_tag,
905 strlen(tag), tag+1) < 0)
906 goto err;
907
908 }
909 if (sam_write1(state->fpr[1], state->h, b[best[1]]) < 0)
910 goto err;
911 if (sam_write1(state->fpr[2], state->h, b[best[2]]) < 0)
912 goto err;
913
914 if (output_index(b[best[1]], b[best[2]], state, opts) < 0)
915 goto err;
916 } else if (score[1] > 0 || score[2] > 0) {
917 if (state->fpse) {
918 // print whichever one exists to fpse
919 if (score[1] > 0) {
920 if (sam_write1(state->fpse, state->h, b[best[1]]) < 0)
921 goto err;
922 } else {
923 if (sam_write1(state->fpse, state->h, b[best[2]]) < 0)
924 goto err;
925 }
926 ++(*n_singletons);
927 } else {
928 if (score[1] > 0) {
929 if (sam_write1(state->fpr[1], state->h, b[best[1]]) < 0)
930 goto err;
931 } else {
932 if (sam_write1(state->fpr[2], state->h, b[best[2]]) < 0)
933 goto err;
934 }
935 }
936
937 if (output_index(score[1] > 0 ? b[best[1]] : NULL,
938 score[2] > 0 ? b[best[2]] : NULL,
939 state, opts) < 0)
940 goto err;
941 }
942
943 if (score[0]) { // single ended data (neither READ1 nor READ2)
944 if (sam_write1(state->fpr[0], state->h, b[best[0]]) < 0)
945 goto err;
946
947 if (output_index(b[best[0]], NULL, state, opts) < 0)
948 goto err;
949 }
950
951 return 0;
952
953 err:
954 return -1;
955}
956
957static bool bam2fq_mainloop(bam2fq_state_t *state, bam2fq_opts_t* opts)
958{
959 int n;
960 char *current_qname = NULL;
961 int64_t n_reads = 0, n_singletons = 0; // Statistics
962 int score[3];
963 int at_eof;
964 bool valid = false;
965 int best[3] = {-1, -1, -1}; // map R0, R1, single to b[] indices;
966 // indexed by [readpart]
967 bam1_t *b[4]; // 3 readparts, plus current record
968
969 for (n = 0; n < 4; n++) {
970 if (!(b[n] = bam_init1())) {
971 perror("[bam2fq_mainloop] Malloc error for bam record buffer.");
972 return false;
973 }
974 }
975
976 n = 0;
977 while (true) {
978 int res = sam_read1(state->fp, state->h, b[n]);
979 if (res < -1) {
980 print_error("bam2fq", "Failed to read bam record");
981 goto err;
982 }
983 at_eof = res < 0;
984
985 if (!at_eof && filter_it_out(b[n], state, opts))
986 continue;
987 if (!at_eof) {
988 ++n_reads;
989
990 // Handle -O option: use OQ for qual
991 uint8_t *oq;
992 if (state->use_oq && (oq = bam_aux_get(b[n],"OQ")) && *oq == 'Z') {
993 int i, l = strlen((char *)++oq);
994 uint8_t *qual = bam_get_qual(b[n]);
995 for (i = 0; i < l && i < b[n]->core.l_qseq; i++)
996 qual[i] = oq[i] - '!';
997 }
998 }
999
1000 if (at_eof
1001 || !current_qname
1002 || (strcmp(current_qname, bam_get_qname(b[n])) != 0)) {
1003 // New name, so flush best examples of previous name.
1004 if (current_qname)
1005 if (flush_rec(state, opts, b, score, best, &n_singletons) < 0)
1006 goto err;
1007
1008 current_qname = bam_get_qname(b[n]);
1009 score[0] = score[1] = score[2] = 0;
1010
1011 if (at_eof) { break; }
1012 }
1013
1014 // Prefer a copy of the read that has base qualities
1015 int b_score = bam_get_qual(b[n])[0] != 0xff? 2 : 1;
1016 readpart rp = which_readpart(b[n]);
1017 if (score[rp] < b_score) {
1018 score[rp] = b_score;
1019 // Record b[n] slot for best copy of readpair and find a new
1020 // slot for next bam read
1021 best[rp] = n;
1022 int used_slot[4] = {0}, i;
1023 for (i = 0; i < 3; i++)
1024 if (best[i] >= 0)
1025 used_slot[best[i]] = 1;
1026 for (i = 0; i < 4 && used_slot[i]; i++)
1027 ;
1028 n = i;
1029 }
1030 }
1031
1032 valid = true;
1033 err:
1034 if (!valid)
1035 print_error_errno("bam2fq", "Error writing to FASTx files.");
1036
1037 for (n = 0; n < 4; n++)
1038 bam_destroy1(b[n]);
1039
1040 fprintf(stderr, "[M::%s] discarded %" PRId64 " singletons\n",
1041 __func__, n_singletons);
1042 fprintf(stderr, "[M::%s] processed %" PRId64 " reads\n",
1043 __func__, n_reads);
1044
1045 return valid;
1046}
1047
1048int main_bam2fq(int argc, char *argv[])
1049{
1050 int status = EXIT_FAILURE;
1051 bam2fq_opts_t* opts = NULL;
1052 bam2fq_state_t* state = NULL;
1053
1054 bool valid = parse_opts(argc, argv, &opts);
1055 if (!valid || opts == NULL) return valid ? EXIT_SUCCESS : EXIT_FAILURE;
1056
1057 if (!init_state(opts, &state)) goto err;
1058
1059 if (!bam2fq_mainloop(state,opts)) goto err;
1060
1061 if (!destroy_state(opts, state, &status)) goto err;
1062
1063 status = EXIT_SUCCESS;
1064 err:
1065 sam_global_args_free(&opts->ga);
1066 free_opts(opts);
1067
1068 return status;
1069}