upstream-dependencies/samtools/stats.c
1/* stats.c -- This is the former bamcheck integrated into samtools/htslib.
2
3 Copyright (C) 2012-2024 Genome Research Ltd.
4
5 Author: Petr Danecek <pd3@sanger.ac.uk>
6 Author: Sam Nicholls <sam@samnicholls.net>
7
8Permission is hereby granted, free of charge, to any person obtaining a copy
9of this software and associated documentation files (the "Software"), to deal
10in the Software without restriction, including without limitation the rights
11to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12copies of the Software, and to permit persons to whom the Software is
13furnished to do so, subject to the following conditions:
14
15The above copyright notice and this permission notice shall be included in
16all copies or substantial portions of the Software.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE. */
25
26/* Assumptions, approximations and other issues:
27 - GC-depth graph does not split reads, the starting position determines which bin is incremented.
28 There are small overlaps between bins (max readlen-1). However, the bins are big (20k).
29 - coverage distribution ignores softclips and deletions
30 - some stats require sorted BAMs
31 - GC content graph can have an untidy, step-like pattern when BAM contains multiple read lengths.
32 - 'bases mapped' (stats->nbases_mapped) is calculated from read lengths given by BAM (core.l_qseq)
33 - With the -t option, the whole reads are used. Except for the number of mapped bases (cigar)
34 counts, no splicing is done, no indels or soft clips are considered, even small overlap is
35 good enough to include the read in the stats.
36 - GC content of reads not calculated for "=" sequences
37
38*/
39
40#include <config.h>
41
42#include <unistd.h> // for isatty()
43#include <stdio.h>
44#include <stdlib.h>
45#include <stdarg.h>
46#include <string.h>
47#include <math.h>
48#include <ctype.h>
49#include <inttypes.h>
50#include <getopt.h>
51#include <errno.h>
52#include <assert.h>
53#include <zlib.h> // for crc32
54#include <htslib/faidx.h>
55#include <htslib/sam.h>
56#include <htslib/hts.h>
57#include <htslib/hts_defs.h>
58#include "samtools.h"
59#include <htslib/khash.h>
60#include <htslib/kstring.h>
61#include "stats_isize.h"
62#include "sam_opts.h"
63#include "bedidx.h"
64
65#define BWA_MIN_RDLEN 35
66#define DEFAULT_CHUNK_NO 8
67#define DEFAULT_PAIR_MAX 10000
68#define ERROR_LIMIT 200
69// From the spec
70// If 0x4 is set, no assumptions can be made about RNAME, POS, CIGAR, MAPQ, bits 0x2, 0x10, 0x100 and 0x800, and the bit 0x20 of the previous read in the template.
71#define IS_PAIRED(bam) ((bam)->core.flag&BAM_FPAIRED)
72#define IS_PAIRED_AND_MAPPED(bam) (((bam)->core.flag&BAM_FPAIRED) && !((bam)->core.flag&BAM_FUNMAP) && !((bam)->core.flag&BAM_FMUNMAP))
73#define IS_PROPERLYPAIRED(bam) (((bam)->core.flag&(BAM_FPAIRED|BAM_FPROPER_PAIR)) == (BAM_FPAIRED|BAM_FPROPER_PAIR) && !((bam)->core.flag&BAM_FUNMAP))
74#define IS_UNMAPPED(bam) ((bam)->core.flag&BAM_FUNMAP)
75#define IS_REVERSE(bam) ((bam)->core.flag&BAM_FREVERSE)
76#define IS_MATE_REVERSE(bam) ((bam)->core.flag&BAM_FMREVERSE)
77#define IS_READ1(bam) ((bam)->core.flag&BAM_FREAD1)
78#define IS_READ2(bam) ((bam)->core.flag&BAM_FREAD2)
79#define IS_DUP(bam) ((bam)->core.flag&BAM_FDUP)
80#define IS_ORIGINAL(bam) (((bam)->core.flag&(BAM_FSECONDARY|BAM_FSUPPLEMENTARY)) == 0)
81
82#define READ_ORDER_NONE 0
83#define READ_ORDER_FIRST 1
84#define READ_ORDER_LAST 2
85#define READ_ORDER_MIDDLE 3
86
87#define REG_INC 100
88#define POS_INC 1000
89
90// The GC-depth graph works as follows: split the reference sequence into
91// segments and calculate GC content and depth in each bin. Then sort
92// these segments by their GC and plot the depth distribution by means
93// of 10th, 25th, etc. depth percentiles.
94typedef struct
95{
96 float gc;
97 uint32_t depth;
98}
99gc_depth_t;
100
101// For coverage distribution, a simple pileup
102typedef struct
103{
104 hts_pos_t pos;
105 int size, start;
106 int *buffer;
107}
108round_buffer_t;
109
110typedef struct
111{
112 int npos, mpos, cpos;
113 hts_pair_pos_t *pos;
114}
115regions_t;
116
117typedef struct
118{
119 uint64_t a;
120 uint64_t c;
121 uint64_t g;
122 uint64_t t;
123 uint64_t n;
124 uint64_t other;
125}
126acgtno_count_t;
127
128typedef struct
129{
130 char tag_name[3];
131 char qual_name[3];
132 uint32_t nbases;
133 int32_t tag_sep; // Index of the separator (if present)
134 int32_t max_qual;
135 uint32_t offset; // Where the tag stats info is located in the allocated memory
136}
137barcode_info_t;
138
139typedef struct
140{
141 // Auxiliary data
142 int flag_require, flag_filter;
143 faidx_t *fai; // Reference sequence for GC-depth graph
144 int argc; // Command line arguments to be printed on the output
145 char **argv;
146 int gcd_bin_size; // The size of GC-depth bin
147 int nisize; // The maximum insert size that the allocated array can hold - 0 indicates no limit
148 int trim_qual; // bwa trim quality
149 float isize_main_bulk; // There are always some unrealistically big insert sizes, report only the main part
150 int cov_min,cov_max,cov_step; // Minimum, maximum coverage and size of the coverage bins
151 samFile* sam;
152 sam_hdr_t* sam_header;
153
154 // Filters
155 int filter_readlen;
156
157 // Misc
158 char *split_tag; // Tag on which to perform stats splitting
159 char *split_prefix; // Path or string prefix for filenames created when splitting
160 int remove_overlaps;
161 int cov_threshold;
162}
163stats_info_t;
164
165typedef struct
166{
167 // Dimensions of the quality histogram holder (quals_1st,quals_2nd), GC content holder (gc_1st,gc_2nd),
168 // insert size histogram holder
169 int nquals; // The number of quality bins
170 int nbases; // The maximum sequence length the allocated array can hold
171 int ngc; // The size of gc_1st and gc_2nd
172 int nindels; // The maximum indel length for indel distribution
173
174 // Arrays for the histogram data
175 uint64_t *quals_1st, *quals_2nd;
176 uint64_t *gc_1st, *gc_2nd;
177 acgtno_count_t *acgtno_cycles_1st, *acgtno_cycles_2nd;
178 acgtno_count_t *acgtno_revcomp;
179 uint64_t *read_lengths, *read_lengths_1st, *read_lengths_2nd;
180 uint64_t *insertions, *deletions;
181 uint64_t *ins_cycles_1st, *ins_cycles_2nd, *del_cycles_1st, *del_cycles_2nd;
182 isize_t *isize;
183 uint64_t* mapping_qualities;
184
185 // The extremes encountered
186 int max_len; // Maximum read length
187 int max_len_1st; // Maximum read length for forward reads
188 int max_len_2nd; // Maximum read length for reverse reads
189 int max_qual; // Maximum quality
190 int is_sorted;
191
192 // Summary numbers
193 uint64_t total_len;
194 uint64_t total_len_1st;
195 uint64_t total_len_2nd;
196 uint64_t total_len_dup;
197 uint64_t nreads_1st;
198 uint64_t nreads_2nd;
199 uint64_t nreads_other;
200 uint64_t nreads_filtered;
201 uint64_t nreads_dup;
202 uint64_t nreads_unmapped;
203 uint64_t nreads_single_mapped;
204 uint64_t nreads_paired_and_mapped;
205 uint64_t nreads_properly_paired;
206 uint64_t nreads_paired_tech;
207 uint64_t nreads_anomalous;
208 uint64_t nreads_mq0;
209 uint64_t nbases_mapped;
210 uint64_t nbases_mapped_cigar;
211 uint64_t nbases_trimmed; // bwa trimmed bases
212 uint64_t nmismatches;
213 uint64_t nreads_QCfailed, nreads_secondary, nreads_supplementary;
214 struct {
215 uint32_t names, reads, quals;
216 } checksum;
217
218 // GC-depth related data
219 uint32_t ngcd, igcd; // The maximum number of GC depth bins and index of the current bin
220 gc_depth_t *gcd; // The GC-depth bins holder
221 int32_t tid; // Position of the current bin
222 hts_pos_t gcd_pos, pos; // Position of the last read
223
224 // Coverage distribution related data
225 int ncov; // The number of coverage bins
226 uint64_t *cov; // The coverage frequencies
227 round_buffer_t cov_rbuf; // Pileup round buffer
228
229 // Mismatches by read cycle
230 uint8_t *rseq_buf; // A buffer for reference sequence to check the mismatches against
231 int mrseq_buf; // The size of the buffer
232 hts_pos_t rseq_pos; // The coordinate of the first base in the buffer
233 int64_t nrseq_buf; // The used part of the buffer
234 uint64_t *mpc_buf; // Mismatches per cycle
235
236 // Target regions
237 int nregions;
238 hts_pos_t reg_from, reg_to;
239 regions_t *regions;
240
241 // Auxiliary data
242 double sum_qual; // For calculating average quality value
243 void *rg_hash; // Read groups to include, the array is null-terminated
244
245 // Split
246 char* split_name;
247
248 stats_info_t* info; // Pointer to options and settings struct
249 hts_pair_pos_t *chunks;
250 uint32_t nchunks;
251
252 uint32_t pair_count; // Number of active pairs in the pairing hash table
253 uint64_t target_count; // Number of bases covered by the target file
254 uint32_t last_pair_tid;
255 uint32_t last_read_flush;
256
257 // Barcode statistics
258 acgtno_count_t *acgtno_barcode;
259 uint64_t *quals_barcode;
260 barcode_info_t *tags_barcode;
261 uint32_t ntags;
262 uint32_t error_number;
263}
264stats_t;
265KHASH_MAP_INIT_STR(c2stats, stats_t*)
266
267typedef struct {
268 uint32_t first; // 1 - first read, 2 - second read
269 uint32_t n, m; // number of chunks, allocated chunks
270 hts_pair_pos_t *chunks; // chunk array of size m
271} pair_t;
272KHASH_MAP_INIT_STR(qn2pair, pair_t*)
273
274KHASH_SET_INIT_STR(rg)
275
276
277static void HTS_NORETURN error(const char *format, ...);
278int is_in_regions(bam1_t *bam_line, stats_t *stats);
279void realloc_buffers(stats_t *stats, int seq_len);
280
281static int regions_lt(const void *r1, const void *r2) {
282 int64_t from_diff = ((hts_pair_pos_t *)r1)->beg - ((hts_pair_pos_t *)r2)->beg;
283 int64_t to_diff = ((hts_pair_pos_t *)r1)->end - ((hts_pair_pos_t *)r2)->end;
284
285 return from_diff > 0 ? 1 : from_diff < 0 ? -1 : to_diff > 0 ? 1 : to_diff < 0 ? -1 : 0;
286}
287
288// Coverage distribution methods
289static inline int coverage_idx(int min, int max, int n, int step, int depth)
290{
291 if ( depth < min )
292 return 0;
293
294 if ( depth > max )
295 return n-1;
296
297 return 1 + (depth - min) / step;
298}
299
300static inline int round_buffer_lidx2ridx(int offset, int size, hts_pos_t refpos, hts_pos_t pos)
301{
302 return (offset + (pos-refpos) % size) % size;
303}
304
305void round_buffer_flush(stats_t *stats, hts_pos_t pos)
306{
307 int ibuf,idp;
308
309 if ( pos==stats->cov_rbuf.pos )
310 return;
311
312 hts_pos_t new_pos = pos;
313 if ( pos==-1 || pos - stats->cov_rbuf.pos >= stats->cov_rbuf.size )
314 {
315 // Flush the whole buffer, but in sequential order,
316 pos = stats->cov_rbuf.pos + stats->cov_rbuf.size - 1;
317 }
318
319 if ( pos < stats->cov_rbuf.pos )
320 error("Expected coordinates in ascending order, got %"PRIhts_pos" after %"PRIhts_pos"\n", pos, stats->cov_rbuf.pos);
321
322 int ifrom = stats->cov_rbuf.start;
323 int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos-1);
324 if ( ifrom>ito )
325 {
326 for (ibuf=ifrom; ibuf<stats->cov_rbuf.size; ibuf++)
327 {
328 if ( !stats->cov_rbuf.buffer[ibuf] )
329 continue;
330 idp = coverage_idx(stats->info->cov_min,stats->info->cov_max,stats->ncov,stats->info->cov_step,stats->cov_rbuf.buffer[ibuf]);
331 stats->cov[idp]++;
332 stats->cov_rbuf.buffer[ibuf] = 0;
333 }
334 ifrom = 0;
335 }
336 for (ibuf=ifrom; ibuf<=ito; ibuf++)
337 {
338 if ( !stats->cov_rbuf.buffer[ibuf] )
339 continue;
340 idp = coverage_idx(stats->info->cov_min,stats->info->cov_max,stats->ncov,stats->info->cov_step,stats->cov_rbuf.buffer[ibuf]);
341 stats->cov[idp]++;
342 stats->cov_rbuf.buffer[ibuf] = 0;
343 }
344 stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos);
345 stats->cov_rbuf.pos = new_pos;
346}
347
348/**
349 * [from, to) - 0 based half-open
350 */
351static void round_buffer_insert_read(round_buffer_t *rbuf, hts_pos_t from, hts_pos_t to)
352{
353 if ( to-from > rbuf->size )
354 error("The read length too big (%"PRIhts_pos"), please increase the buffer length (currently %d)\n", to-from, rbuf->size);
355 if ( from < rbuf->pos )
356 error("The reads are not sorted (%"PRIhts_pos" comes after %"PRIhts_pos").\n", from, rbuf->pos);
357
358 int ifrom, ito, ibuf;
359 ifrom = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, from);
360 ito = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, to);
361 if ( ifrom>ito )
362 {
363 for (ibuf=ifrom; ibuf<rbuf->size; ibuf++)
364 rbuf->buffer[ibuf]++;
365 ifrom = 0;
366 }
367 for (ibuf=ifrom; ibuf<ito; ibuf++)
368 rbuf->buffer[ibuf]++;
369}
370
371// Calculate the number of bases in the read trimmed by BWA
372int bwa_trim_read(int trim_qual, uint8_t *quals, int len, int reverse)
373{
374 if ( len<BWA_MIN_RDLEN ) return 0;
375
376 // Although the name implies that the read cannot be trimmed to more than BWA_MIN_RDLEN,
377 // the calculation can in fact trim it to (BWA_MIN_RDLEN-1). (bwa_trim_read in bwa/bwaseqio.c).
378 int max_trimmed = len - BWA_MIN_RDLEN + 1;
379 int l, sum=0, max_sum=0, max_l=0;
380
381 for (l=0; l<max_trimmed; l++)
382 {
383 sum += trim_qual - quals[ reverse ? l : len-1-l ];
384 if ( sum<0 ) break;
385 if ( sum>max_sum )
386 {
387 max_sum = sum;
388 // This is the correct way, but bwa clips from some reason one base less
389 // max_l = l+1;
390 max_l = l;
391 }
392 }
393 return max_l;
394}
395
396
397void count_indels(stats_t *stats,bam1_t *bam_line)
398{
399 int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
400 uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
401 int icig;
402 int icycle = 0;
403 int read_len = bam_line->core.l_qseq;
404 for (icig=0; icig<bam_line->core.n_cigar; icig++)
405 {
406 int cig = bam_cigar_op(bam_get_cigar(bam_line)[icig]);
407 int ncig = bam_cigar_oplen(bam_get_cigar(bam_line)[icig]);
408 if ( !ncig ) continue; // curiously, this can happen: 0D
409
410 if ( cig==BAM_CINS )
411 {
412 int idx = is_fwd ? icycle : read_len-icycle-ncig;
413 if ( idx<0 )
414 error("FIXME: read_len=%d vs icycle=%d\n", read_len,icycle);
415 if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%"PRIhts_pos" %s\n", idx, stats->nbases, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
416 if ( order == READ_ORDER_FIRST )
417 stats->ins_cycles_1st[idx]++;
418 if ( order == READ_ORDER_LAST )
419 stats->ins_cycles_2nd[idx]++;
420 icycle += ncig;
421 if ( ncig<=stats->nindels )
422 stats->insertions[ncig-1]++;
423 continue;
424 }
425 if ( cig==BAM_CDEL )
426 {
427 int idx = is_fwd ? icycle-1 : read_len-icycle-1;
428 if ( idx<0 ) continue; // discard meaningless deletions
429 if ( idx >= stats->nbases ) error("FIXME: %d vs %d\n", idx,stats->nbases);
430 if ( order == READ_ORDER_FIRST )
431 stats->del_cycles_1st[idx]++;
432 if ( order == READ_ORDER_LAST )
433 stats->del_cycles_2nd[idx]++;
434 if ( ncig<=stats->nindels )
435 stats->deletions[ncig-1]++;
436 continue;
437 }
438 if ( cig!=BAM_CREF_SKIP && cig!=BAM_CHARD_CLIP && cig!=BAM_CPAD )
439 icycle += ncig;
440 }
441}
442
443int unclipped_length(bam1_t *bam_line)
444{
445 int icig, read_len = bam_line->core.l_qseq;
446 for (icig=0; icig<bam_line->core.n_cigar; icig++)
447 {
448 int cig = bam_cigar_op(bam_get_cigar(bam_line)[icig]);
449 if ( cig==BAM_CHARD_CLIP )
450 read_len += bam_cigar_oplen(bam_get_cigar(bam_line)[icig]);
451 }
452 return read_len;
453}
454
455void count_mismatches_per_cycle(stats_t *stats, bam1_t *bam_line, int read_len)
456{
457 int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
458 int icig, iread=0, icycle=0;
459 hts_pos_t iref = bam_line->core.pos - stats->rseq_pos;
460 uint8_t *read = bam_get_seq(bam_line);
461 uint8_t *quals = bam_get_qual(bam_line);
462 uint64_t *mpc_buf = stats->mpc_buf;
463 for (icig=0; icig<bam_line->core.n_cigar; icig++)
464 {
465 int cig = bam_cigar_op(bam_get_cigar(bam_line)[icig]);
466 int ncig = bam_cigar_oplen(bam_get_cigar(bam_line)[icig]);
467 if ( cig==BAM_CINS )
468 {
469 iread += ncig;
470 icycle += ncig;
471 continue;
472 }
473 if ( cig==BAM_CDEL )
474 {
475 iref += ncig;
476 continue;
477 }
478 if ( cig==BAM_CSOFT_CLIP )
479 {
480 icycle += ncig;
481 // Soft-clips are present in the sequence, but the position of the read marks a start of the sequence after clipping
482 // iref += ncig;
483 iread += ncig;
484 continue;
485 }
486 if ( cig==BAM_CHARD_CLIP )
487 {
488 icycle += ncig;
489 continue;
490 }
491 // Ignore H and N CIGARs. The letter are inserted e.g. by TopHat and often require very large
492 // chunk of refseq in memory. Not very frequent and not noticeable in the stats.
493 if ( cig==BAM_CREF_SKIP || cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) continue;
494 if ( cig!=BAM_CMATCH && cig!=BAM_CEQUAL && cig!=BAM_CDIFF ) // not relying on precalculated diffs
495 error("TODO: cigar %d, %s:%"PRIhts_pos" %s\n", cig, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
496
497 if ( ncig+iref > stats->nrseq_buf )
498 error("FIXME: %d+%"PRIhts_pos" > %"PRId64", %s, %s:%"PRIhts_pos"\n", ncig, iref, stats->nrseq_buf, bam_get_qname(bam_line), sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1);
499
500 int im;
501 for (im=0; im<ncig; im++)
502 {
503 uint8_t cread = bam_seqi(read,iread);
504 uint8_t cref = stats->rseq_buf[iref];
505
506 // ---------------15
507 // =ACMGRSVTWYHKDBN
508 if ( cread==15 )
509 {
510 int idx = is_fwd ? icycle : read_len-icycle-1;
511 if ( idx>stats->max_len )
512 error("mpc: %d>%d\n",idx,stats->max_len);
513 idx = idx*stats->nquals;
514 if ( idx>=stats->nquals*stats->nbases )
515 error("FIXME: mpc_buf overflow\n");
516 mpc_buf[idx]++;
517 }
518 else if ( cref && cread && cref!=cread )
519 {
520 uint8_t qual = quals[iread] + 1;
521 if ( qual>=stats->nquals )
522 error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
523
524 int idx = is_fwd ? icycle : read_len-icycle-1;
525 if ( idx>stats->max_len )
526 error("mpc: %d>%d (%s %"PRIhts_pos" %s)\n", idx, stats->max_len, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
527
528 idx = idx*stats->nquals + qual;
529 if ( idx>=stats->nquals*stats->nbases )
530 error("FIXME: mpc_buf overflow\n");
531 mpc_buf[idx]++;
532 }
533
534 iref++;
535 iread++;
536 icycle++;
537 }
538 }
539}
540
541void read_ref_seq(stats_t *stats, int32_t tid, hts_pos_t pos, hts_pos_t end)
542{
543 int i;
544 hts_pos_t fai_ref_len;
545 char *fai_ref;
546
547 if (end < pos+stats->mrseq_buf-1)
548 end = pos+stats->mrseq_buf-1;
549 else if (stats->mrseq_buf < end - pos) {
550 size_t sz = end - pos;
551 uint8_t *new_rseq = realloc(stats->rseq_buf, sz);
552 if (!new_rseq)
553 error("Couldn't expand the reference sequence buffer\n");
554 stats->rseq_buf = new_rseq;
555 stats->mrseq_buf = sz;
556 }
557
558 fai_ref = faidx_fetch_seq64(stats->info->fai, sam_hdr_tid2name(stats->info->sam_header, tid), pos, pos+stats->mrseq_buf-1, &fai_ref_len);
559 if ( fai_ref_len < 0 ) error("Failed to fetch the sequence \"%s\"\n", sam_hdr_tid2name(stats->info->sam_header, tid));
560
561 uint8_t *ptr = stats->rseq_buf;
562 for (i=0; i<fai_ref_len; i++)
563 {
564 // Conversion between uint8_t coding and ACGT
565 // -12-4---8-------
566 // =ACMGRSVTWYHKDBN
567 switch (fai_ref[i])
568 {
569 case 'A':
570 case 'a': *ptr = 1; break;
571 case 'C':
572 case 'c': *ptr = 2; break;
573 case 'G':
574 case 'g': *ptr = 4; break;
575 case 'T':
576 case 't': *ptr = 8; break;
577 default: *ptr = 0; break;
578 }
579 ptr++;
580 }
581 free(fai_ref);
582
583 if ( fai_ref_len < stats->mrseq_buf ) memset(ptr,0, stats->mrseq_buf - fai_ref_len);
584 stats->nrseq_buf = fai_ref_len;
585 stats->rseq_pos = pos;
586 stats->tid = tid;
587}
588
589float fai_gc_content(stats_t *stats, hts_pos_t pos, int len)
590{
591 uint32_t gc,count,c;
592 hts_pos_t i = pos - stats->rseq_pos, ito = i + len;
593 assert( i>=0 );
594
595 if ( ito > stats->nrseq_buf ) ito = stats->nrseq_buf;
596
597 // Count GC content
598 gc = count = 0;
599 for (; i<ito; i++)
600 {
601 c = stats->rseq_buf[i];
602 if ( c==2 || c==4 )
603 {
604 gc++;
605 count++;
606 }
607 else if ( c==1 || c==8 )
608 count++;
609 }
610 return count ? (float)gc/count : 0;
611}
612
613void realloc_rseq_buffer(stats_t *stats)
614{
615 int n = stats->nbases*10;
616 if ( stats->info->gcd_bin_size > n ) n = stats->info->gcd_bin_size;
617 if ( stats->mrseq_buf<n )
618 {
619 stats->rseq_buf = realloc(stats->rseq_buf,sizeof(uint8_t)*n);
620 if (!stats->rseq_buf) {
621 error("Could not reallocate reference sequence buffer");
622 }
623 stats->mrseq_buf = n;
624 }
625}
626
627void realloc_gcd_buffer(stats_t *stats, int seq_len)
628{
629 hts_expand0(gc_depth_t,stats->igcd+1,stats->ngcd,stats->gcd);
630 realloc_rseq_buffer(stats);
631}
632
633void realloc_buffers(stats_t *stats, int seq_len)
634{
635 int n = 2*(1 + seq_len - stats->nbases) + stats->nbases;
636
637 stats->quals_1st = realloc(stats->quals_1st, n*stats->nquals*sizeof(uint64_t));
638 if ( !stats->quals_1st )
639 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
640 memset(stats->quals_1st + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
641
642 stats->quals_2nd = realloc(stats->quals_2nd, n*stats->nquals*sizeof(uint64_t));
643 if ( !stats->quals_2nd )
644 error("Could not realloc buffers, the sequence too long: %d (2x%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
645 memset(stats->quals_2nd + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
646
647 if ( stats->mpc_buf )
648 {
649 stats->mpc_buf = realloc(stats->mpc_buf, n*stats->nquals*sizeof(uint64_t));
650 if ( !stats->mpc_buf )
651 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
652 memset(stats->mpc_buf + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
653 }
654
655 stats->acgtno_cycles_1st = realloc(stats->acgtno_cycles_1st, n*sizeof(acgtno_count_t));
656 if ( !stats->acgtno_cycles_1st )
657 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len, n*sizeof(acgtno_count_t));
658 memset(stats->acgtno_cycles_1st + stats->nbases, 0, (n-stats->nbases)*sizeof(acgtno_count_t));
659
660 stats->acgtno_cycles_2nd = realloc(stats->acgtno_cycles_2nd, n*sizeof(acgtno_count_t));
661 if ( !stats->acgtno_cycles_2nd )
662 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len, n*sizeof(acgtno_count_t));
663 memset(stats->acgtno_cycles_2nd + stats->nbases, 0, (n-stats->nbases)*sizeof(acgtno_count_t));
664
665 stats->acgtno_revcomp = realloc(stats->acgtno_revcomp, n*sizeof(acgtno_count_t));
666 if ( !stats->acgtno_revcomp )
667 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len, n*sizeof(acgtno_count_t));
668 memset(stats->acgtno_revcomp + stats->nbases, 0, (n-stats->nbases)*sizeof(acgtno_count_t));
669
670 stats->read_lengths = realloc(stats->read_lengths, n*sizeof(uint64_t));
671 if ( !stats->read_lengths )
672 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
673 memset(stats->read_lengths + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
674
675 stats->read_lengths_1st = realloc(stats->read_lengths_1st, n*sizeof(uint64_t));
676 if ( !stats->read_lengths_1st )
677 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
678 memset(stats->read_lengths_1st + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
679
680 stats->read_lengths_2nd = realloc(stats->read_lengths_2nd, n*sizeof(uint64_t));
681 if ( !stats->read_lengths_2nd )
682 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
683 memset(stats->read_lengths_2nd + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
684
685 stats->insertions = realloc(stats->insertions, n*sizeof(uint64_t));
686 if ( !stats->insertions )
687 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
688 memset(stats->insertions + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
689
690 stats->deletions = realloc(stats->deletions, n*sizeof(uint64_t));
691 if ( !stats->deletions )
692 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
693 memset(stats->deletions + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
694
695 stats->ins_cycles_1st = realloc(stats->ins_cycles_1st, (n+1)*sizeof(uint64_t));
696 if ( !stats->ins_cycles_1st )
697 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
698 memset(stats->ins_cycles_1st + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
699
700 stats->ins_cycles_2nd = realloc(stats->ins_cycles_2nd, (n+1)*sizeof(uint64_t));
701 if ( !stats->ins_cycles_2nd )
702 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
703 memset(stats->ins_cycles_2nd + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
704
705 stats->del_cycles_1st = realloc(stats->del_cycles_1st, (n+1)*sizeof(uint64_t));
706 if ( !stats->del_cycles_1st )
707 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
708 memset(stats->del_cycles_1st + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
709
710 stats->del_cycles_2nd = realloc(stats->del_cycles_2nd, (n+1)*sizeof(uint64_t));
711 if ( !stats->del_cycles_2nd )
712 error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
713 memset(stats->del_cycles_2nd + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
714
715 stats->nbases = n;
716
717 // Realloc the coverage distribution buffer
718 int *rbuffer = calloc(sizeof(int),seq_len*5);
719 if (!rbuffer) {
720 error("Could not allocate coverage distribution buffer");
721 }
722 n = stats->cov_rbuf.size-stats->cov_rbuf.start;
723 memcpy(rbuffer,stats->cov_rbuf.buffer+stats->cov_rbuf.start,n);
724 if ( stats->cov_rbuf.start>1 )
725 memcpy(rbuffer+n,stats->cov_rbuf.buffer,stats->cov_rbuf.start);
726 stats->cov_rbuf.start = 0;
727 free(stats->cov_rbuf.buffer);
728 stats->cov_rbuf.buffer = rbuffer;
729 stats->cov_rbuf.size = seq_len*5;
730
731 realloc_rseq_buffer(stats);
732}
733
734void update_checksum(bam1_t *bam_line, stats_t *stats)
735{
736 uint8_t *name = (uint8_t*) bam_get_qname(bam_line);
737 int len = 0;
738 while ( name[len] ) len++;
739 stats->checksum.names += crc32(0L, name, len);
740
741 int seq_len = bam_line->core.l_qseq;
742 if ( !seq_len ) return;
743
744 uint8_t *seq = bam_get_seq(bam_line);
745 stats->checksum.reads += crc32(0L, seq, (seq_len+1)/2);
746
747 uint8_t *qual = bam_get_qual(bam_line);
748 stats->checksum.quals += crc32(0L, qual, (seq_len+1)/2);
749}
750
751// Collect statistics about the barcode tags specified by init_barcode_tags method
752static void collect_barcode_stats(bam1_t* bam_line, stats_t* stats) {
753 uint32_t nbases, tag, i;
754 acgtno_count_t *acgtno;
755 uint64_t *quals;
756 int32_t *separator, *maxqual;
757
758 for (tag = 0; tag < stats->ntags; tag++) {
759 const char *barcode_tag = stats->tags_barcode[tag].tag_name, *qual_tag = stats->tags_barcode[tag].qual_name;
760 uint8_t* bc = bam_aux_get(bam_line, barcode_tag);
761 if (!bc)
762 continue;
763
764 char* barcode = bam_aux2Z(bc);
765 if (!barcode)
766 continue;
767
768 uint32_t barcode_len = strlen(barcode);
769 if (!barcode_len) {
770 continue; //consider 0 size barcode same as no barcode - avoids issues with realloc below
771 }
772 if (!stats->tags_barcode[tag].nbases) { // tag seen for the first time
773 uint32_t offset = 0;
774 for (i = 0; i < stats->ntags; i++)
775 offset += stats->tags_barcode[i].nbases;
776
777 stats->tags_barcode[tag].offset = offset;
778 stats->tags_barcode[tag].nbases = barcode_len;
779 stats->acgtno_barcode = realloc(stats->acgtno_barcode, (offset + barcode_len) * sizeof(acgtno_count_t));
780 stats->quals_barcode = realloc(stats->quals_barcode, (offset + barcode_len) * stats->nquals * sizeof(uint64_t));
781
782 if (!stats->acgtno_barcode || !stats->quals_barcode)
783 error("Error allocating memory. Aborting!\n");
784
785 memset(stats->acgtno_barcode + offset, 0, barcode_len*sizeof(acgtno_count_t));
786 memset(stats->quals_barcode + offset*stats->nquals, 0, barcode_len*stats->nquals*sizeof(uint64_t));
787 }
788
789 nbases = stats->tags_barcode[tag].nbases;
790 if (barcode_len > nbases) {
791 fprintf(stderr, "Barcodes with tag %s differ in length at sequence '%s'\n", barcode_tag, bam_get_qname(bam_line));
792 continue;
793 }
794
795 acgtno = stats->acgtno_barcode + stats->tags_barcode[tag].offset;
796 quals = stats->quals_barcode + stats->tags_barcode[tag].offset*stats->nquals;
797 maxqual = &stats->tags_barcode[tag].max_qual;
798 separator = &stats->tags_barcode[tag].tag_sep;
799 int error_flag = 0;
800
801 for (i = 0; i < barcode_len; i++) {
802 switch (barcode[i]) {
803 case 'A':
804 acgtno[i].a++;
805 break;
806 case 'C':
807 acgtno[i].c++;
808 break;
809 case 'G':
810 acgtno[i].g++;
811 break;
812 case 'T':
813 acgtno[i].t++;
814 break;
815 case 'N':
816 acgtno[i].n++;
817 break;
818 default:
819 if (*separator >= 0) {
820 if (*separator != i) {
821 if (stats->error_number < ERROR_LIMIT) {
822 fprintf(stderr, "Barcode separator for tag %s is in a different position or wrong barcode content('%s') at sequence '%s'\n", barcode_tag, barcode, bam_get_qname(bam_line));
823 stats->error_number++;
824 }
825 error_flag = 1;
826 }
827 } else {
828 *separator = i;
829 }
830 }
831
832 /* don't process the rest of the tag bases */
833 if (error_flag)
834 break;
835 }
836
837 /* skip to the next tag */
838 if (error_flag)
839 continue;
840
841 uint8_t* qt = bam_aux_get(bam_line, qual_tag);
842 if (!qt)
843 continue;
844
845 char* barqual = bam_aux2Z(qt);
846 if (!barqual)
847 continue;
848
849 uint32_t barqual_len = strlen(barqual);
850 if (barqual_len == barcode_len) {
851 for (i = 0; i < barcode_len; i++) {
852 int32_t qual = (int32_t)barqual[i] - '!'; // Phred + 33
853 if (qual >= 0 && qual < stats->nquals) {
854 quals[i * stats->nquals + qual]++;
855 if (qual > *maxqual)
856 *maxqual = qual;
857 }
858 }
859 } else {
860 if (stats->error_number++ < ERROR_LIMIT) {
861 fprintf(stderr, "%s length and %s length don't match for sequence '%s'\n", barcode_tag, qual_tag, bam_get_qname(bam_line));
862 }
863 }
864 }
865}
866
867// These stats should only be calculated for the original reads ignoring
868// supplementary artificial reads otherwise we'll accidentally double count
869void collect_orig_read_stats(bam1_t *bam_line, stats_t *stats, int* gc_count_out)
870{
871 int seq_len = bam_line->core.l_qseq;
872 stats->total_len += seq_len; // This ignores clipping so only count primary
873
874 if ( bam_line->core.flag & BAM_FQCFAIL ) stats->nreads_QCfailed++;
875 if ( bam_line->core.flag & BAM_FPAIRED ) stats->nreads_paired_tech++;
876
877 uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
878
879 // Count GC and ACGT per cycle. Note that cycle is approximate, clipping is ignored
880 uint8_t *seq = bam_get_seq(bam_line);
881 int i, read_cycle, gc_count = 0, reverse = IS_REVERSE(bam_line);
882
883 acgtno_count_t *acgtno_cycles = (order == READ_ORDER_FIRST) ? stats->acgtno_cycles_1st : (order == READ_ORDER_LAST) ? stats->acgtno_cycles_2nd : NULL ;
884 if (acgtno_cycles) {
885 for (i=0; i<seq_len; i++)
886 {
887 // Read cycle for current index
888 read_cycle = (reverse ? seq_len-i-1 : i);
889
890 // Conversion from uint8_t coding:
891 // -12-4---8------5
892 // =ACMGRSVTWYHKDBN
893 switch (bam_seqi(seq, i)) {
894 case 1:
895 acgtno_cycles[ read_cycle ].a++;
896 reverse ? stats->acgtno_revcomp[ read_cycle ].t++ : stats->acgtno_revcomp[ read_cycle ].a++;
897 break;
898 case 2:
899 acgtno_cycles[ read_cycle ].c++;
900 reverse ? stats->acgtno_revcomp[ read_cycle ].g++ : stats->acgtno_revcomp[ read_cycle ].c++;
901 gc_count++;
902 break;
903 case 4:
904 acgtno_cycles[ read_cycle ].g++;
905 reverse ? stats->acgtno_revcomp[ read_cycle ].c++ : stats->acgtno_revcomp[ read_cycle ].g++;
906 gc_count++;
907 break;
908 case 8:
909 reverse ? stats->acgtno_revcomp[ read_cycle ].a++ : stats->acgtno_revcomp[ read_cycle ].t++;
910 acgtno_cycles[ read_cycle ].t++;
911 break;
912 case 15:
913 acgtno_cycles[ read_cycle ].n++;
914 break;
915 default:
916 /*
917 * count "=" sequences in "other" along
918 * with MRSVWYHKDB ambiguity codes
919 */
920 acgtno_cycles[ read_cycle ].other++;
921 break;
922 }
923 }
924 }
925 int gc_idx_min = gc_count*(stats->ngc-1)/seq_len;
926 int gc_idx_max = (gc_count+1)*(stats->ngc-1)/seq_len;
927 if ( gc_idx_max >= stats->ngc ) gc_idx_max = stats->ngc - 1;
928
929 // Determine which array (1st or 2nd read) will these stats go to,
930 // trim low quality bases from end the same way BWA does,
931 // fill GC histogram
932 uint64_t *quals = NULL;
933 uint8_t *bam_quals = bam_get_qual(bam_line);
934
935 switch (order) {
936 case READ_ORDER_FIRST:
937 quals = stats->quals_1st;
938 stats->nreads_1st++;
939 stats->total_len_1st += seq_len;
940 for (i=gc_idx_min; i<gc_idx_max; i++)
941 stats->gc_1st[i]++;
942 break;
943 case READ_ORDER_LAST:
944 quals = stats->quals_2nd;
945 stats->nreads_2nd++;
946 stats->total_len_2nd += seq_len;
947 for (i=gc_idx_min; i<gc_idx_max; i++)
948 stats->gc_2nd[i]++;
949 break;
950 default:
951 stats->nreads_other++;
952 }
953 if ( stats->info->trim_qual>0 )
954 stats->nbases_trimmed += bwa_trim_read(stats->info->trim_qual, bam_quals, seq_len, reverse);
955
956 // Quality histogram and average quality. Clipping is neglected.
957 if (quals) {
958 for (i=0; i<seq_len; i++)
959 {
960 uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
961 if ( qual>=stats->nquals )
962 error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
963 if ( qual>stats->max_qual )
964 stats->max_qual = qual;
965
966 quals[ i*stats->nquals+qual ]++;
967 stats->sum_qual += qual;
968 }
969 }
970
971 // Barcode statistics
972 if (order == READ_ORDER_FIRST) {
973 collect_barcode_stats(bam_line, stats);
974 }
975
976 // Look at the flags and increment appropriate counters (mapped, paired, etc)
977 if ( IS_UNMAPPED(bam_line) )
978 {
979 stats->nreads_unmapped++;
980 }
981 else
982 {
983 stats->nbases_mapped += seq_len; // This ignores clipping so only count primary
984
985 if ( !bam_line->core.qual )
986 stats->nreads_mq0++;
987 if ( !IS_PAIRED_AND_MAPPED(bam_line) )
988 stats->nreads_single_mapped++;
989 else
990 {
991 stats->nreads_paired_and_mapped++;
992
993 if (IS_PROPERLYPAIRED(bam_line)) stats->nreads_properly_paired++;
994
995 if ( bam_line->core.tid!=bam_line->core.mtid )
996 stats->nreads_anomalous++;
997 }
998 }
999 *gc_count_out = gc_count;
1000}
1001
1002static int cleanup_overlaps(khash_t(qn2pair) *read_pairs, hts_pos_t max) {
1003 if ( !read_pairs )
1004 return 0;
1005
1006 int count = 0;
1007 khint_t k;
1008 for (k = kh_begin(read_pairs); k < kh_end(read_pairs); k++) {
1009 if ( kh_exist(read_pairs, k) ) {
1010 char *key = (char *)kh_key(read_pairs, k);
1011 pair_t *val = kh_val(read_pairs, k);
1012 if ( val && val->chunks ) {
1013 if ( val->chunks[val->n-1].end < max ) {
1014 free(val->chunks);
1015 free(val);
1016 free(key);
1017 kh_del(qn2pair, read_pairs, k);
1018 count++;
1019 }
1020 } else {
1021 free(key);
1022 kh_del(qn2pair, read_pairs, k);
1023 count++;
1024 }
1025 }
1026 }
1027 if ( max == INT64_MAX )
1028 kh_destroy(qn2pair, read_pairs);
1029
1030 return count;
1031}
1032
1033/**
1034 * [pmin, pmax) - 0 based half-open
1035 */
1036static void remove_overlaps(bam1_t *bam_line, khash_t(qn2pair) *read_pairs, stats_t *stats, hts_pos_t pmin, hts_pos_t pmax) {
1037 if ( !bam_line || !read_pairs || !stats )
1038 return;
1039
1040 uint32_t order = (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0);
1041 if ( !(bam_line->core.flag & BAM_FPAIRED) ||
1042 (bam_line->core.flag & BAM_FMUNMAP) ||
1043 (llabs(bam_line->core.isize) >= 2*bam_line->core.l_qseq) ||
1044 (order != READ_ORDER_FIRST && order != READ_ORDER_LAST) ) {
1045 if ( pmin >= 0 )
1046 round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
1047 return;
1048 }
1049
1050 char *qname = bam_get_qname(bam_line);
1051 if ( !qname ) {
1052 fprintf(stderr, "Error retrieving qname for line starting at pos %"PRIhts_pos"\n", bam_line->core.pos);
1053 return;
1054 }
1055
1056 khint_t k = kh_get(qn2pair, read_pairs, qname);
1057 if ( k == kh_end(read_pairs) ) { //first chunk from this template
1058 if ( pmin == -1 )
1059 return;
1060
1061 int ret;
1062 char *s = strdup(qname);
1063 if ( !s ) {
1064 fprintf(stderr, "Error allocating memory\n");
1065 return;
1066 }
1067
1068 k = kh_put(qn2pair, read_pairs, s, &ret);
1069 if ( -1 == ret ) {
1070 error("Error inserting read '%s' in pair hash table\n", qname);
1071 }
1072
1073 pair_t *pc = calloc(1, sizeof(pair_t));
1074 if ( !pc ) {
1075 fprintf(stderr, "Error allocating memory\n");
1076 return;
1077 }
1078
1079 pc->m = DEFAULT_CHUNK_NO;
1080 pc->chunks = calloc(pc->m, sizeof(hts_pair_pos_t));
1081 if ( !pc->chunks ) {
1082 fprintf(stderr, "Error allocating memory\n");
1083 free(pc);
1084 return;
1085 }
1086
1087 pc->chunks[0].beg = pmin;
1088 pc->chunks[0].end = pmax;
1089 pc->n = 1;
1090 pc->first = order;
1091
1092 kh_val(read_pairs, k) = pc;
1093 stats->pair_count++;
1094 } else { //template already present
1095 pair_t *pc = kh_val(read_pairs, k);
1096 if ( !pc ) {
1097 fprintf(stderr, "Invalid hash table entry\n");
1098 return;
1099 }
1100
1101 if ( order == pc->first ) { //chunk from an existing line
1102 if ( pmin == -1 )
1103 return;
1104
1105 if ( pc->n == pc->m ) {
1106 hts_pair_pos_t *tmp = realloc(pc->chunks, (pc->m<<1)*sizeof(hts_pair_pos_t));
1107 if ( !tmp ) {
1108 fprintf(stderr, "Error allocating memory\n");
1109 return;
1110 }
1111 pc->chunks = tmp;
1112 pc->m<<=1;
1113 }
1114
1115 pc->chunks[pc->n].beg = pmin;
1116 pc->chunks[pc->n].end = pmax;
1117 pc->n++;
1118 } else { //the other line, check for overlapping
1119 if ( pmin == -1 && kh_exist(read_pairs, k) ) { //job done, delete entry
1120 char *key = (char *)kh_key(read_pairs, k);
1121 pair_t *val = kh_val(read_pairs, k);
1122 if ( val) {
1123 free(val->chunks);
1124 free(val);
1125 }
1126 free(key);
1127 kh_del(qn2pair, read_pairs, k);
1128 stats->pair_count--;
1129 return;
1130 }
1131
1132 int i;
1133 for (i=0; i<pc->n; i++) {
1134 if ( pmin >= pc->chunks[i].end )
1135 continue;
1136
1137 if ( pmax <= pc->chunks[i].beg ) //no overlap
1138 break;
1139
1140 if ( pmin < pc->chunks[i].beg ) { //overlap at the beginning
1141 round_buffer_insert_read(&(stats->cov_rbuf), pmin, pc->chunks[i].beg);
1142 pmin = pc->chunks[i].beg;
1143 }
1144
1145 if ( pmax <= pc->chunks[i].end ) { //completely contained
1146 stats->nbases_mapped_cigar -= (pmax - pmin);
1147 return;
1148 } else { //overlap at the end
1149 stats->nbases_mapped_cigar -= (pc->chunks[i].end - pmin);
1150 pmin = pc->chunks[i].end;
1151 }
1152 }
1153 }
1154 }
1155 round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
1156}
1157
1158void collect_stats(bam1_t *bam_line, stats_t *stats, khash_t(qn2pair) *read_pairs)
1159{
1160 if ( !is_in_regions(bam_line,stats) )
1161 return;
1162 if ( stats->rg_hash )
1163 {
1164 const uint8_t *rg = bam_aux_get(bam_line, "RG");
1165 if ( !rg ) return; // certain read groups were requested but this record has none
1166 khint_t k = kh_get(rg, stats->rg_hash, (const char*)(rg + 1));
1167 if ( k == kh_end((kh_rg_t *)stats->rg_hash) ) return;
1168 }
1169 if ( stats->info->flag_require && (bam_line->core.flag & stats->info->flag_require)!=stats->info->flag_require )
1170 {
1171 stats->nreads_filtered++;
1172 return;
1173 }
1174 if ( stats->info->flag_filter && (bam_line->core.flag & stats->info->flag_filter) )
1175 {
1176 stats->nreads_filtered++;
1177 return;
1178 }
1179 if ( stats->info->filter_readlen!=-1 && bam_line->core.l_qseq!=stats->info->filter_readlen )
1180 return;
1181
1182 update_checksum(bam_line, stats);
1183
1184 // Secondary reads don't count for most stats purposes
1185 if ( bam_line->core.flag & BAM_FSECONDARY )
1186 {
1187 stats->nreads_secondary++;
1188 return;
1189 }
1190
1191 if ( bam_line->core.flag & BAM_FSUPPLEMENTARY )
1192 {
1193 stats->nreads_supplementary++;
1194 }
1195
1196 // If line has no sequence cannot continue
1197 int seq_len = bam_line->core.l_qseq;
1198 if ( !seq_len ) return;
1199
1200 if ( IS_DUP(bam_line) )
1201 {
1202 stats->total_len_dup += seq_len;
1203 stats->nreads_dup++;
1204 }
1205
1206 uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
1207
1208 int read_len = unclipped_length(bam_line);
1209 if ( read_len >= stats->nbases )
1210 realloc_buffers(stats,read_len);
1211 // Update max_len observed
1212 if ( stats->max_len<read_len )
1213 stats->max_len = read_len;
1214 if ( order == READ_ORDER_FIRST && stats->max_len_1st < read_len )
1215 stats->max_len_1st = read_len;
1216 if ( order == READ_ORDER_LAST && stats->max_len_2nd < read_len )
1217 stats->max_len_2nd = read_len;
1218 if ( ( bam_line->core.flag & (BAM_FUNMAP|BAM_FSECONDARY|BAM_FSUPPLEMENTARY|BAM_FQCFAIL|BAM_FDUP) ) == 0 )
1219 stats->mapping_qualities[bam_line->core.qual]++;
1220
1221 int i;
1222 int gc_count = 0;
1223
1224 // These stats should only be calculated for the original reads ignoring supplementary artificial reads
1225 // otherwise we'll accidentally double count
1226 if ( IS_ORIGINAL(bam_line) ) {
1227 stats->read_lengths[read_len]++;
1228 if ( order == READ_ORDER_FIRST ) stats->read_lengths_1st[read_len]++;
1229 if ( order == READ_ORDER_LAST ) stats->read_lengths_2nd[read_len]++;
1230 collect_orig_read_stats(bam_line, stats, &gc_count);
1231 }
1232
1233 // Look at the flags and increment appropriate counters (mapped, paired, etc)
1234 if ( IS_UNMAPPED(bam_line) ) return;
1235
1236 count_indels(stats, bam_line);
1237
1238 if ( IS_PAIRED_AND_MAPPED(bam_line) && IS_ORIGINAL(bam_line) )
1239 {
1240 // The insert size is tricky, because for long inserts the libraries are
1241 // prepared differently and the pairs point in other direction. BWA does
1242 // not set the paired flag for them. Similar thing is true also for 454
1243 // reads. Mates mapped to different chromosomes have isize==0.
1244 int32_t isize = bam_line->core.isize;
1245 if ( isize<0 ) isize = -isize;
1246 if ( stats->info->nisize > 0 && isize > stats->info->nisize )
1247 isize = stats->info->nisize;
1248 if ( isize>0 || bam_line->core.tid==bam_line->core.mtid )
1249 {
1250 hts_pos_t pos_fst = bam_line->core.mpos - bam_line->core.pos;
1251 int is_fst = IS_READ1(bam_line) ? 1 : -1;
1252 int is_fwd = IS_REVERSE(bam_line) ? -1 : 1;
1253 int is_mfwd = IS_MATE_REVERSE(bam_line) ? -1 : 1;
1254
1255 if ( is_fwd*is_mfwd>0 )
1256 stats->isize->inc_other(stats->isize->data, isize);
1257 else if ( is_fst*pos_fst>=0 )
1258 {
1259 if ( is_fst*is_fwd>0 )
1260 stats->isize->inc_inward(stats->isize->data, isize);
1261 else
1262 stats->isize->inc_outward(stats->isize->data, isize);
1263 }
1264 else if ( is_fst*pos_fst<0 )
1265 {
1266 if ( is_fst*is_fwd>0 )
1267 stats->isize->inc_outward(stats->isize->data, isize);
1268 else
1269 stats->isize->inc_inward(stats->isize->data, isize);
1270 }
1271 }
1272 }
1273
1274 // Number of mismatches
1275 uint8_t *nm = bam_aux_get(bam_line,"NM");
1276 if (nm)
1277 stats->nmismatches += bam_aux2i(nm);
1278
1279 // Number of mapped bases from cigar
1280 if ( bam_line->core.n_cigar == 0)
1281 error("FIXME: mapped read with no cigar?\n");
1282 int readlen=seq_len;
1283 if ( stats->regions )
1284 {
1285 // Count only on-target bases
1286 hts_pos_t iref = bam_line->core.pos + 1;
1287 for (i=0; i<bam_line->core.n_cigar; i++)
1288 {
1289 int cig = bam_cigar_op(bam_get_cigar(bam_line)[i]);
1290 int ncig = bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1291 if ( !ncig ) continue; // curiously, this can happen: 0D
1292 if ( cig==BAM_CDEL ) readlen += ncig;
1293 else if ( cig==BAM_CMATCH || cig==BAM_CEQUAL || cig==BAM_CDIFF )
1294 {
1295 if ( iref < stats->reg_from ) ncig -= stats->reg_from-iref;
1296 else if ( iref+ncig-1 > stats->reg_to ) ncig -= iref+ncig-1 - stats->reg_to;
1297 if ( ncig<0 ) ncig = 0;
1298 stats->nbases_mapped_cigar += ncig;
1299 iref += bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1300 }
1301 else if ( cig==BAM_CINS )
1302 {
1303 iref += ncig;
1304 if ( iref>=stats->reg_from && iref<=stats->reg_to )
1305 stats->nbases_mapped_cigar += ncig;
1306 }
1307 }
1308 }
1309 else
1310 {
1311 // Count the whole read
1312 for (i=0; i<bam_line->core.n_cigar; i++)
1313 {
1314 int cig = bam_cigar_op(bam_get_cigar(bam_line)[i]);
1315 if ( cig==BAM_CMATCH || cig==BAM_CINS || cig==BAM_CEQUAL || cig==BAM_CDIFF )
1316 stats->nbases_mapped_cigar += bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1317 if ( cig==BAM_CDEL )
1318 readlen += bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1319 }
1320 }
1321
1322 if ( stats->tid==bam_line->core.tid && bam_line->core.pos<stats->pos )
1323 stats->is_sorted = 0;
1324 stats->pos = bam_line->core.pos;
1325
1326 if ( stats->is_sorted )
1327 {
1328 if ( stats->tid==-1 || stats->tid!=bam_line->core.tid ) {
1329 round_buffer_flush(stats, -1);
1330 }
1331
1332 //cleanup the pair hash table to free memory
1333 stats->last_read_flush++;
1334 if ( stats->pair_count > DEFAULT_PAIR_MAX && stats->last_read_flush > DEFAULT_PAIR_MAX) {
1335 stats->pair_count -= cleanup_overlaps(read_pairs, bam_line->core.pos);
1336 stats->last_read_flush = 0;
1337 }
1338
1339 if ( stats->last_pair_tid != bam_line->core.tid) {
1340 stats->pair_count -= cleanup_overlaps(read_pairs, INT64_MAX-1);
1341 stats->last_pair_tid = bam_line->core.tid;
1342 stats->last_read_flush = 0;
1343 }
1344
1345 // Mismatches per cycle and GC-depth graph. For simplicity, reads overlapping GCD bins
1346 // are not splitted which results in up to seq_len-1 overlaps. The default bin size is
1347 // 20kbp, so the effect is negligible.
1348 if ( stats->info->fai )
1349 {
1350 hts_pos_t inc_ref = 0;
1351 int inc_gcd = 0;
1352 // First pass or new chromosome, or read goes beyond the rseq buffer
1353 if ( stats->rseq_pos==-1 || stats->tid != bam_line->core.tid
1354 || stats->rseq_pos+stats->nrseq_buf < bam_line->core.pos+readlen) {
1355 inc_ref=bam_line->core.pos+readlen;
1356 inc_gcd=1;
1357 }
1358 // Read overlaps the next gcd bin
1359 else if ( stats->gcd_pos+stats->info->gcd_bin_size < bam_line->core.pos+readlen )
1360 {
1361 inc_gcd = 1;
1362 if ( stats->rseq_pos+stats->nrseq_buf < bam_line->core.pos+stats->info->gcd_bin_size ) inc_ref = bam_line->core.pos+stats->info->gcd_bin_size;
1363 }
1364 if ( inc_gcd )
1365 {
1366 stats->igcd++;
1367 if ( stats->igcd >= stats->ngcd )
1368 realloc_gcd_buffer(stats, readlen);
1369 if ( inc_ref )
1370 read_ref_seq(stats, bam_line->core.tid,
1371 bam_line->core.pos, inc_ref);
1372 stats->gcd_pos = bam_line->core.pos;
1373 stats->gcd[ stats->igcd ].gc = fai_gc_content(stats, stats->gcd_pos, stats->info->gcd_bin_size);
1374 }
1375
1376 count_mismatches_per_cycle(stats,bam_line,read_len);
1377 }
1378 // No reference and first pass, new chromosome or sequence going beyond the end of the gcd bin
1379 else if ( stats->gcd_pos==-1 || stats->tid != bam_line->core.tid || bam_line->core.pos - stats->gcd_pos > stats->info->gcd_bin_size )
1380 {
1381 // First pass or a new chromosome
1382 stats->tid = bam_line->core.tid;
1383 stats->gcd_pos = bam_line->core.pos;
1384 stats->igcd++;
1385 if ( stats->igcd >= stats->ngcd )
1386 realloc_gcd_buffer(stats, readlen);
1387 }
1388 stats->gcd[ stats->igcd ].depth++;
1389 // When no reference sequence is given, approximate the GC from the read (much shorter window, but otherwise OK)
1390 if ( !stats->info->fai )
1391 stats->gcd[ stats->igcd ].gc += (float) gc_count / seq_len;
1392
1393 // Coverage distribution graph
1394 round_buffer_flush(stats,bam_line->core.pos);
1395 if ( stats->regions ) {
1396 hts_pos_t p = bam_line->core.pos, pnew, pmin = 0, pmax = 0;
1397 uint32_t j = 0;
1398 i = 0;
1399 while ( j < bam_line->core.n_cigar && i < stats->nchunks ) {
1400 int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
1401 int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
1402 switch(op) {
1403 case BAM_CMATCH:
1404 case BAM_CEQUAL:
1405 case BAM_CDIFF:
1406 pmin = MAX(p, stats->chunks[i].beg-1); // 0 based
1407 pmax = MIN(p+oplen, stats->chunks[i].end); // 1 based
1408 if ( pmax > pmin ) {
1409 if ( stats->info->remove_overlaps )
1410 remove_overlaps(bam_line, read_pairs, stats, pmin, pmax);
1411 else
1412 round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
1413 }
1414 break;
1415 case BAM_CDEL:
1416 break;
1417 }
1418 pnew = p + (bam_cigar_type(op)&2 ? oplen : 0); // consumes reference
1419
1420 if ( pnew >= stats->chunks[i].end ) {
1421 // go to the next chunk
1422 i++;
1423 } else {
1424 // go to the next CIGAR op
1425 j++;
1426 p = pnew;
1427 }
1428 }
1429 } else {
1430 hts_pos_t p = bam_line->core.pos;
1431 uint32_t j;
1432 for (j = 0; j < bam_line->core.n_cigar; j++) {
1433 int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
1434 int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
1435 switch(op) {
1436 case BAM_CMATCH:
1437 case BAM_CEQUAL:
1438 case BAM_CDIFF:
1439 if ( stats->info->remove_overlaps )
1440 remove_overlaps(bam_line, read_pairs, stats, p, p+oplen);
1441 else
1442 round_buffer_insert_read(&(stats->cov_rbuf), p, p+oplen);
1443 break;
1444 case BAM_CDEL:
1445 break;
1446 }
1447 p += bam_cigar_type(op)&2 ? oplen : 0; // consumes reference
1448 }
1449 }
1450 if ( stats->info->remove_overlaps )
1451 remove_overlaps(bam_line, read_pairs, stats, -1LL, -1LL); //remove the line from the hash table
1452 }
1453}
1454
1455// Sort by GC and depth
1456#define GCD_t(x) ((gc_depth_t *)x)
1457static int gcd_cmp(const void *a, const void *b)
1458{
1459 if ( GCD_t(a)->gc < GCD_t(b)->gc ) return -1;
1460 if ( GCD_t(a)->gc > GCD_t(b)->gc ) return 1;
1461 if ( GCD_t(a)->depth < GCD_t(b)->depth ) return -1;
1462 if ( GCD_t(a)->depth > GCD_t(b)->depth ) return 1;
1463 return 0;
1464}
1465#undef GCD_t
1466
1467float gcd_percentile(gc_depth_t *gcd, int N, int p)
1468{
1469 float n,d;
1470 int k;
1471
1472 n = (float)p*(N+1)/100;
1473 k = n;
1474 if ( k<=0 )
1475 return gcd[0].depth;
1476 if ( k>=N )
1477 return gcd[N-1].depth;
1478
1479 d = n - k;
1480 return gcd[k-1].depth + d*(gcd[k].depth - gcd[k-1].depth);
1481}
1482
1483void output_stats(FILE *to, stats_t *stats, int sparse)
1484{
1485 // Calculate average insert size and standard deviation (from the main bulk data only)
1486 int isize, ibulk=0, icov, imapq=0;
1487 uint64_t nisize=0, nisize_inward=0, nisize_outward=0, nisize_other=0, cov_sum=0;
1488 double bulk=0, avg_isize=0, sd_isize=0;
1489 for (isize=0; isize<stats->isize->nitems(stats->isize->data); isize++)
1490 {
1491 // Each pair was counted twice
1492 stats->isize->set_inward(stats->isize->data, isize, stats->isize->inward(stats->isize->data, isize) * 0.5);
1493 stats->isize->set_outward(stats->isize->data, isize, stats->isize->outward(stats->isize->data, isize) * 0.5);
1494 stats->isize->set_other(stats->isize->data, isize, stats->isize->other(stats->isize->data, isize) * 0.5);
1495
1496 nisize_inward += stats->isize->inward(stats->isize->data, isize);
1497 nisize_outward += stats->isize->outward(stats->isize->data, isize);
1498 nisize_other += stats->isize->other(stats->isize->data, isize);
1499 nisize += stats->isize->inward(stats->isize->data, isize) + stats->isize->outward(stats->isize->data, isize) + stats->isize->other(stats->isize->data, isize);
1500 }
1501
1502 for (isize=0; isize<stats->isize->nitems(stats->isize->data); isize++)
1503 {
1504 uint64_t num = stats->isize->inward(stats->isize->data, isize) + stats->isize->outward(stats->isize->data, isize) + stats->isize->other(stats->isize->data, isize);
1505 if (num > 0) ibulk = isize + 1;
1506 bulk += num;
1507 avg_isize += isize * (stats->isize->inward(stats->isize->data, isize) + stats->isize->outward(stats->isize->data, isize) + stats->isize->other(stats->isize->data, isize));
1508
1509 if ( bulk/nisize > stats->info->isize_main_bulk )
1510 {
1511 ibulk = isize+1;
1512 nisize = bulk;
1513 break;
1514 }
1515 }
1516 avg_isize /= nisize ? nisize : 1;
1517 for (isize=1; isize<ibulk; isize++)
1518 sd_isize += (stats->isize->inward(stats->isize->data, isize) + stats->isize->outward(stats->isize->data, isize) +stats->isize->other(stats->isize->data, isize)) * (isize-avg_isize)*(isize-avg_isize) / (nisize ? nisize : 1);
1519 sd_isize = sqrt(sd_isize);
1520
1521 fprintf(to, "# This file was produced by samtools stats (%s+htslib-%s) and can be plotted using plot-bamstats\n", samtools_version(), hts_version());
1522 if( stats->split_name != NULL ){
1523 fprintf(to, "# This file contains statistics only for reads with tag: %s=%s\n", stats->info->split_tag, stats->split_name);
1524 }
1525 else{
1526 fprintf(to, "# This file contains statistics for all reads.\n");
1527 }
1528 fprintf(to, "# The command line was: %s",stats->info->argv[0]);
1529 int i;
1530 for (i=1; i<stats->info->argc; i++)
1531 fprintf(to, " %s", stats->info->argv[i]);
1532 fprintf(to, "\n");
1533 fprintf(to, "# CHK, Checksum\t[2]Read Names\t[3]Sequences\t[4]Qualities\n");
1534 fprintf(to, "# CHK, CRC32 of reads which passed filtering followed by addition (32bit overflow)\n");
1535 fprintf(to, "CHK\t%08x\t%08x\t%08x\n", stats->checksum.names,stats->checksum.reads,stats->checksum.quals);
1536 fprintf(to, "# Summary Numbers. Use `grep ^SN | cut -f 2-` to extract this part.\n");
1537 fprintf(to, "SN\traw total sequences:\t%ld\t# excluding supplementary and secondary reads\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)); // not counting excluded seqs (and none of the below)
1538 fprintf(to, "SN\tfiltered sequences:\t%ld\n", (long)stats->nreads_filtered);
1539 fprintf(to, "SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other));
1540 fprintf(to, "SN\tis sorted:\t%d\n", stats->is_sorted ? 1 : 0);
1541 fprintf(to, "SN\t1st fragments:\t%ld\n", (long)stats->nreads_1st);
1542 fprintf(to, "SN\tlast fragments:\t%ld\n", (long)stats->nreads_2nd);
1543 fprintf(to, "SN\treads mapped:\t%ld\n", (long)(stats->nreads_paired_and_mapped+stats->nreads_single_mapped));
1544 fprintf(to, "SN\treads mapped and paired:\t%ld\t# paired-end technology bit set + both mates mapped\n", (long)stats->nreads_paired_and_mapped);
1545 fprintf(to, "SN\treads unmapped:\t%ld\n", (long)stats->nreads_unmapped);
1546 fprintf(to, "SN\treads properly paired:\t%ld\t# proper-pair bit set\n", (long)stats->nreads_properly_paired);
1547 fprintf(to, "SN\treads paired:\t%ld\t# paired-end technology bit set\n", (long)stats->nreads_paired_tech);
1548 fprintf(to, "SN\treads duplicated:\t%ld\t# PCR or optical duplicate bit set\n", (long)stats->nreads_dup);
1549 fprintf(to, "SN\treads MQ0:\t%ld\t# mapped and MQ=0\n", (long)stats->nreads_mq0);
1550 fprintf(to, "SN\treads QC failed:\t%ld\n", (long)stats->nreads_QCfailed);
1551 fprintf(to, "SN\tnon-primary alignments:\t%ld\n", (long)stats->nreads_secondary);
1552 fprintf(to, "SN\tsupplementary alignments:\t%ld\n", (long)stats->nreads_supplementary);
1553 fprintf(to, "SN\ttotal length:\t%ld\t# ignores clipping\n", (long)stats->total_len);
1554 fprintf(to, "SN\ttotal first fragment length:\t%ld\t# ignores clipping\n", (long)stats->total_len_1st);
1555 fprintf(to, "SN\ttotal last fragment length:\t%ld\t# ignores clipping\n", (long)stats->total_len_2nd);
1556 fprintf(to, "SN\tbases mapped:\t%ld\t# ignores clipping\n", (long)stats->nbases_mapped); // the length of the whole read goes here, including soft-clips etc.
1557 fprintf(to, "SN\tbases mapped (cigar):\t%ld\t# more accurate\n", (long)stats->nbases_mapped_cigar); // only matched and inserted bases are counted here
1558 fprintf(to, "SN\tbases trimmed:\t%ld\n", (long)stats->nbases_trimmed);
1559 fprintf(to, "SN\tbases duplicated:\t%ld\n", (long)stats->total_len_dup);
1560 fprintf(to, "SN\tmismatches:\t%ld\t# from NM fields\n", (long)stats->nmismatches);
1561 fprintf(to, "SN\terror rate:\t%e\t# mismatches / bases mapped (cigar)\n", stats->nbases_mapped_cigar ? (float)stats->nmismatches/stats->nbases_mapped_cigar : 0);
1562 float avg_read_length = (stats->nreads_1st +
1563 stats->nreads_2nd +
1564 stats->nreads_other)
1565 ? (float)stats->total_len / (stats->nreads_1st +
1566 stats->nreads_2nd +
1567 stats->nreads_other)
1568 : 0;
1569 fprintf(to, "SN\taverage length:\t%.0f\n", avg_read_length);
1570 fprintf(to, "SN\taverage first fragment length:\t%.0f\n", stats->nreads_1st? (float)stats->total_len_1st/stats->nreads_1st:0);
1571 fprintf(to, "SN\taverage last fragment length:\t%.0f\n", stats->nreads_2nd? (float)stats->total_len_2nd/stats->nreads_2nd:0);
1572 fprintf(to, "SN\tmaximum length:\t%d\n", stats->max_len);
1573 fprintf(to, "SN\tmaximum first fragment length:\t%d\n", stats->max_len_1st);
1574 fprintf(to, "SN\tmaximum last fragment length:\t%d\n", stats->max_len_2nd);
1575 fprintf(to, "SN\taverage quality:\t%.1f\n", stats->total_len?stats->sum_qual/stats->total_len:0);
1576 fprintf(to, "SN\tinsert size average:\t%.1f\n", avg_isize);
1577 fprintf(to, "SN\tinsert size standard deviation:\t%.1f\n", sd_isize);
1578 fprintf(to, "SN\tinward oriented pairs:\t%ld\n", (long)nisize_inward);
1579 fprintf(to, "SN\toutward oriented pairs:\t%ld\n", (long)nisize_outward);
1580 fprintf(to, "SN\tpairs with other orientation:\t%ld\n", (long)nisize_other);
1581 fprintf(to, "SN\tpairs on different chromosomes:\t%ld\n", (long)stats->nreads_anomalous/2);
1582 fprintf(to, "SN\tpercentage of properly paired reads (%%):\t%.1f\n", (stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)? (float)(100*stats->nreads_properly_paired)/(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other):0);
1583 if ( stats->target_count ) {
1584 fprintf(to, "SN\tbases inside the target:\t%" PRIu64 "\n", stats->target_count);
1585 for (icov=stats->info->cov_threshold+1; icov<stats->ncov; icov++)
1586 cov_sum += stats->cov[icov];
1587 fprintf(to, "SN\tpercentage of target genome with coverage > %d (%%):\t%.2f\n", stats->info->cov_threshold, (float)(100*cov_sum)/stats->target_count);
1588 }
1589
1590 int ibase,iqual;
1591 if ( stats->max_len<stats->nbases ) stats->max_len++;
1592 if ( stats->max_qual+1<stats->nquals ) stats->max_qual++;
1593 fprintf(to, "# First Fragment Qualities. Use `grep ^FFQ | cut -f 2-` to extract this part.\n");
1594 fprintf(to, "# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n");
1595 for (ibase=0; ibase<stats->max_len_1st; ibase++)
1596 {
1597 fprintf(to, "FFQ\t%d",ibase+1);
1598 for (iqual=0; iqual<=stats->max_qual; iqual++)
1599 {
1600 fprintf(to, "\t%ld", (long)stats->quals_1st[ibase*stats->nquals+iqual]);
1601 }
1602 fprintf(to, "\n");
1603 }
1604 fprintf(to, "# Last Fragment Qualities. Use `grep ^LFQ | cut -f 2-` to extract this part.\n");
1605 fprintf(to, "# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n");
1606 for (ibase=0; ibase<stats->max_len_2nd; ibase++)
1607 {
1608 fprintf(to, "LFQ\t%d",ibase+1);
1609 for (iqual=0; iqual<=stats->max_qual; iqual++)
1610 {
1611 fprintf(to, "\t%ld", (long)stats->quals_2nd[ibase*stats->nquals+iqual]);
1612 }
1613 fprintf(to, "\n");
1614 }
1615 if ( stats->mpc_buf )
1616 {
1617 fprintf(to, "# Mismatches per cycle and quality. Use `grep ^MPC | cut -f 2-` to extract this part.\n");
1618 fprintf(to, "# Columns correspond to qualities, rows to cycles. First column is the cycle number, second\n");
1619 fprintf(to, "# is the number of N's and the rest is the number of mismatches\n");
1620 for (ibase=0; ibase<stats->max_len; ibase++)
1621 {
1622 fprintf(to, "MPC\t%d",ibase+1);
1623 for (iqual=0; iqual<=stats->max_qual; iqual++)
1624 {
1625 fprintf(to, "\t%ld", (long)stats->mpc_buf[ibase*stats->nquals+iqual]);
1626 }
1627 fprintf(to, "\n");
1628 }
1629 }
1630 fprintf(to, "# GC Content of first fragments. Use `grep ^GCF | cut -f 2-` to extract this part.\n");
1631 int ibase_prev = 0;
1632 for (ibase=0; ibase<stats->ngc; ibase++)
1633 {
1634 if ( stats->gc_1st[ibase]==stats->gc_1st[ibase_prev] ) continue;
1635 fprintf(to, "GCF\t%.2f\t%ld\n", (ibase+ibase_prev)*0.5*100./(stats->ngc-1), (long)stats->gc_1st[ibase_prev]);
1636 ibase_prev = ibase;
1637 }
1638 fprintf(to, "# GC Content of last fragments. Use `grep ^GCL | cut -f 2-` to extract this part.\n");
1639 ibase_prev = 0;
1640 for (ibase=0; ibase<stats->ngc; ibase++)
1641 {
1642 if ( stats->gc_2nd[ibase]==stats->gc_2nd[ibase_prev] ) continue;
1643 fprintf(to, "GCL\t%.2f\t%ld\n", (ibase+ibase_prev)*0.5*100./(stats->ngc-1), (long)stats->gc_2nd[ibase_prev]);
1644 ibase_prev = ibase;
1645 }
1646 fprintf(to, "# ACGT content per cycle. Use `grep ^GCC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
1647 for (ibase=0; ibase<stats->max_len; ibase++)
1648 {
1649 acgtno_count_t *acgtno_count_1st = &(stats->acgtno_cycles_1st[ibase]);
1650 acgtno_count_t *acgtno_count_2nd = &(stats->acgtno_cycles_2nd[ibase]);
1651 uint64_t acgt_sum = acgtno_count_1st->a + acgtno_count_1st->c + acgtno_count_1st->g + acgtno_count_1st->t +
1652 acgtno_count_2nd->a + acgtno_count_2nd->c + acgtno_count_2nd->g + acgtno_count_2nd->t;
1653 if ( ! acgt_sum ) continue;
1654 fprintf(to, "GCC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1655 100.*(acgtno_count_1st->a + acgtno_count_2nd->a)/acgt_sum,
1656 100.*(acgtno_count_1st->c + acgtno_count_2nd->c)/acgt_sum,
1657 100.*(acgtno_count_1st->g + acgtno_count_2nd->g)/acgt_sum,
1658 100.*(acgtno_count_1st->t + acgtno_count_2nd->t)/acgt_sum,
1659 100.*(acgtno_count_1st->n + acgtno_count_2nd->n)/acgt_sum,
1660 100.*(acgtno_count_1st->other + acgtno_count_2nd->other)/acgt_sum);
1661 }
1662 fprintf(to, "# ACGT content per cycle, read oriented. Use `grep ^GCT | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]\n");
1663 for (ibase=0; ibase<stats->max_len; ibase++)
1664 {
1665 acgtno_count_t *acgtno_count = &(stats->acgtno_revcomp[ibase]);
1666 uint64_t acgt_sum = acgtno_count->a + acgtno_count->c + acgtno_count->g + acgtno_count->t;
1667 if ( ! acgt_sum ) continue;
1668 fprintf(to, "GCT\t%d\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1669 100.*(acgtno_count->a)/acgt_sum,
1670 100.*(acgtno_count->c)/acgt_sum,
1671 100.*(acgtno_count->g)/acgt_sum,
1672 100.*(acgtno_count->t)/acgt_sum);
1673 }
1674
1675 uint64_t tA=0, tC=0, tG=0, tT=0, tN=0;
1676 fprintf(to, "# ACGT content per cycle for first fragments. Use `grep ^FBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
1677 for (ibase=0; ibase<stats->max_len; ibase++)
1678 {
1679 acgtno_count_t *acgtno_count_1st = &(stats->acgtno_cycles_1st[ibase]);
1680 uint64_t acgt_sum_1st = acgtno_count_1st->a + acgtno_count_1st->c + acgtno_count_1st->g + acgtno_count_1st->t;
1681 tA += acgtno_count_1st->a;
1682 tC += acgtno_count_1st->c;
1683 tG += acgtno_count_1st->g;
1684 tT += acgtno_count_1st->t;
1685 tN += acgtno_count_1st->n;
1686
1687 if ( acgt_sum_1st )
1688 fprintf(to, "FBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1689 100.*acgtno_count_1st->a/acgt_sum_1st,
1690 100.*acgtno_count_1st->c/acgt_sum_1st,
1691 100.*acgtno_count_1st->g/acgt_sum_1st,
1692 100.*acgtno_count_1st->t/acgt_sum_1st,
1693 100.*acgtno_count_1st->n/acgt_sum_1st,
1694 100.*acgtno_count_1st->other/acgt_sum_1st);
1695
1696 }
1697 fprintf(to, "# ACGT raw counters for first fragments. Use `grep ^FTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
1698 fprintf(to, "FTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
1699 tA=0, tC=0, tG=0, tT=0, tN=0;
1700 fprintf(to, "# ACGT content per cycle for last fragments. Use `grep ^LBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
1701 for (ibase=0; ibase<stats->max_len; ibase++)
1702 {
1703 acgtno_count_t *acgtno_count_2nd = &(stats->acgtno_cycles_2nd[ibase]);
1704 uint64_t acgt_sum_2nd = acgtno_count_2nd->a + acgtno_count_2nd->c + acgtno_count_2nd->g + acgtno_count_2nd->t;
1705 tA += acgtno_count_2nd->a;
1706 tC += acgtno_count_2nd->c;
1707 tG += acgtno_count_2nd->g;
1708 tT += acgtno_count_2nd->t;
1709 tN += acgtno_count_2nd->n;
1710
1711 if ( acgt_sum_2nd )
1712 fprintf(to, "LBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1713 100.*acgtno_count_2nd->a/acgt_sum_2nd,
1714 100.*acgtno_count_2nd->c/acgt_sum_2nd,
1715 100.*acgtno_count_2nd->g/acgt_sum_2nd,
1716 100.*acgtno_count_2nd->t/acgt_sum_2nd,
1717 100.*acgtno_count_2nd->n/acgt_sum_2nd,
1718 100.*acgtno_count_2nd->other/acgt_sum_2nd);
1719
1720 }
1721 fprintf(to, "# ACGT raw counters for last fragments. Use `grep ^LTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
1722 fprintf(to, "LTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
1723
1724 int tag;
1725 for (tag=0; tag<stats->ntags; tag++) {
1726 if (stats->tags_barcode[tag].nbases) {
1727 fprintf(to, "# ACGT content per cycle for barcodes. Use `grep ^%sC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N counts as a percentage of all A/C/G/T bases [%%]\n",
1728 stats->tags_barcode[tag].tag_name);
1729 for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
1730 {
1731 if (ibase == stats->tags_barcode[tag].tag_sep)
1732 continue;
1733
1734 acgtno_count_t *acgtno_count = stats->acgtno_barcode + stats->tags_barcode[tag].offset + ibase;
1735 uint64_t acgt_sum = acgtno_count->a + acgtno_count->c + acgtno_count->g + acgtno_count->t;
1736
1737 if ( acgt_sum )
1738 fprintf(to, "%sC%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", stats->tags_barcode[tag].tag_name,
1739 stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
1740 stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep,
1741 100.*acgtno_count->a/acgt_sum,
1742 100.*acgtno_count->c/acgt_sum,
1743 100.*acgtno_count->g/acgt_sum,
1744 100.*acgtno_count->t/acgt_sum,
1745 100.*acgtno_count->n/acgt_sum);
1746 }
1747
1748 fprintf(to, "# Barcode Qualities. Use `grep ^%sQ | cut -f 2-` to extract this part.\n", stats->tags_barcode[tag].qual_name);
1749 fprintf(to, "# Columns correspond to qualities and rows to barcode cycles. First column is the cycle number.\n");
1750 for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
1751 {
1752 if (ibase == stats->tags_barcode[tag].tag_sep)
1753 continue;
1754
1755 fprintf(to, "%sQ%d\t%d", stats->tags_barcode[tag].qual_name,
1756 stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
1757 stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep);
1758 for (iqual=0; iqual<=stats->tags_barcode[tag].max_qual; iqual++)
1759 {
1760 fprintf(to, "\t%ld", (long)stats->quals_barcode[(stats->tags_barcode[tag].offset + ibase)*stats->nquals+iqual]);
1761 }
1762 fprintf(to, "\n");
1763 }
1764 }
1765 }
1766
1767 fprintf(to, "# Insert sizes. Use `grep ^IS | cut -f 2-` to extract this part. The columns are: insert size, pairs total, inward oriented pairs, outward oriented pairs, other pairs\n");
1768 for (isize=0; isize<ibulk; isize++) {
1769 long in = (long)(stats->isize->inward(stats->isize->data, isize));
1770 long out = (long)(stats->isize->outward(stats->isize->data, isize));
1771 long other = (long)(stats->isize->other(stats->isize->data, isize));
1772 if (!sparse || in + out + other > 0) {
1773 fprintf(to, "IS\t%d\t%ld\t%ld\t%ld\t%ld\n", isize, in+out+other,
1774 in , out, other);
1775 }
1776 }
1777
1778 fprintf(to, "# Read lengths. Use `grep ^RL | cut -f 2-` to extract this part. The columns are: read length, count\n");
1779 int ilen;
1780 for (ilen=0; ilen<stats->max_len; ilen++)
1781 {
1782 if ( stats->read_lengths[ilen+1]>0 )
1783 fprintf(to, "RL\t%d\t%ld\n", ilen+1, (long)stats->read_lengths[ilen+1]);
1784 }
1785
1786 fprintf(to, "# Read lengths - first fragments. Use `grep ^FRL | cut -f 2-` to extract this part. The columns are: read length, count\n");
1787 for (ilen=0; ilen<stats->max_len_1st; ilen++)
1788 {
1789 if ( stats->read_lengths_1st[ilen+1]>0 )
1790 fprintf(to, "FRL\t%d\t%ld\n", ilen+1, (long)stats->read_lengths_1st[ilen+1]);
1791 }
1792
1793 fprintf(to, "# Read lengths - last fragments. Use `grep ^LRL | cut -f 2-` to extract this part. The columns are: read length, count\n");
1794 for (ilen=0; ilen<stats->max_len_2nd; ilen++)
1795 {
1796 if ( stats->read_lengths_2nd[ilen+1]>0 )
1797 fprintf(to, "LRL\t%d\t%ld\n", ilen+1, (long)stats->read_lengths_2nd[ilen+1]);
1798 }
1799
1800 fprintf(to, "# Mapping qualities for reads !(UNMAP|SECOND|SUPPL|QCFAIL|DUP). Use `grep ^MAPQ | cut -f 2-` to extract this part. The columns are: mapq, count\n");
1801 for (imapq=0; imapq < 256; imapq++)
1802 {
1803 if ( stats->mapping_qualities[imapq]>0 )
1804 fprintf(to, "MAPQ\t%d\t%ld\n", imapq, (long)stats->mapping_qualities[imapq]);
1805 }
1806
1807 fprintf(to, "# Indel distribution. Use `grep ^ID | cut -f 2-` to extract this part. The columns are: length, number of insertions, number of deletions\n");
1808
1809 for (ilen=0; ilen<stats->nindels; ilen++)
1810 {
1811 if ( stats->insertions[ilen]>0 || stats->deletions[ilen]>0 )
1812 fprintf(to, "ID\t%d\t%ld\t%ld\n", ilen+1, (long)stats->insertions[ilen], (long)stats->deletions[ilen]);
1813 }
1814
1815 fprintf(to, "# Indels per cycle. Use `grep ^IC | cut -f 2-` to extract this part. The columns are: cycle, number of insertions (fwd), .. (rev) , number of deletions (fwd), .. (rev)\n");
1816 for (ilen=0; ilen<=stats->nbases; ilen++)
1817 {
1818 // For deletions we print the index of the cycle before the deleted base (1-based) and for insertions
1819 // the index of the cycle of the first inserted base (also 1-based)
1820 if ( stats->ins_cycles_1st[ilen]>0 || stats->ins_cycles_2nd[ilen]>0 || stats->del_cycles_1st[ilen]>0 || stats->del_cycles_2nd[ilen]>0 )
1821 fprintf(to, "IC\t%d\t%ld\t%ld\t%ld\t%ld\n", ilen+1, (long)stats->ins_cycles_1st[ilen], (long)stats->ins_cycles_2nd[ilen], (long)stats->del_cycles_1st[ilen], (long)stats->del_cycles_2nd[ilen]);
1822 }
1823
1824 fprintf(to, "# Coverage distribution. Use `grep ^COV | cut -f 2-` to extract this part.\n");
1825 if ( stats->cov[0] )
1826 fprintf(to, "COV\t[<%d]\t%d\t%ld\n",stats->info->cov_min,stats->info->cov_min-1, (long)stats->cov[0]);
1827 for (icov=1; icov<stats->ncov-1; icov++)
1828 if ( stats->cov[icov] )
1829 fprintf(to, "COV\t[%d-%d]\t%d\t%ld\n",stats->info->cov_min + (icov-1)*stats->info->cov_step, stats->info->cov_min + icov*stats->info->cov_step-1,stats->info->cov_min + icov*stats->info->cov_step-1, (long)stats->cov[icov]);
1830 if ( stats->cov[stats->ncov-1] )
1831 fprintf(to, "COV\t[%d<]\t%d\t%ld\n",stats->info->cov_min + (stats->ncov-2)*stats->info->cov_step-1,stats->info->cov_min + (stats->ncov-2)*stats->info->cov_step-1, (long)stats->cov[stats->ncov-1]);
1832
1833 // Calculate average GC content, then sort by GC and depth
1834 fprintf(to, "# GC-depth. Use `grep ^GCD | cut -f 2-` to extract this part. The columns are: GC%%, unique sequence percentiles, 10th, 25th, 50th, 75th and 90th depth percentile\n");
1835 uint32_t igcd;
1836 for (igcd=0; igcd<stats->igcd; igcd++)
1837 {
1838 if ( stats->info->fai )
1839 stats->gcd[igcd].gc = rint(100. * stats->gcd[igcd].gc);
1840 else
1841 if ( stats->gcd[igcd].depth )
1842 stats->gcd[igcd].gc = rint(100. * stats->gcd[igcd].gc / stats->gcd[igcd].depth);
1843 }
1844 if ( stats->ngcd )
1845 qsort(stats->gcd, stats->igcd+1, sizeof(gc_depth_t), gcd_cmp);
1846 igcd = 0;
1847 while ( igcd < stats->igcd )
1848 {
1849 // Calculate percentiles (10,25,50,75,90th) for the current GC content and print
1850 uint32_t nbins=0, itmp=igcd;
1851 float gc = stats->gcd[igcd].gc;
1852 while ( itmp<stats->igcd && fabs(stats->gcd[itmp].gc-gc)<0.1 )
1853 {
1854 nbins++;
1855 itmp++;
1856 }
1857 fprintf(to, "GCD\t%.1f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n", gc, (igcd+nbins+1)*100./(stats->igcd+1),
1858 gcd_percentile(&(stats->gcd[igcd]),nbins,10) *avg_read_length/stats->info->gcd_bin_size,
1859 gcd_percentile(&(stats->gcd[igcd]),nbins,25) *avg_read_length/stats->info->gcd_bin_size,
1860 gcd_percentile(&(stats->gcd[igcd]),nbins,50) *avg_read_length/stats->info->gcd_bin_size,
1861 gcd_percentile(&(stats->gcd[igcd]),nbins,75) *avg_read_length/stats->info->gcd_bin_size,
1862 gcd_percentile(&(stats->gcd[igcd]),nbins,90) *avg_read_length/stats->info->gcd_bin_size
1863 );
1864 igcd += nbins;
1865 }
1866}
1867
1868static void init_regions(stats_t *stats, const char *file, stats_info_t* info)
1869{
1870 FILE *fp = fopen(file,"r");
1871 if ( !fp ) error("%s: %s\n",file,strerror(errno));
1872
1873 kstring_t line = { 0, 0, NULL };
1874 int warned = 0, r, p, new_p;
1875 int prev_tid=-1;
1876 hts_pos_t prev_pos=-1LL;
1877 while (line.l = 0, kgetline(&line, (kgets_func *)fgets, fp) >= 0)
1878 {
1879 if ( line.s[0] == '#' ) continue;
1880
1881 int i = 0;
1882 while ( i<line.l && !isspace(line.s[i]) ) i++;
1883 if ( i>=line.l ) error("Could not parse the file: %s [%s]\n", file, line.s);
1884 line.s[i] = '\0';
1885
1886 int tid = bam_name2id(stats->info->sam_header, line.s);
1887 if ( tid < 0 )
1888 {
1889 if ( !warned )
1890 fprintf(stderr,"Warning: Some sequences not present in the BAM, e.g. \"%s\". This message is printed only once.\n", line.s);
1891 warned = 1;
1892 continue;
1893 }
1894
1895 if ( tid >= stats->nregions )
1896 {
1897 if(!(stats->regions = realloc(stats->regions,sizeof(regions_t)*(tid+REG_INC))))
1898 error("Could not allocate memory for region.\n");
1899
1900 int j;
1901 for (j=stats->nregions; j<tid+REG_INC; j++)
1902 {
1903 stats->regions[j].npos = stats->regions[j].mpos = stats->regions[j].cpos = 0;
1904 stats->regions[j].pos = NULL;
1905 }
1906 stats->nregions = tid+REG_INC;
1907 }
1908 int npos = stats->regions[tid].npos;
1909 if ( npos >= stats->regions[tid].mpos )
1910 {
1911 stats->regions[tid].mpos = npos+POS_INC;
1912 if (!(stats->regions[tid].pos = realloc(stats->regions[tid].pos, sizeof(hts_pair_pos_t)*stats->regions[tid].mpos)))
1913 error("Could not allocate memory for interval.\n");
1914 }
1915
1916 if ( (sscanf(&line.s[i+1],"%"SCNd64" %"SCNd64, &stats->regions[tid].pos[npos].beg, &stats->regions[tid].pos[npos].end))!=2 ) error("Could not parse the region [%s]\n", &line.s[i+1]);
1917 if ( prev_tid==-1 || prev_tid!=tid )
1918 {
1919 prev_tid = tid;
1920 prev_pos = stats->regions[tid].pos[npos].beg;
1921 }
1922 if ( prev_pos>stats->regions[tid].pos[npos].beg )
1923 error("The positions are not in chromosomal order (%s:%"PRIhts_pos" comes after %"PRIhts_pos")\n", line.s, stats->regions[tid].pos[npos].beg, prev_pos);
1924 stats->regions[tid].npos++;
1925 if ( stats->regions[tid].npos > stats->nchunks )
1926 stats->nchunks = stats->regions[tid].npos;
1927 }
1928 free(line.s);
1929 if ( !stats->regions ) error("Unable to map the -t sequences to the BAM sequences.\n");
1930 fclose(fp);
1931
1932 // sort region intervals and remove duplicates
1933 for (r = 0; r < stats->nregions; r++) {
1934 regions_t *reg = &stats->regions[r];
1935 if ( reg->npos > 1 ) {
1936 qsort(reg->pos, reg->npos, sizeof(hts_pair_pos_t), regions_lt);
1937 for (new_p = 0, p = 1; p < reg->npos; p++) {
1938 if ( reg->pos[new_p].end < reg->pos[p].beg )
1939 reg->pos[++new_p] = reg->pos[p];
1940 else if ( reg->pos[new_p].end < reg->pos[p].end )
1941 reg->pos[new_p].end = reg->pos[p].end;
1942 }
1943 reg->npos = ++new_p;
1944 }
1945 for (p = 0; p < reg->npos; p++) {
1946 if (reg->pos[p].end < HTS_POS_MAX) {
1947 stats->target_count += (reg->pos[p].end - reg->pos[p].beg + 1);
1948 } else {
1949 uint64_t hdr_end = sam_hdr_tid2len(info->sam_header, r);
1950 if (hdr_end)
1951 stats->target_count += (hdr_end - reg->pos[p].beg + 1);
1952 }
1953 }
1954 }
1955
1956 if (!(stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t))))
1957 error("Could not allocate memory for chunk.\n");
1958}
1959
1960void destroy_regions(stats_t *stats)
1961{
1962 int i;
1963 for (i=0; i<stats->nregions; i++)
1964 {
1965 if ( !stats->regions[i].mpos ) continue;
1966 free(stats->regions[i].pos);
1967 }
1968 if ( stats->regions ) free(stats->regions);
1969 if ( stats->chunks ) free(stats->chunks);
1970}
1971
1972void reset_regions(stats_t *stats)
1973{
1974 int i;
1975 for (i=0; i<stats->nregions; i++)
1976 stats->regions[i].cpos = 0;
1977}
1978
1979int is_in_regions(bam1_t *bam_line, stats_t *stats)
1980{
1981 if ( !stats->regions ) return 1;
1982
1983 if ( bam_line->core.tid >= stats->nregions || bam_line->core.tid<0 ) return 0;
1984 if ( !stats->is_sorted ) error("The BAM must be sorted in order for -t to work.\n");
1985
1986 regions_t *reg = &stats->regions[bam_line->core.tid];
1987 if ( reg->cpos==reg->npos ) return 0; // done for this chr
1988
1989 // Find a matching interval or skip this read. No splicing of reads is done, no indels or soft clips considered,
1990 // even small overlap is enough to include the read in the stats.
1991 int i = reg->cpos;
1992 while ( i<reg->npos && reg->pos[i].end<=bam_line->core.pos ) i++;
1993 if ( i>=reg->npos ) { reg->cpos = reg->npos; return 0; }
1994 int64_t endpos = bam_endpos(bam_line);
1995 if ( endpos < reg->pos[i].beg ) return 0;
1996
1997 //found a read overlapping a region
1998 reg->cpos = i;
1999 stats->reg_from = reg->pos[i].beg;
2000 stats->reg_to = reg->pos[i].end;
2001
2002 //now find all the overlapping chunks
2003 stats->nchunks = 0;
2004 while (i < reg->npos) {
2005 if (bam_line->core.pos < reg->pos[i].end && endpos >= reg->pos[i].beg) {
2006 stats->chunks[stats->nchunks].beg = MAX(bam_line->core.pos+1, reg->pos[i].beg);
2007 stats->chunks[stats->nchunks].end = MIN(endpos, reg->pos[i].end);
2008 stats->nchunks++;
2009 }
2010 i++;
2011 }
2012
2013 return 1;
2014}
2015
2016int replicate_regions(stats_t *stats, hts_itr_multi_t *iter, stats_info_t *info) {
2017 if ( !stats || !iter)
2018 return 1;
2019
2020 int i, j, tid;
2021 stats->nregions = iter->n_reg;
2022 stats->regions = calloc(stats->nregions, sizeof(regions_t));
2023 stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t));
2024 if ( !stats->regions || !stats->chunks )
2025 return 1;
2026
2027 for (i = 0; i < iter->n_reg; i++) {
2028 tid = iter->reg_list[i].tid;
2029 if ( tid < 0 )
2030 continue;
2031
2032 if ( tid >= stats->nregions ) {
2033 regions_t *tmp = realloc(stats->regions, (tid+10) * sizeof(regions_t));
2034 if ( !tmp )
2035 return 1;
2036 stats->regions = tmp;
2037 memset(stats->regions + stats->nregions, 0,
2038 (tid+10-stats->nregions) * sizeof(regions_t));
2039 stats->nregions = tid+10;
2040 }
2041
2042 stats->regions[tid].mpos = stats->regions[tid].npos = iter->reg_list[i].count;
2043 stats->regions[tid].pos = calloc(stats->regions[tid].mpos, sizeof(hts_pair_pos_t));
2044 if ( !stats->regions[tid].pos )
2045 return 1;
2046
2047 for (j = 0; j < stats->regions[tid].npos; j++) {
2048 stats->regions[tid].pos[j].beg = iter->reg_list[i].intervals[j].beg+1;
2049 stats->regions[tid].pos[j].end = iter->reg_list[i].intervals[j].end;
2050 if (stats->regions[tid].pos[j].end < HTS_POS_MAX) {
2051 stats->target_count += (stats->regions[tid].pos[j].end - stats->regions[tid].pos[j].beg + 1);
2052 } else {
2053 uint64_t hdr_end = sam_hdr_tid2len(info->sam_header, tid);
2054 if (hdr_end)
2055 stats->target_count += (hdr_end - stats->regions[tid].pos[j].beg + 1);
2056 }
2057 }
2058 }
2059
2060 return 0;
2061}
2062
2063static void init_group_id(stats_t *stats, stats_info_t *info, const char *id)
2064{
2065 stats->rg_hash = kh_init(rg);
2066 if (!stats->rg_hash) error("Could not initialise RG set\n");
2067 sam_hdr_t *hdr = info->sam_header;
2068 const char *key;
2069 kstring_t sm = KS_INITIALIZE;
2070 int i, ret, nrg = sam_hdr_count_lines(hdr, "RG");
2071 if (nrg < 0) error("Could not parse header\n");
2072
2073 for (i=0; i<nrg; i++) {
2074 key = sam_hdr_line_name(hdr, "RG", i);
2075 if (!strcmp(key, id)) {
2076 kh_put(rg, stats->rg_hash, key, &ret);
2077 if (ret == -1) { ks_free(&sm); error("Could not add key \"%s\" to RG set\n", key); }
2078 } else { /* Check for SM name, as per manual */
2079 if (!sam_hdr_find_tag_pos(hdr, "RG", i, "SM", &sm)) {
2080 if (!strcmp(ks_c_str(&sm), id)) {
2081 kh_put(rg, stats->rg_hash, key, &ret);
2082 if (ret == -1) { ks_free(&sm); error("Could not add key \"%s\" to RG set\n", key); }
2083 }
2084 }
2085 }
2086 }
2087
2088 ks_free(&sm);
2089}
2090
2091
2092static void HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) HTS_NORETURN
2093error(const char *format, ...)
2094{
2095 if ( !format )
2096 {
2097 printf("About: The program collects statistics from BAM files. The output can be visualized using plot-bamstats.\n");
2098 printf("Usage: samtools stats [OPTIONS] file.bam\n");
2099 printf(" samtools stats [OPTIONS] file.bam chr:from-to\n");
2100 printf("Options:\n");
2101 printf(" -c, --coverage <int>,<int>,<int> Coverage distribution min,max,step [1,1000,1]\n");
2102 printf(" -d, --remove-dups Exclude from statistics reads marked as duplicates\n");
2103 printf(" -X, --customized-index-file Use a customized index file\n");
2104 printf(" -f, --required-flag <str|int> Required flag, 0 for unset. See also `samtools flags` [0]\n");
2105 printf(" -F, --filtering-flag <str|int> Filtering flag, 0 for unset. See also `samtools flags` [0]\n");
2106 printf(" --GC-depth <float> the size of GC-depth bins (decreasing bin size increases memory requirement) [2e4]\n");
2107 printf(" -h, --help This help message\n");
2108 printf(" -i, --insert-size <int> Maximum insert size [8000]\n");
2109 printf(" -I, --id <string> Include only listed read group or sample name\n");
2110 printf(" -l, --read-length <int> Include in the statistics only reads with the given read length [-1]\n");
2111 printf(" -m, --most-inserts <float> Report only the main part of inserts [0.99]\n");
2112 printf(" -P, --split-prefix <str> Path or string prefix for filepaths output by -S (default is input filename)\n");
2113 printf(" -q, --trim-quality <int> The BWA trimming parameter [0]\n");
2114 printf(" -r, --ref-seq <file> Reference sequence (required for GC-depth and mismatches-per-cycle calculation).\n");
2115 printf(" -s, --sam Ignored (input format is auto-detected).\n");
2116 printf(" -S, --split <tag> Also write statistics to separate files split by tagged field.\n");
2117 printf(" -t, --target-regions <file> Do stats in these regions only. Tab-delimited file chr,from,to, 1-based, inclusive.\n");
2118 printf(" -x, --sparse Suppress outputting IS rows where there are no insertions.\n");
2119 printf(" -p, --remove-overlaps Remove overlaps of paired-end reads from coverage and base count computations.\n");
2120 printf(" -g, --cov-threshold <int> Only bases with coverage above this value will be included in the target percentage computation [0]\n");
2121 sam_global_opt_help(stdout, "-.--.@-.");
2122 printf("\n");
2123 }
2124 else
2125 {
2126 va_list ap;
2127 va_start(ap, format);
2128 vfprintf(stderr, format, ap);
2129 va_end(ap);
2130 }
2131 exit(1);
2132}
2133
2134void cleanup_stats_info(stats_info_t* info){
2135 if (info->fai) fai_destroy(info->fai);
2136 if (info->sam) sam_close(info->sam);
2137 free(info);
2138}
2139
2140void cleanup_stats(stats_t* stats)
2141{
2142 free(stats->cov_rbuf.buffer); free(stats->cov);
2143 free(stats->quals_1st); free(stats->quals_2nd);
2144 free(stats->gc_1st); free(stats->gc_2nd);
2145 stats->isize->isize_free(stats->isize->data);
2146 free(stats->isize);
2147 free(stats->gcd);
2148 free(stats->rseq_buf);
2149 free(stats->mpc_buf);
2150 free(stats->acgtno_cycles_1st);
2151 free(stats->acgtno_cycles_2nd);
2152 free(stats->acgtno_revcomp);
2153 free(stats->read_lengths);
2154 free(stats->read_lengths_1st);
2155 free(stats->read_lengths_2nd);
2156 free(stats->insertions);
2157 free(stats->deletions);
2158 free(stats->ins_cycles_1st);
2159 free(stats->ins_cycles_2nd);
2160 free(stats->del_cycles_1st);
2161 free(stats->del_cycles_2nd);
2162 if (stats->acgtno_barcode) free(stats->acgtno_barcode);
2163 if (stats->quals_barcode) free(stats->quals_barcode);
2164 free(stats->tags_barcode);
2165 destroy_regions(stats);
2166 if ( stats->rg_hash ) kh_destroy(rg, stats->rg_hash);
2167 free(stats->split_name);
2168 free(stats->mapping_qualities);
2169 free(stats);
2170}
2171
2172void output_split_stats(khash_t(c2stats) *split_hash, char* bam_fname, int sparse)
2173{
2174 int i = 0;
2175 kstring_t output_filename = { 0, 0, NULL };
2176 stats_t *curr_stats = NULL;
2177 for(i = kh_begin(split_hash); i != kh_end(split_hash); ++i){
2178 if(!kh_exist(split_hash, i)) continue;
2179 curr_stats = kh_value(split_hash, i);
2180 round_buffer_flush(curr_stats, -1);
2181
2182 output_filename.l = 0;
2183 if (curr_stats->info->split_prefix)
2184 kputs(curr_stats->info->split_prefix, &output_filename);
2185 else
2186 kputs(bam_fname, &output_filename);
2187 kputc('_', &output_filename);
2188 kputs(curr_stats->split_name, &output_filename);
2189 kputs(".bamstat", &output_filename);
2190
2191 FILE *to = fopen(output_filename.s, "w");
2192 if(to == NULL){
2193 error("Could not open '%s' for writing.\n", output_filename.s);
2194 }
2195 output_stats(to, curr_stats, sparse);
2196 fclose(to);
2197 }
2198
2199 free(output_filename.s);
2200}
2201
2202void destroy_split_stats(khash_t(c2stats) *split_hash)
2203{
2204 if (!split_hash)
2205 return;
2206
2207 int i = 0;
2208 stats_t *curr_stats = NULL;
2209 for(i = kh_begin(split_hash); i != kh_end(split_hash); ++i){
2210 if(!kh_exist(split_hash, i)) continue;
2211 curr_stats = kh_value(split_hash, i);
2212 cleanup_stats(curr_stats);
2213 }
2214 kh_destroy(c2stats, split_hash);
2215}
2216
2217stats_info_t* stats_info_init(int argc, char *argv[])
2218{
2219 stats_info_t* info = calloc(1, sizeof(stats_info_t));
2220 if (!info) {
2221 return NULL;
2222 }
2223
2224 info->nisize = 8000;
2225 info->isize_main_bulk = 0.99; // There are always outliers at the far end
2226 info->gcd_bin_size = 20e3;
2227 info->cov_min = 1;
2228 info->cov_max = 1000;
2229 info->cov_step = 1;
2230 info->filter_readlen = -1;
2231 info->argc = argc;
2232 info->argv = argv;
2233 info->remove_overlaps = 0;
2234 info->cov_threshold = 0;
2235
2236 return info;
2237}
2238
2239int init_stat_info_fname(stats_info_t* info, const char* bam_fname, const htsFormat* in_fmt)
2240{
2241 // .. bam
2242 samFile* sam;
2243 if ((sam = sam_open_format(bam_fname, "r", in_fmt)) == 0) {
2244 print_error_errno("stats", "failed to open \"%s\"", bam_fname);
2245 return 1;
2246 }
2247 info->sam = sam;
2248 info->sam_header = sam_hdr_read(sam);
2249 if (info->sam_header == NULL) {
2250 print_error("stats", "failed to read header for \"%s\"", bam_fname);
2251 return 1;
2252 }
2253 return 0;
2254}
2255
2256stats_t* stats_init(void)
2257{
2258 stats_t *stats = calloc(1,sizeof(stats_t));
2259 if (!stats)
2260 return NULL;
2261
2262 stats->ngc = 200;
2263 stats->nquals = 256;
2264 stats->nbases = 300;
2265 stats->rseq_pos = -1;
2266 stats->tid = -1;
2267 stats->gcd_pos = -1LL;
2268 stats->igcd = 0;
2269 stats->is_sorted = 1;
2270 stats->nindels = stats->nbases;
2271 stats->split_name = NULL;
2272 stats->nchunks = 0;
2273 stats->pair_count = 0;
2274 stats->last_pair_tid = -2;
2275 stats->last_read_flush = 0;
2276 stats->target_count = 0;
2277
2278 return stats;
2279}
2280
2281static int init_barcode_tags(stats_t* stats) {
2282 stats->ntags = 4;
2283 stats->tags_barcode = calloc(stats->ntags, sizeof(barcode_info_t));
2284 if (!stats->tags_barcode)
2285 return -1;
2286 stats->tags_barcode[0] = (barcode_info_t){"BC", "QT", 0, -1, -1, 0};
2287 stats->tags_barcode[1] = (barcode_info_t){"CR", "CY", 0, -1, -1, 0};
2288 stats->tags_barcode[2] = (barcode_info_t){"OX", "BZ", 0, -1, -1, 0};
2289 stats->tags_barcode[3] = (barcode_info_t){"RX", "QX", 0, -1, -1, 0};
2290 return 0;
2291}
2292
2293static void init_stat_structs(stats_t* stats, stats_info_t* info, const char* group_id, const char* targets)
2294{
2295 // Give stats_t a pointer to the info struct
2296 // This saves us having to pass the stats_info_t to every function
2297 stats->info = info;
2298
2299 // Init structures
2300 // .. coverage bins and round buffer
2301 if ( info->cov_step > info->cov_max - info->cov_min + 1 )
2302 {
2303 info->cov_step = info->cov_max - info->cov_min;
2304 if ( info->cov_step <= 0 )
2305 info->cov_step = 1;
2306 }
2307 stats->ncov = 3 + (info->cov_max-info->cov_min) / info->cov_step;
2308 info->cov_max = info->cov_min + ((info->cov_max-info->cov_min)/info->cov_step +1)*info->cov_step - 1;
2309 stats->cov = calloc(sizeof(uint64_t),stats->ncov);
2310 if (!stats->cov) goto nomem;
2311 stats->cov_rbuf.size = stats->nbases*5;
2312 stats->cov_rbuf.buffer = calloc(sizeof(int32_t),stats->cov_rbuf.size);
2313 if (!stats->cov_rbuf.buffer) goto nomem;
2314 if ( group_id ) init_group_id(stats, info, group_id);
2315 // .. arrays
2316 stats->quals_1st = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
2317 if (!stats->quals_1st) goto nomem;
2318 stats->quals_2nd = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
2319 if (!stats->quals_2nd) goto nomem;
2320 stats->gc_1st = calloc(stats->ngc,sizeof(uint64_t));
2321 if (!stats->gc_1st) goto nomem;
2322 stats->gc_2nd = calloc(stats->ngc,sizeof(uint64_t));
2323 if (!stats->gc_2nd) goto nomem;
2324 stats->isize = init_isize_t(info->nisize ?info->nisize+1 :0);
2325 if (!stats->isize) goto nomem;
2326 stats->gcd = calloc(stats->ngcd,sizeof(gc_depth_t));
2327 if (!stats->gcd) goto nomem;
2328 if (info->fai) {
2329 stats->mpc_buf = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
2330 if (!stats->mpc_buf) goto nomem;
2331 } else {
2332 stats->mpc_buf = NULL;
2333 }
2334 stats->acgtno_cycles_1st = calloc(stats->nbases,sizeof(acgtno_count_t));
2335 if (!stats->acgtno_cycles_1st) goto nomem;
2336 stats->acgtno_cycles_2nd = calloc(stats->nbases,sizeof(acgtno_count_t));
2337 if (!stats->acgtno_cycles_2nd) goto nomem;
2338 stats->acgtno_revcomp = calloc(stats->nbases,sizeof(acgtno_count_t));
2339 if (!stats->acgtno_revcomp) goto nomem;
2340 stats->read_lengths = calloc(stats->nbases,sizeof(uint64_t));
2341 if (!stats->read_lengths) goto nomem;
2342 stats->read_lengths_1st = calloc(stats->nbases,sizeof(uint64_t));
2343 if (!stats->read_lengths_1st) goto nomem;
2344 stats->read_lengths_2nd = calloc(stats->nbases,sizeof(uint64_t));
2345 if (!stats->read_lengths_2nd) goto nomem;
2346 stats->insertions = calloc(stats->nbases,sizeof(uint64_t));
2347 if (!stats->insertions) goto nomem;
2348 stats->deletions = calloc(stats->nbases,sizeof(uint64_t));
2349 if (!stats->deletions) goto nomem;
2350 stats->ins_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
2351 if (!stats->ins_cycles_1st) goto nomem;
2352 stats->ins_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
2353 if (!stats->ins_cycles_2nd) goto nomem;
2354 stats->del_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
2355 if (!stats->del_cycles_1st) goto nomem;
2356 stats->del_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
2357 if (!stats->del_cycles_2nd) goto nomem;
2358 stats->mapping_qualities = calloc(256,sizeof(uint64_t));
2359 if(!stats->mapping_qualities) goto nomem;
2360 if (init_barcode_tags(stats) < 0)
2361 goto nomem;
2362 realloc_rseq_buffer(stats);
2363 if ( targets )
2364 init_regions(stats, targets, info);
2365 return;
2366 nomem:
2367 error("Out of memory");
2368}
2369
2370static stats_t* get_curr_split_stats(bam1_t* bam_line, khash_t(c2stats)* split_hash, stats_info_t* info, char* targets)
2371{
2372 stats_t *curr_stats = NULL;
2373 const uint8_t *tag_val = bam_aux_get(bam_line, info->split_tag);
2374 if(tag_val == 0){
2375 error("Tag '%s' not found in bam_line.\n", info->split_tag);
2376 }
2377 char* split_name = strdup(bam_aux2Z(tag_val));
2378
2379 // New stats object, under split
2380 khiter_t k = kh_get(c2stats, split_hash, split_name);
2381 if(k == kh_end(split_hash)){
2382 curr_stats = stats_init(); // mallocs new instance
2383 if (!curr_stats) {
2384 error("Couldn't allocate split stats");
2385 }
2386 init_stat_structs(curr_stats, info, NULL, targets);
2387 curr_stats->split_name = split_name;
2388
2389 // Record index in hash
2390 int ret = 0;
2391 khiter_t iter = kh_put(c2stats, split_hash, split_name, &ret);
2392 if( ret < 0 ){
2393 error("Failed to insert key '%s' into split_hash", split_name);
2394 }
2395 kh_val(split_hash, iter) = curr_stats; // store pointer to stats
2396 }
2397 else{
2398 curr_stats = kh_value(split_hash, k);
2399 free(split_name); // don't need to hold on to this if it wasn't new
2400 }
2401 return curr_stats;
2402}
2403
2404int main_stats(int argc, char *argv[])
2405{
2406 char *targets = NULL;
2407 char *bam_fname = NULL;
2408 char *bam_idx_fname = NULL;
2409 char *group_id = NULL;
2410 int sparse = 0, has_index_file = 0, ret = 1;
2411 sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
2412
2413 stats_info_t *info = stats_info_init(argc, argv);
2414 if (!info) {
2415 fprintf(stderr, "Could not allocate memory for info.\n");
2416 return 1;
2417 }
2418
2419 static const struct option loptions[] =
2420 {
2421 SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
2422 {"help", no_argument, NULL, 'h'},
2423 {"remove-dups", no_argument, NULL, 'd'},
2424 {"sam", no_argument, NULL, 's'},
2425 {"customized-index-file", required_argument, NULL, 'X'},
2426 {"ref-seq", required_argument, NULL, 'r'},
2427 {"coverage", required_argument, NULL, 'c'},
2428 {"read-length", required_argument, NULL, 'l'},
2429 {"insert-size", required_argument, NULL, 'i'},
2430 {"most-inserts", required_argument, NULL, 'm'},
2431 {"trim-quality", required_argument, NULL, 'q'},
2432 {"target-regions", required_argument, NULL, 't'},
2433 {"required-flag", required_argument, NULL, 'f'},
2434 {"filtering-flag", required_argument, NULL, 'F'},
2435 {"id", required_argument, NULL, 'I'},
2436 {"GC-depth", required_argument, NULL, 1},
2437 {"sparse", no_argument, NULL, 'x'},
2438 {"split", required_argument, NULL, 'S'},
2439 {"split-prefix", required_argument, NULL, 'P'},
2440 {"remove-overlaps", no_argument, NULL, 'p'},
2441 {"cov-threshold", required_argument, NULL, 'g'},
2442 {NULL, 0, NULL, 0}
2443 };
2444 int opt, tmp_flag;
2445
2446 while ( (opt=getopt_long(argc,argv,"?hdsXxpr:c:l:i:t:m:q:f:F:g:I:S:P:@:",loptions,NULL))>0 )
2447 {
2448 switch (opt)
2449 {
2450 case 'f':
2451 tmp_flag = bam_str2flag(optarg);
2452
2453 if (tmp_flag < 0) {
2454 print_error("stats", "Unknown flag '%s'", optarg);
2455 return 1;
2456 }
2457
2458 info->flag_require = tmp_flag;
2459 break;
2460
2461 case 'F':
2462 tmp_flag = bam_str2flag(optarg);
2463
2464 if (tmp_flag < 0) {
2465 print_error("stats", "Unknown flag '%s'", optarg);
2466 return 1;
2467 }
2468
2469 info->flag_filter |= tmp_flag;
2470 break;
2471
2472 case 'd': info->flag_filter |= BAM_FDUP; break;
2473 case 'X': has_index_file = 1; break;
2474 case 's': break;
2475 case 'r': info->fai = fai_load(optarg);
2476 if (info->fai==NULL)
2477 error("Could not load faidx: %s\n", optarg);
2478 break;
2479 case 1 : info->gcd_bin_size = atof(optarg); break;
2480 case 'c': if ( sscanf(optarg,"%d,%d,%d",&info->cov_min,&info->cov_max,&info->cov_step)!= 3 )
2481 error("Unable to parse -c %s\n", optarg);
2482 break;
2483 case 'l': info->filter_readlen = atoi(optarg); break;
2484 case 'i': info->nisize = atoi(optarg); break;
2485 case 'm': info->isize_main_bulk = atof(optarg); break;
2486 case 'q': info->trim_qual = atoi(optarg); break;
2487 case 't': targets = optarg; break;
2488 case 'I': group_id = optarg; break;
2489 case 'x': sparse = 1; break;
2490 case 'S': info->split_tag = optarg; break;
2491 case 'P': info->split_prefix = optarg; break;
2492 case 'p': info->remove_overlaps = 1; break;
2493 case 'g': info->cov_threshold = atoi(optarg);
2494 if ( info->cov_threshold < 0 || info->cov_threshold == INT_MAX )
2495 error("Unsupported value for coverage threshold %d\n", info->cov_threshold);
2496 break;
2497 case '?':
2498 case 'h': error(NULL);
2499 /* no break */
2500 default:
2501 if (parse_sam_global_opt(opt, optarg, loptions, &ga) != 0)
2502 error("Unknown argument: %s\n", optarg);
2503 break;
2504 }
2505 }
2506
2507 bam_fname = argv[optind++];
2508 if ( !bam_fname )
2509 {
2510 if ( isatty(STDIN_FILENO) )
2511 error(NULL);
2512 bam_fname = "-";
2513 }
2514
2515 if (init_stat_info_fname(info, bam_fname, &ga.in)) {
2516 cleanup_stats_info(info);
2517 return 1;
2518 }
2519
2520 if (has_index_file && !(bam_idx_fname = argv[optind++])) {
2521 fprintf(stderr, "No index file provided\n");
2522 cleanup_stats_info(info);
2523 return 1;
2524 }
2525
2526 if (ga.nthreads > 0)
2527 hts_set_threads(info->sam, ga.nthreads);
2528
2529 stats_t *all_stats = stats_init();
2530 if (!all_stats) {
2531 fprintf(stderr, "Could not allocate memory for stats.\n");
2532 cleanup_stats_info(info);
2533 return 1;
2534 }
2535 stats_t *curr_stats = NULL;
2536 init_stat_structs(all_stats, info, group_id, targets);
2537 // Init
2538 // .. hash
2539 khash_t(c2stats)* split_hash = kh_init(c2stats);
2540 if (!split_hash) goto cleanup_all_stats;
2541
2542 khash_t(qn2pair)* read_pairs = kh_init(qn2pair);
2543 if (!read_pairs) goto cleanup_split_hash;
2544
2545 // Collect statistics
2546 bam1_t *bam_line = bam_init1();
2547 if (!bam_line) goto cleanup_read_pairs;
2548
2549 if (optind < argc) {
2550 // Region:interval arguments in the command line
2551 hts_idx_t *bam_idx = NULL;
2552 if (has_index_file) {
2553 bam_idx = sam_index_load2(info->sam, bam_fname, bam_idx_fname);
2554 } else {
2555 // If an index filename has not been specified, look alongside the alignment file
2556 bam_idx = sam_index_load(info->sam, bam_fname);
2557 }
2558
2559 if (bam_idx) {
2560 hts_itr_multi_t *iter = sam_itr_regarray(bam_idx, info->sam_header, &argv[optind], argc - optind);
2561 if (iter) {
2562 if (!targets) {
2563 all_stats->nchunks = argc-optind;
2564 if (replicate_regions(all_stats, iter, info))
2565 fprintf(stderr, "Replications of the regions failed\n");
2566 }
2567
2568 if ( all_stats->nregions && all_stats->regions ) {
2569 while ((ret = sam_itr_next(info->sam, iter, bam_line)) >= 0) {
2570 if (info->split_tag) {
2571 curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
2572 collect_stats(bam_line, curr_stats, read_pairs);
2573 }
2574 collect_stats(bam_line, all_stats, read_pairs);
2575 }
2576
2577 if (ret < -1) {
2578 fprintf(stderr, "Failure while running the iterator\n");
2579 hts_itr_multi_destroy(iter);
2580 hts_idx_destroy(bam_idx);
2581 goto cleanup;
2582 }
2583 }
2584 hts_itr_multi_destroy(iter);
2585 } else {
2586 fprintf(stderr, "Multi-region iterator could not be created\n");
2587 hts_idx_destroy(bam_idx);
2588 goto cleanup;
2589 }
2590 hts_idx_destroy(bam_idx);
2591 } else {
2592 if (has_index_file)
2593 fprintf(stderr, "Invalid index file '%s'\n", bam_idx_fname);
2594 fprintf(stderr, "Random alignment retrieval only works for indexed files\n");
2595 goto cleanup;
2596 }
2597 } else {
2598 if ( info->cov_threshold > 0 && !targets ) {
2599 fprintf(stderr, "Coverage percentage calculation requires a list of target regions\n");
2600 goto cleanup;
2601 }
2602
2603 // Stream through the entire BAM ignoring off-target regions if -t is given
2604 while ((ret = sam_read1(info->sam, info->sam_header, bam_line)) >= 0) {
2605 if (info->split_tag) {
2606 curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
2607 collect_stats(bam_line, curr_stats, read_pairs);
2608 }
2609 collect_stats(bam_line, all_stats, read_pairs);
2610 }
2611
2612 if (ret < -1) {
2613 fprintf(stderr, "Failure while decoding file\n");
2614 goto cleanup;
2615 }
2616 }
2617
2618 round_buffer_flush(all_stats, -1);
2619 output_stats(stdout, all_stats, sparse);
2620 if (info->split_tag)
2621 output_split_stats(split_hash, bam_fname, sparse);
2622
2623 ret = 0;
2624cleanup:
2625 bam_destroy1(bam_line);
2626 sam_hdr_destroy(info->sam_header);
2627 sam_global_args_free(&ga);
2628
2629cleanup_read_pairs:
2630 cleanup_overlaps(read_pairs, INT64_MAX);
2631cleanup_split_hash:
2632 destroy_split_stats(split_hash);
2633cleanup_all_stats:
2634 cleanup_stats(all_stats);
2635 cleanup_stats_info(info);
2636
2637 return ret;
2638}