Source code

Download source archive ↓

upstream-dependencies/samtools/stats.c

1/*  stats.c -- This is the former bamcheck integrated into samtools/htslib.
2
3    Copyright (C) 2012-2024 Genome Research Ltd.
4
5    Author: Petr Danecek <pd3@sanger.ac.uk>
6    Author: Sam Nicholls <sam@samnicholls.net>
7
8Permission is hereby granted, free of charge, to any person obtaining a copy
9of this software and associated documentation files (the "Software"), to deal
10in the Software without restriction, including without limitation the rights
11to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12copies of the Software, and to permit persons to whom the Software is
13furnished to do so, subject to the following conditions:
14
15The above copyright notice and this permission notice shall be included in
16all copies or substantial portions of the Software.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE.  */
25
26/*  Assumptions, approximations and other issues:
27        - GC-depth graph does not split reads, the starting position determines which bin is incremented.
28            There are small overlaps between bins (max readlen-1). However, the bins are big (20k).
29        - coverage distribution ignores softclips and deletions
30        - some stats require sorted BAMs
31        - GC content graph can have an untidy, step-like pattern when BAM contains multiple read lengths.
32        - 'bases mapped' (stats->nbases_mapped) is calculated from read lengths given by BAM (core.l_qseq)
33        - With the -t option, the whole reads are used. Except for the number of mapped bases (cigar)
34            counts, no splicing is done, no indels or soft clips are considered, even small overlap is
35            good enough to include the read in the stats.
36        - GC content of reads not calculated for "=" sequences
37
38*/
39
40#include <config.h>
41
42#include <unistd.h> // for isatty()
43#include <stdio.h>
44#include <stdlib.h>
45#include <stdarg.h>
46#include <string.h>
47#include <math.h>
48#include <ctype.h>
49#include <inttypes.h>
50#include <getopt.h>
51#include <errno.h>
52#include <assert.h>
53#include <zlib.h>   // for crc32
54#include <htslib/faidx.h>
55#include <htslib/sam.h>
56#include <htslib/hts.h>
57#include <htslib/hts_defs.h>
58#include "samtools.h"
59#include <htslib/khash.h>
60#include <htslib/kstring.h>
61#include "stats_isize.h"
62#include "sam_opts.h"
63#include "bedidx.h"
64
65#define BWA_MIN_RDLEN 35
66#define DEFAULT_CHUNK_NO 8
67#define DEFAULT_PAIR_MAX 10000
68#define ERROR_LIMIT 200
69// From the spec
70// If 0x4 is set, no assumptions can be made about RNAME, POS, CIGAR, MAPQ, bits 0x2, 0x10, 0x100 and 0x800, and the bit 0x20 of the previous read in the template.
71#define IS_PAIRED(bam) ((bam)->core.flag&BAM_FPAIRED)
72#define IS_PAIRED_AND_MAPPED(bam) (((bam)->core.flag&BAM_FPAIRED) && !((bam)->core.flag&BAM_FUNMAP) && !((bam)->core.flag&BAM_FMUNMAP))
73#define IS_PROPERLYPAIRED(bam) (((bam)->core.flag&(BAM_FPAIRED|BAM_FPROPER_PAIR)) == (BAM_FPAIRED|BAM_FPROPER_PAIR) && !((bam)->core.flag&BAM_FUNMAP))
74#define IS_UNMAPPED(bam) ((bam)->core.flag&BAM_FUNMAP)
75#define IS_REVERSE(bam) ((bam)->core.flag&BAM_FREVERSE)
76#define IS_MATE_REVERSE(bam) ((bam)->core.flag&BAM_FMREVERSE)
77#define IS_READ1(bam) ((bam)->core.flag&BAM_FREAD1)
78#define IS_READ2(bam) ((bam)->core.flag&BAM_FREAD2)
79#define IS_DUP(bam) ((bam)->core.flag&BAM_FDUP)
80#define IS_ORIGINAL(bam) (((bam)->core.flag&(BAM_FSECONDARY|BAM_FSUPPLEMENTARY)) == 0)
81
82#define READ_ORDER_NONE 0
83#define READ_ORDER_FIRST 1
84#define READ_ORDER_LAST 2
85#define READ_ORDER_MIDDLE 3
86
87#define REG_INC 100
88#define POS_INC 1000
89
90// The GC-depth graph works as follows: split the reference sequence into
91// segments and calculate GC content and depth in each bin. Then sort
92// these segments by their GC and plot the depth distribution by means
93// of 10th, 25th, etc. depth percentiles.
94typedef struct
95{
96    float gc;
97    uint32_t depth;
98}
99gc_depth_t;
100
101// For coverage distribution, a simple pileup
102typedef struct
103{
104    hts_pos_t pos;
105    int size, start;
106    int *buffer;
107}
108round_buffer_t;
109
110typedef struct
111{
112    int npos, mpos, cpos;
113    hts_pair_pos_t *pos;
114}
115regions_t;
116
117typedef struct
118{
119    uint64_t a;
120    uint64_t c;
121    uint64_t g;
122    uint64_t t;
123    uint64_t n;
124    uint64_t other;
125}
126acgtno_count_t;
127
128typedef struct
129{
130    char tag_name[3];
131    char qual_name[3];
132    uint32_t nbases;
133    int32_t tag_sep;    // Index of the separator (if present)
134    int32_t max_qual;
135    uint32_t offset;    // Where the tag stats info is located in the allocated memory
136}
137barcode_info_t;
138
139typedef struct
140{
141    // Auxiliary data
142    int flag_require, flag_filter;
143    faidx_t *fai;                   // Reference sequence for GC-depth graph
144    int argc;                       // Command line arguments to be printed on the output
145    char **argv;
146    int gcd_bin_size;           // The size of GC-depth bin
147    int nisize;         // The maximum insert size that the allocated array can hold - 0 indicates no limit
148    int trim_qual;      // bwa trim quality
149    float isize_main_bulk;  // There are always some unrealistically big insert sizes, report only the main part
150    int cov_min,cov_max,cov_step;   // Minimum, maximum coverage and size of the coverage bins
151    samFile* sam;
152    sam_hdr_t* sam_header;
153
154    // Filters
155    int filter_readlen;
156
157    // Misc
158    char *split_tag;      // Tag on which to perform stats splitting
159    char *split_prefix;   // Path or string prefix for filenames created when splitting
160    int remove_overlaps;
161    int cov_threshold;
162}
163stats_info_t;
164
165typedef struct
166{
167    // Dimensions of the quality histogram holder (quals_1st,quals_2nd), GC content holder (gc_1st,gc_2nd),
168    //  insert size histogram holder
169    int nquals;         // The number of quality bins
170    int nbases;         // The maximum sequence length the allocated array can hold
171    int ngc;            // The size of gc_1st and gc_2nd
172    int nindels;        // The maximum indel length for indel distribution
173
174    // Arrays for the histogram data
175    uint64_t *quals_1st, *quals_2nd;
176    uint64_t *gc_1st, *gc_2nd;
177    acgtno_count_t *acgtno_cycles_1st, *acgtno_cycles_2nd;
178    acgtno_count_t *acgtno_revcomp;
179    uint64_t *read_lengths, *read_lengths_1st, *read_lengths_2nd;
180    uint64_t *insertions, *deletions;
181    uint64_t *ins_cycles_1st, *ins_cycles_2nd, *del_cycles_1st, *del_cycles_2nd;
182    isize_t *isize;
183    uint64_t* mapping_qualities;
184
185    // The extremes encountered
186    int max_len;            // Maximum read length
187    int max_len_1st;        // Maximum read length for forward reads
188    int max_len_2nd;        // Maximum read length for reverse reads
189    int max_qual;           // Maximum quality
190    int is_sorted;
191
192    // Summary numbers
193    uint64_t total_len;
194    uint64_t total_len_1st;
195    uint64_t total_len_2nd;
196    uint64_t total_len_dup;
197    uint64_t nreads_1st;
198    uint64_t nreads_2nd;
199    uint64_t nreads_other;
200    uint64_t nreads_filtered;
201    uint64_t nreads_dup;
202    uint64_t nreads_unmapped;
203    uint64_t nreads_single_mapped;
204    uint64_t nreads_paired_and_mapped;
205    uint64_t nreads_properly_paired;
206    uint64_t nreads_paired_tech;
207    uint64_t nreads_anomalous;
208    uint64_t nreads_mq0;
209    uint64_t nbases_mapped;
210    uint64_t nbases_mapped_cigar;
211    uint64_t nbases_trimmed;  // bwa trimmed bases
212    uint64_t nmismatches;
213    uint64_t nreads_QCfailed, nreads_secondary, nreads_supplementary;
214    struct {
215        uint32_t names, reads, quals;
216    } checksum;
217
218    // GC-depth related data
219    uint32_t ngcd, igcd;        // The maximum number of GC depth bins and index of the current bin
220    gc_depth_t *gcd;            // The GC-depth bins holder
221    int32_t tid;                // Position of the current bin
222    hts_pos_t gcd_pos, pos;     // Position of the last read
223
224    // Coverage distribution related data
225    int ncov;                       // The number of coverage bins
226    uint64_t *cov;                  // The coverage frequencies
227    round_buffer_t cov_rbuf;        // Pileup round buffer
228
229    // Mismatches by read cycle
230    uint8_t *rseq_buf;              // A buffer for reference sequence to check the mismatches against
231    int mrseq_buf;                  // The size of the buffer
232    hts_pos_t rseq_pos;             // The coordinate of the first base in the buffer
233    int64_t nrseq_buf;              // The used part of the buffer
234    uint64_t *mpc_buf;              // Mismatches per cycle
235
236    // Target regions
237    int nregions;
238    hts_pos_t reg_from, reg_to;
239    regions_t *regions;
240
241    // Auxiliary data
242    double sum_qual;                // For calculating average quality value
243    void *rg_hash;                  // Read groups to include, the array is null-terminated
244
245    // Split
246    char* split_name;
247
248    stats_info_t* info;             // Pointer to options and settings struct
249    hts_pair_pos_t *chunks;
250    uint32_t nchunks;
251
252    uint32_t pair_count;          // Number of active pairs in the pairing hash table
253    uint64_t target_count;        // Number of bases covered by the target file
254    uint32_t last_pair_tid;
255    uint32_t last_read_flush;
256
257    // Barcode statistics
258    acgtno_count_t *acgtno_barcode;
259    uint64_t *quals_barcode;
260    barcode_info_t *tags_barcode;
261    uint32_t ntags;
262    uint32_t error_number;
263}
264stats_t;
265KHASH_MAP_INIT_STR(c2stats, stats_t*)
266
267typedef struct {
268    uint32_t first;     // 1 - first read, 2 - second read
269    uint32_t n, m;      // number of chunks, allocated chunks
270    hts_pair_pos_t *chunks;      // chunk array of size m
271} pair_t;
272KHASH_MAP_INIT_STR(qn2pair, pair_t*)
273
274KHASH_SET_INIT_STR(rg)
275
276
277static void HTS_NORETURN error(const char *format, ...);
278int is_in_regions(bam1_t *bam_line, stats_t *stats);
279void realloc_buffers(stats_t *stats, int seq_len);
280
281static int regions_lt(const void *r1, const void *r2) {
282    int64_t from_diff = ((hts_pair_pos_t *)r1)->beg - ((hts_pair_pos_t *)r2)->beg;
283    int64_t to_diff   = ((hts_pair_pos_t *)r1)->end - ((hts_pair_pos_t *)r2)->end;
284
285    return from_diff > 0 ? 1 : from_diff < 0 ? -1 : to_diff > 0 ? 1 : to_diff < 0 ? -1 : 0;
286}
287
288// Coverage distribution methods
289static inline int coverage_idx(int min, int max, int n, int step, int depth)
290{
291    if ( depth < min )
292        return 0;
293
294    if ( depth > max )
295        return n-1;
296
297    return 1 + (depth - min) / step;
298}
299
300static inline int round_buffer_lidx2ridx(int offset, int size, hts_pos_t refpos, hts_pos_t pos)
301{
302    return (offset + (pos-refpos) % size) % size;
303}
304
305void round_buffer_flush(stats_t *stats, hts_pos_t pos)
306{
307    int ibuf,idp;
308
309    if ( pos==stats->cov_rbuf.pos )
310        return;
311
312    hts_pos_t new_pos = pos;
313    if ( pos==-1 || pos - stats->cov_rbuf.pos >= stats->cov_rbuf.size )
314    {
315        // Flush the whole buffer, but in sequential order,
316        pos = stats->cov_rbuf.pos + stats->cov_rbuf.size - 1;
317    }
318
319    if ( pos < stats->cov_rbuf.pos )
320        error("Expected coordinates in ascending order, got %"PRIhts_pos" after %"PRIhts_pos"\n", pos, stats->cov_rbuf.pos);
321
322    int ifrom = stats->cov_rbuf.start;
323    int ito = round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos-1);
324    if ( ifrom>ito )
325    {
326        for (ibuf=ifrom; ibuf<stats->cov_rbuf.size; ibuf++)
327        {
328            if ( !stats->cov_rbuf.buffer[ibuf] )
329                continue;
330            idp = coverage_idx(stats->info->cov_min,stats->info->cov_max,stats->ncov,stats->info->cov_step,stats->cov_rbuf.buffer[ibuf]);
331            stats->cov[idp]++;
332            stats->cov_rbuf.buffer[ibuf] = 0;
333        }
334        ifrom = 0;
335    }
336    for (ibuf=ifrom; ibuf<=ito; ibuf++)
337    {
338        if ( !stats->cov_rbuf.buffer[ibuf] )
339            continue;
340        idp = coverage_idx(stats->info->cov_min,stats->info->cov_max,stats->ncov,stats->info->cov_step,stats->cov_rbuf.buffer[ibuf]);
341        stats->cov[idp]++;
342        stats->cov_rbuf.buffer[ibuf] = 0;
343    }
344    stats->cov_rbuf.start = (new_pos==-1) ? 0 : round_buffer_lidx2ridx(stats->cov_rbuf.start, stats->cov_rbuf.size, stats->cov_rbuf.pos, pos);
345    stats->cov_rbuf.pos   = new_pos;
346}
347
348/**
349 * [from, to) - 0 based half-open
350 */
351static void round_buffer_insert_read(round_buffer_t *rbuf, hts_pos_t from, hts_pos_t to)
352{
353    if ( to-from > rbuf->size )
354        error("The read length too big (%"PRIhts_pos"), please increase the buffer length (currently %d)\n", to-from, rbuf->size);
355    if ( from < rbuf->pos )
356        error("The reads are not sorted (%"PRIhts_pos" comes after %"PRIhts_pos").\n", from, rbuf->pos);
357
358    int ifrom, ito, ibuf;
359    ifrom = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, from);
360    ito   = round_buffer_lidx2ridx(rbuf->start, rbuf->size, rbuf->pos, to);
361    if ( ifrom>ito )
362    {
363        for (ibuf=ifrom; ibuf<rbuf->size; ibuf++)
364            rbuf->buffer[ibuf]++;
365        ifrom = 0;
366    }
367    for (ibuf=ifrom; ibuf<ito; ibuf++)
368        rbuf->buffer[ibuf]++;
369}
370
371// Calculate the number of bases in the read trimmed by BWA
372int bwa_trim_read(int trim_qual, uint8_t *quals, int len, int reverse)
373{
374    if ( len<BWA_MIN_RDLEN ) return 0;
375
376    // Although the name implies that the read cannot be trimmed to more than BWA_MIN_RDLEN,
377    //  the calculation can in fact trim it to (BWA_MIN_RDLEN-1). (bwa_trim_read in bwa/bwaseqio.c).
378    int max_trimmed = len - BWA_MIN_RDLEN + 1;
379    int l, sum=0, max_sum=0, max_l=0;
380
381    for (l=0; l<max_trimmed; l++)
382    {
383        sum += trim_qual - quals[ reverse ? l : len-1-l ];
384        if ( sum<0 ) break;
385        if ( sum>max_sum )
386        {
387            max_sum = sum;
388            // This is the correct way, but bwa clips from some reason one base less
389            // max_l   = l+1;
390            max_l   = l;
391        }
392    }
393    return max_l;
394}
395
396
397void count_indels(stats_t *stats,bam1_t *bam_line)
398{
399    int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
400    uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
401    int icig;
402    int icycle = 0;
403    int read_len = bam_line->core.l_qseq;
404    for (icig=0; icig<bam_line->core.n_cigar; icig++)
405    {
406        int cig  = bam_cigar_op(bam_get_cigar(bam_line)[icig]);
407        int ncig = bam_cigar_oplen(bam_get_cigar(bam_line)[icig]);
408        if ( !ncig ) continue;  // curiously, this can happen: 0D
409
410        if ( cig==BAM_CINS )
411        {
412            int idx = is_fwd ? icycle : read_len-icycle-ncig;
413            if ( idx<0 )
414                error("FIXME: read_len=%d vs icycle=%d\n", read_len,icycle);
415            if ( idx >= stats->nbases || idx<0 ) error("FIXME: %d vs %d, %s:%"PRIhts_pos" %s\n", idx, stats->nbases, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
416            if ( order == READ_ORDER_FIRST )
417                stats->ins_cycles_1st[idx]++;
418            if ( order == READ_ORDER_LAST )
419                stats->ins_cycles_2nd[idx]++;
420            icycle += ncig;
421            if ( ncig<=stats->nindels )
422                stats->insertions[ncig-1]++;
423            continue;
424        }
425        if ( cig==BAM_CDEL )
426        {
427            int idx = is_fwd ? icycle-1 : read_len-icycle-1;
428            if ( idx<0 ) continue;  // discard meaningless deletions
429            if ( idx >= stats->nbases ) error("FIXME: %d vs %d\n", idx,stats->nbases);
430            if ( order == READ_ORDER_FIRST )
431                stats->del_cycles_1st[idx]++;
432            if ( order == READ_ORDER_LAST )
433                stats->del_cycles_2nd[idx]++;
434            if ( ncig<=stats->nindels )
435                stats->deletions[ncig-1]++;
436            continue;
437        }
438        if ( cig!=BAM_CREF_SKIP && cig!=BAM_CHARD_CLIP && cig!=BAM_CPAD )
439            icycle += ncig;
440    }
441}
442
443int unclipped_length(bam1_t *bam_line)
444{
445    int icig, read_len = bam_line->core.l_qseq;
446    for (icig=0; icig<bam_line->core.n_cigar; icig++)
447    {
448        int cig = bam_cigar_op(bam_get_cigar(bam_line)[icig]);
449        if ( cig==BAM_CHARD_CLIP )
450            read_len += bam_cigar_oplen(bam_get_cigar(bam_line)[icig]);
451    }
452    return read_len;
453}
454
455void count_mismatches_per_cycle(stats_t *stats, bam1_t *bam_line, int read_len)
456{
457    int is_fwd = IS_REVERSE(bam_line) ? 0 : 1;
458    int icig, iread=0, icycle=0;
459    hts_pos_t iref = bam_line->core.pos - stats->rseq_pos;
460    uint8_t *read  = bam_get_seq(bam_line);
461    uint8_t *quals = bam_get_qual(bam_line);
462    uint64_t *mpc_buf = stats->mpc_buf;
463    for (icig=0; icig<bam_line->core.n_cigar; icig++)
464    {
465        int cig  = bam_cigar_op(bam_get_cigar(bam_line)[icig]);
466        int ncig = bam_cigar_oplen(bam_get_cigar(bam_line)[icig]);
467        if ( cig==BAM_CINS )
468        {
469            iread  += ncig;
470            icycle += ncig;
471            continue;
472        }
473        if ( cig==BAM_CDEL )
474        {
475            iref += ncig;
476            continue;
477        }
478        if ( cig==BAM_CSOFT_CLIP )
479        {
480            icycle += ncig;
481            // Soft-clips are present in the sequence, but the position of the read marks a start of the sequence after clipping
482            //   iref += ncig;
483            iread  += ncig;
484            continue;
485        }
486        if ( cig==BAM_CHARD_CLIP )
487        {
488            icycle += ncig;
489            continue;
490        }
491        // Ignore H and N CIGARs. The letter are inserted e.g. by TopHat and often require very large
492        //  chunk of refseq in memory. Not very frequent and not noticeable in the stats.
493        if ( cig==BAM_CREF_SKIP || cig==BAM_CHARD_CLIP || cig==BAM_CPAD ) continue;
494        if ( cig!=BAM_CMATCH && cig!=BAM_CEQUAL && cig!=BAM_CDIFF ) // not relying on precalculated diffs
495            error("TODO: cigar %d, %s:%"PRIhts_pos" %s\n", cig, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
496
497        if ( ncig+iref > stats->nrseq_buf )
498            error("FIXME: %d+%"PRIhts_pos" > %"PRId64", %s, %s:%"PRIhts_pos"\n", ncig, iref, stats->nrseq_buf, bam_get_qname(bam_line), sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1);
499
500        int im;
501        for (im=0; im<ncig; im++)
502        {
503            uint8_t cread = bam_seqi(read,iread);
504            uint8_t cref  = stats->rseq_buf[iref];
505
506            // ---------------15
507            // =ACMGRSVTWYHKDBN
508            if ( cread==15 )
509            {
510                int idx = is_fwd ? icycle : read_len-icycle-1;
511                if ( idx>stats->max_len )
512                    error("mpc: %d>%d\n",idx,stats->max_len);
513                idx = idx*stats->nquals;
514                if ( idx>=stats->nquals*stats->nbases )
515                    error("FIXME: mpc_buf overflow\n");
516                mpc_buf[idx]++;
517            }
518            else if ( cref && cread && cref!=cread )
519            {
520                uint8_t qual = quals[iread] + 1;
521                if ( qual>=stats->nquals )
522                    error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
523
524                int idx = is_fwd ? icycle : read_len-icycle-1;
525                if ( idx>stats->max_len )
526                    error("mpc: %d>%d (%s %"PRIhts_pos" %s)\n", idx, stats->max_len, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
527
528                idx = idx*stats->nquals + qual;
529                if ( idx>=stats->nquals*stats->nbases )
530                    error("FIXME: mpc_buf overflow\n");
531                mpc_buf[idx]++;
532            }
533
534            iref++;
535            iread++;
536            icycle++;
537        }
538    }
539}
540
541void read_ref_seq(stats_t *stats, int32_t tid, hts_pos_t pos, hts_pos_t end)
542{
543    int i;
544    hts_pos_t fai_ref_len;
545    char *fai_ref;
546
547    if (end < pos+stats->mrseq_buf-1)
548        end = pos+stats->mrseq_buf-1;
549    else if (stats->mrseq_buf < end - pos) {
550        size_t sz = end - pos;
551        uint8_t *new_rseq = realloc(stats->rseq_buf, sz);
552        if (!new_rseq)
553            error("Couldn't expand the reference sequence buffer\n");
554        stats->rseq_buf = new_rseq;
555        stats->mrseq_buf = sz;
556    }
557
558    fai_ref = faidx_fetch_seq64(stats->info->fai, sam_hdr_tid2name(stats->info->sam_header, tid), pos, pos+stats->mrseq_buf-1, &fai_ref_len);
559    if ( fai_ref_len < 0 ) error("Failed to fetch the sequence \"%s\"\n", sam_hdr_tid2name(stats->info->sam_header, tid));
560
561    uint8_t *ptr = stats->rseq_buf;
562    for (i=0; i<fai_ref_len; i++)
563    {
564        // Conversion between uint8_t coding and ACGT
565        //      -12-4---8-------
566        //      =ACMGRSVTWYHKDBN
567        switch (fai_ref[i])
568        {
569            case 'A':
570            case 'a': *ptr = 1; break;
571            case 'C':
572            case 'c': *ptr = 2; break;
573            case 'G':
574            case 'g': *ptr = 4; break;
575            case 'T':
576            case 't': *ptr = 8; break;
577            default:  *ptr = 0; break;
578        }
579        ptr++;
580    }
581    free(fai_ref);
582
583    if ( fai_ref_len < stats->mrseq_buf ) memset(ptr,0, stats->mrseq_buf - fai_ref_len);
584    stats->nrseq_buf = fai_ref_len;
585    stats->rseq_pos  = pos;
586    stats->tid       = tid;
587}
588
589float fai_gc_content(stats_t *stats, hts_pos_t pos, int len)
590{
591    uint32_t gc,count,c;
592    hts_pos_t i = pos - stats->rseq_pos, ito = i + len;
593    assert( i>=0 );
594
595    if (  ito > stats->nrseq_buf ) ito = stats->nrseq_buf;
596
597    // Count GC content
598    gc = count = 0;
599    for (; i<ito; i++)
600    {
601        c = stats->rseq_buf[i];
602        if ( c==2 || c==4 )
603        {
604            gc++;
605            count++;
606        }
607        else if ( c==1 || c==8 )
608            count++;
609    }
610    return count ? (float)gc/count : 0;
611}
612
613void realloc_rseq_buffer(stats_t *stats)
614{
615    int n = stats->nbases*10;
616    if ( stats->info->gcd_bin_size > n ) n = stats->info->gcd_bin_size;
617    if ( stats->mrseq_buf<n )
618    {
619        stats->rseq_buf = realloc(stats->rseq_buf,sizeof(uint8_t)*n);
620        if (!stats->rseq_buf) {
621            error("Could not reallocate reference sequence buffer");
622        }
623        stats->mrseq_buf = n;
624    }
625}
626
627void realloc_gcd_buffer(stats_t *stats, int seq_len)
628{
629    hts_expand0(gc_depth_t,stats->igcd+1,stats->ngcd,stats->gcd);
630    realloc_rseq_buffer(stats);
631}
632
633void realloc_buffers(stats_t *stats, int seq_len)
634{
635    int n = 2*(1 + seq_len - stats->nbases) + stats->nbases;
636
637    stats->quals_1st = realloc(stats->quals_1st, n*stats->nquals*sizeof(uint64_t));
638    if ( !stats->quals_1st )
639        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
640    memset(stats->quals_1st + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
641
642    stats->quals_2nd = realloc(stats->quals_2nd, n*stats->nquals*sizeof(uint64_t));
643    if ( !stats->quals_2nd )
644        error("Could not realloc buffers, the sequence too long: %d (2x%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
645    memset(stats->quals_2nd + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
646
647    if ( stats->mpc_buf )
648    {
649        stats->mpc_buf = realloc(stats->mpc_buf, n*stats->nquals*sizeof(uint64_t));
650        if ( !stats->mpc_buf )
651            error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*stats->nquals*sizeof(uint64_t));
652        memset(stats->mpc_buf + stats->nbases*stats->nquals, 0, (n-stats->nbases)*stats->nquals*sizeof(uint64_t));
653    }
654
655    stats->acgtno_cycles_1st = realloc(stats->acgtno_cycles_1st, n*sizeof(acgtno_count_t));
656    if ( !stats->acgtno_cycles_1st )
657        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len, n*sizeof(acgtno_count_t));
658    memset(stats->acgtno_cycles_1st + stats->nbases, 0, (n-stats->nbases)*sizeof(acgtno_count_t));
659
660    stats->acgtno_cycles_2nd = realloc(stats->acgtno_cycles_2nd, n*sizeof(acgtno_count_t));
661    if ( !stats->acgtno_cycles_2nd )
662        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len, n*sizeof(acgtno_count_t));
663    memset(stats->acgtno_cycles_2nd + stats->nbases, 0, (n-stats->nbases)*sizeof(acgtno_count_t));
664
665    stats->acgtno_revcomp = realloc(stats->acgtno_revcomp, n*sizeof(acgtno_count_t));
666    if ( !stats->acgtno_revcomp )
667        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len, n*sizeof(acgtno_count_t));
668    memset(stats->acgtno_revcomp + stats->nbases, 0, (n-stats->nbases)*sizeof(acgtno_count_t));
669
670    stats->read_lengths = realloc(stats->read_lengths, n*sizeof(uint64_t));
671    if ( !stats->read_lengths )
672        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
673    memset(stats->read_lengths + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
674
675    stats->read_lengths_1st = realloc(stats->read_lengths_1st, n*sizeof(uint64_t));
676    if ( !stats->read_lengths_1st )
677        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
678    memset(stats->read_lengths_1st + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
679
680    stats->read_lengths_2nd = realloc(stats->read_lengths_2nd, n*sizeof(uint64_t));
681    if ( !stats->read_lengths_2nd )
682        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
683    memset(stats->read_lengths_2nd + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
684
685    stats->insertions = realloc(stats->insertions, n*sizeof(uint64_t));
686    if ( !stats->insertions )
687        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
688    memset(stats->insertions + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
689
690    stats->deletions = realloc(stats->deletions, n*sizeof(uint64_t));
691    if ( !stats->deletions )
692        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,n*sizeof(uint64_t));
693    memset(stats->deletions + stats->nbases, 0, (n-stats->nbases)*sizeof(uint64_t));
694
695    stats->ins_cycles_1st = realloc(stats->ins_cycles_1st, (n+1)*sizeof(uint64_t));
696    if ( !stats->ins_cycles_1st )
697        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
698    memset(stats->ins_cycles_1st + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
699
700    stats->ins_cycles_2nd = realloc(stats->ins_cycles_2nd, (n+1)*sizeof(uint64_t));
701    if ( !stats->ins_cycles_2nd )
702        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
703    memset(stats->ins_cycles_2nd + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
704
705    stats->del_cycles_1st = realloc(stats->del_cycles_1st, (n+1)*sizeof(uint64_t));
706    if ( !stats->del_cycles_1st )
707        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
708    memset(stats->del_cycles_1st + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
709
710    stats->del_cycles_2nd = realloc(stats->del_cycles_2nd, (n+1)*sizeof(uint64_t));
711    if ( !stats->del_cycles_2nd )
712        error("Could not realloc buffers, the sequence too long: %d (%ld)\n", seq_len,(n+1)*sizeof(uint64_t));
713    memset(stats->del_cycles_2nd + stats->nbases + 1, 0, (n-stats->nbases)*sizeof(uint64_t));
714
715    stats->nbases = n;
716
717    // Realloc the coverage distribution buffer
718    int *rbuffer = calloc(sizeof(int),seq_len*5);
719    if (!rbuffer) {
720        error("Could not allocate coverage distribution buffer");
721    }
722    n = stats->cov_rbuf.size-stats->cov_rbuf.start;
723    memcpy(rbuffer,stats->cov_rbuf.buffer+stats->cov_rbuf.start,n);
724    if ( stats->cov_rbuf.start>1 )
725        memcpy(rbuffer+n,stats->cov_rbuf.buffer,stats->cov_rbuf.start);
726    stats->cov_rbuf.start = 0;
727    free(stats->cov_rbuf.buffer);
728    stats->cov_rbuf.buffer = rbuffer;
729    stats->cov_rbuf.size = seq_len*5;
730
731    realloc_rseq_buffer(stats);
732}
733
734void update_checksum(bam1_t *bam_line, stats_t *stats)
735{
736    uint8_t *name = (uint8_t*) bam_get_qname(bam_line);
737    int len = 0;
738    while ( name[len] ) len++;
739    stats->checksum.names +=  crc32(0L, name, len);
740
741    int seq_len = bam_line->core.l_qseq;
742    if ( !seq_len ) return;
743
744    uint8_t *seq = bam_get_seq(bam_line);
745    stats->checksum.reads += crc32(0L, seq, (seq_len+1)/2);
746
747    uint8_t *qual = bam_get_qual(bam_line);
748    stats->checksum.quals += crc32(0L, qual, (seq_len+1)/2);
749}
750
751// Collect statistics about the barcode tags specified by init_barcode_tags method
752static void collect_barcode_stats(bam1_t* bam_line, stats_t* stats) {
753    uint32_t nbases, tag, i;
754    acgtno_count_t *acgtno;
755    uint64_t *quals;
756    int32_t *separator, *maxqual;
757
758    for (tag = 0; tag < stats->ntags; tag++) {
759        const char *barcode_tag = stats->tags_barcode[tag].tag_name, *qual_tag = stats->tags_barcode[tag].qual_name;
760        uint8_t* bc = bam_aux_get(bam_line, barcode_tag);
761        if (!bc)
762            continue;
763
764        char* barcode = bam_aux2Z(bc);
765        if (!barcode)
766            continue;
767
768        uint32_t barcode_len = strlen(barcode);
769        if (!barcode_len) {
770            continue;        //consider 0 size barcode same as no barcode - avoids issues with realloc below
771        }
772        if (!stats->tags_barcode[tag].nbases) { // tag seen for the first time
773            uint32_t offset = 0;
774            for (i = 0; i < stats->ntags; i++)
775                offset += stats->tags_barcode[i].nbases;
776
777            stats->tags_barcode[tag].offset = offset;
778            stats->tags_barcode[tag].nbases = barcode_len;
779            stats->acgtno_barcode = realloc(stats->acgtno_barcode, (offset + barcode_len) * sizeof(acgtno_count_t));
780            stats->quals_barcode  = realloc(stats->quals_barcode, (offset + barcode_len) * stats->nquals * sizeof(uint64_t));
781
782            if (!stats->acgtno_barcode || !stats->quals_barcode)
783                error("Error allocating memory. Aborting!\n");
784
785            memset(stats->acgtno_barcode + offset, 0, barcode_len*sizeof(acgtno_count_t));
786            memset(stats->quals_barcode + offset*stats->nquals, 0, barcode_len*stats->nquals*sizeof(uint64_t));
787        }
788
789        nbases = stats->tags_barcode[tag].nbases;
790        if (barcode_len > nbases) {
791            fprintf(stderr, "Barcodes with tag %s differ in length at sequence '%s'\n", barcode_tag, bam_get_qname(bam_line));
792            continue;
793        }
794
795        acgtno = stats->acgtno_barcode + stats->tags_barcode[tag].offset;
796        quals = stats->quals_barcode + stats->tags_barcode[tag].offset*stats->nquals;
797        maxqual = &stats->tags_barcode[tag].max_qual;
798        separator = &stats->tags_barcode[tag].tag_sep;
799        int error_flag = 0;
800
801        for (i = 0; i < barcode_len; i++) {
802            switch (barcode[i]) {
803            case 'A':
804                acgtno[i].a++;
805                break;
806            case 'C':
807                acgtno[i].c++;
808                break;
809            case 'G':
810                acgtno[i].g++;
811                break;
812            case 'T':
813                acgtno[i].t++;
814                break;
815            case 'N':
816                acgtno[i].n++;
817                break;
818            default:
819                if (*separator >= 0) {
820                    if (*separator != i) {
821                        if (stats->error_number < ERROR_LIMIT) {
822                            fprintf(stderr, "Barcode separator for tag %s is in a different position or wrong barcode content('%s') at sequence '%s'\n", barcode_tag, barcode, bam_get_qname(bam_line));
823                            stats->error_number++;
824                        }
825                        error_flag = 1;
826                    }
827                } else {
828                    *separator = i;
829                }
830            }
831
832            /* don't process the rest of the tag bases */
833            if (error_flag)
834                break;
835        }
836
837        /* skip to the next tag */
838        if (error_flag)
839            continue;
840
841        uint8_t* qt = bam_aux_get(bam_line, qual_tag);
842        if (!qt)
843            continue;
844
845        char* barqual = bam_aux2Z(qt);
846        if (!barqual)
847            continue;
848
849        uint32_t barqual_len = strlen(barqual);
850        if (barqual_len == barcode_len) {
851            for (i = 0; i < barcode_len; i++) {
852                int32_t qual = (int32_t)barqual[i] - '!';  // Phred + 33
853                if (qual >= 0 && qual < stats->nquals) {
854                    quals[i * stats->nquals + qual]++;
855                    if (qual > *maxqual)
856                        *maxqual = qual;
857                }
858            }
859        } else {
860            if (stats->error_number++ < ERROR_LIMIT) {
861                fprintf(stderr, "%s length and %s length don't match for sequence '%s'\n", barcode_tag, qual_tag, bam_get_qname(bam_line));
862            }
863        }
864    }
865}
866
867// These stats should only be calculated for the original reads ignoring
868// supplementary artificial reads otherwise we'll accidentally double count
869void collect_orig_read_stats(bam1_t *bam_line, stats_t *stats, int* gc_count_out)
870{
871    int seq_len = bam_line->core.l_qseq;
872    stats->total_len += seq_len; // This ignores clipping so only count primary
873
874    if ( bam_line->core.flag & BAM_FQCFAIL ) stats->nreads_QCfailed++;
875    if ( bam_line->core.flag & BAM_FPAIRED ) stats->nreads_paired_tech++;
876
877    uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
878
879    // Count GC and ACGT per cycle. Note that cycle is approximate, clipping is ignored
880    uint8_t *seq  = bam_get_seq(bam_line);
881    int i, read_cycle, gc_count = 0, reverse = IS_REVERSE(bam_line);
882
883    acgtno_count_t *acgtno_cycles = (order == READ_ORDER_FIRST) ? stats->acgtno_cycles_1st : (order == READ_ORDER_LAST) ?  stats->acgtno_cycles_2nd : NULL ;
884    if (acgtno_cycles) {
885        for (i=0; i<seq_len; i++)
886        {
887            // Read cycle for current index
888            read_cycle = (reverse ? seq_len-i-1 : i);
889
890            // Conversion from uint8_t coding:
891            //      -12-4---8------5
892            //      =ACMGRSVTWYHKDBN
893            switch (bam_seqi(seq, i)) {
894            case 1:
895                acgtno_cycles[ read_cycle ].a++;
896                reverse ? stats->acgtno_revcomp[ read_cycle ].t++ : stats->acgtno_revcomp[ read_cycle ].a++;
897                break;
898            case 2:
899                acgtno_cycles[ read_cycle ].c++;
900                reverse ? stats->acgtno_revcomp[ read_cycle ].g++ : stats->acgtno_revcomp[ read_cycle ].c++;
901                gc_count++;
902                break;
903            case 4:
904                acgtno_cycles[ read_cycle ].g++;
905                reverse ? stats->acgtno_revcomp[ read_cycle ].c++ : stats->acgtno_revcomp[ read_cycle ].g++;
906                gc_count++;
907                break;
908            case 8:
909                reverse ? stats->acgtno_revcomp[ read_cycle ].a++ : stats->acgtno_revcomp[ read_cycle ].t++;
910                acgtno_cycles[ read_cycle ].t++;
911                break;
912            case 15:
913                acgtno_cycles[ read_cycle ].n++;
914                break;
915            default:
916                /*
917                 * count "=" sequences in "other" along
918                 * with MRSVWYHKDB ambiguity codes
919                 */
920                acgtno_cycles[ read_cycle ].other++;
921                break;
922            }
923        }
924    }
925    int gc_idx_min = gc_count*(stats->ngc-1)/seq_len;
926    int gc_idx_max = (gc_count+1)*(stats->ngc-1)/seq_len;
927    if ( gc_idx_max >= stats->ngc ) gc_idx_max = stats->ngc - 1;
928
929    // Determine which array (1st or 2nd read) will these stats go to,
930    //  trim low quality bases from end the same way BWA does,
931    //  fill GC histogram
932    uint64_t *quals = NULL;
933    uint8_t *bam_quals = bam_get_qual(bam_line);
934
935    switch (order) {
936    case READ_ORDER_FIRST:
937        quals = stats->quals_1st;
938        stats->nreads_1st++;
939        stats->total_len_1st += seq_len;
940        for (i=gc_idx_min; i<gc_idx_max; i++)
941            stats->gc_1st[i]++;
942        break;
943    case READ_ORDER_LAST:
944        quals  = stats->quals_2nd;
945        stats->nreads_2nd++;
946        stats->total_len_2nd += seq_len;
947        for (i=gc_idx_min; i<gc_idx_max; i++)
948            stats->gc_2nd[i]++;
949        break;
950    default:
951        stats->nreads_other++;
952    }
953    if ( stats->info->trim_qual>0 )
954        stats->nbases_trimmed += bwa_trim_read(stats->info->trim_qual, bam_quals, seq_len, reverse);
955
956    // Quality histogram and average quality. Clipping is neglected.
957    if (quals) {
958        for (i=0; i<seq_len; i++)
959        {
960            uint8_t qual = bam_quals[ reverse ? seq_len-i-1 : i];
961            if ( qual>=stats->nquals )
962                error("TODO: quality too high %d>=%d (%s %"PRIhts_pos" %s)\n", qual, stats->nquals, sam_hdr_tid2name(stats->info->sam_header, bam_line->core.tid), bam_line->core.pos+1, bam_get_qname(bam_line));
963            if ( qual>stats->max_qual )
964                stats->max_qual = qual;
965
966            quals[ i*stats->nquals+qual ]++;
967            stats->sum_qual += qual;
968        }
969    }
970
971    // Barcode statistics
972    if (order == READ_ORDER_FIRST) {
973        collect_barcode_stats(bam_line, stats);
974    }
975
976    // Look at the flags and increment appropriate counters (mapped, paired, etc)
977    if ( IS_UNMAPPED(bam_line) )
978    {
979        stats->nreads_unmapped++;
980    }
981    else
982    {
983        stats->nbases_mapped += seq_len; // This ignores clipping so only count primary
984
985        if ( !bam_line->core.qual )
986            stats->nreads_mq0++;
987        if ( !IS_PAIRED_AND_MAPPED(bam_line) )
988            stats->nreads_single_mapped++;
989        else
990        {
991            stats->nreads_paired_and_mapped++;
992
993            if (IS_PROPERLYPAIRED(bam_line)) stats->nreads_properly_paired++;
994
995            if ( bam_line->core.tid!=bam_line->core.mtid )
996                stats->nreads_anomalous++;
997        }
998    }
999    *gc_count_out = gc_count;
1000}
1001
1002static int cleanup_overlaps(khash_t(qn2pair) *read_pairs, hts_pos_t max) {
1003    if ( !read_pairs )
1004        return 0;
1005
1006    int count = 0;
1007    khint_t k;
1008    for (k = kh_begin(read_pairs); k < kh_end(read_pairs); k++) {
1009        if ( kh_exist(read_pairs, k) ) {
1010            char *key = (char *)kh_key(read_pairs, k);
1011            pair_t *val = kh_val(read_pairs, k);
1012            if ( val && val->chunks ) {
1013                if ( val->chunks[val->n-1].end < max ) {
1014                    free(val->chunks);
1015                    free(val);
1016                    free(key);
1017                    kh_del(qn2pair, read_pairs, k);
1018                    count++;
1019                }
1020            } else {
1021                free(key);
1022                kh_del(qn2pair, read_pairs, k);
1023                count++;
1024            }
1025        }
1026    }
1027    if ( max == INT64_MAX )
1028        kh_destroy(qn2pair, read_pairs);
1029
1030    return count;
1031}
1032
1033/**
1034 * [pmin, pmax) - 0 based half-open
1035 */
1036static void remove_overlaps(bam1_t *bam_line, khash_t(qn2pair) *read_pairs, stats_t *stats, hts_pos_t pmin, hts_pos_t pmax) {
1037    if ( !bam_line || !read_pairs || !stats )
1038        return;
1039
1040    uint32_t order = (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0);
1041    if ( !(bam_line->core.flag & BAM_FPAIRED) ||
1042         (bam_line->core.flag & BAM_FMUNMAP) ||
1043         (llabs(bam_line->core.isize) >= 2*bam_line->core.l_qseq) ||
1044         (order != READ_ORDER_FIRST && order != READ_ORDER_LAST) ) {
1045        if ( pmin >= 0 )
1046            round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
1047        return;
1048    }
1049
1050    char *qname = bam_get_qname(bam_line);
1051    if ( !qname ) {
1052        fprintf(stderr, "Error retrieving qname for line starting at pos %"PRIhts_pos"\n", bam_line->core.pos);
1053        return;
1054    }
1055
1056    khint_t k = kh_get(qn2pair, read_pairs, qname);
1057    if ( k == kh_end(read_pairs) ) { //first chunk from this template
1058        if ( pmin == -1 )
1059            return;
1060
1061        int ret;
1062        char *s = strdup(qname);
1063        if ( !s ) {
1064            fprintf(stderr, "Error allocating memory\n");
1065            return;
1066        }
1067
1068        k = kh_put(qn2pair, read_pairs, s, &ret);
1069        if ( -1 == ret ) {
1070            error("Error inserting read '%s' in pair hash table\n", qname);
1071        }
1072
1073        pair_t *pc = calloc(1, sizeof(pair_t));
1074        if ( !pc ) {
1075            fprintf(stderr, "Error allocating memory\n");
1076            return;
1077        }
1078
1079        pc->m = DEFAULT_CHUNK_NO;
1080        pc->chunks = calloc(pc->m, sizeof(hts_pair_pos_t));
1081        if ( !pc->chunks ) {
1082            fprintf(stderr, "Error allocating memory\n");
1083            free(pc);
1084            return;
1085        }
1086
1087        pc->chunks[0].beg = pmin;
1088        pc->chunks[0].end = pmax;
1089        pc->n = 1;
1090        pc->first = order;
1091
1092        kh_val(read_pairs, k) = pc;
1093        stats->pair_count++;
1094    } else { //template already present
1095        pair_t *pc = kh_val(read_pairs, k);
1096        if ( !pc ) {
1097            fprintf(stderr, "Invalid hash table entry\n");
1098            return;
1099        }
1100
1101        if ( order == pc->first ) { //chunk from an existing line
1102            if ( pmin == -1 )
1103                return;
1104
1105            if ( pc->n == pc->m ) {
1106                hts_pair_pos_t *tmp = realloc(pc->chunks, (pc->m<<1)*sizeof(hts_pair_pos_t));
1107                if ( !tmp ) {
1108                    fprintf(stderr, "Error allocating memory\n");
1109                    return;
1110                }
1111                pc->chunks = tmp;
1112                pc->m<<=1;
1113            }
1114
1115            pc->chunks[pc->n].beg = pmin;
1116            pc->chunks[pc->n].end = pmax;
1117            pc->n++;
1118        } else { //the other line, check for overlapping
1119            if ( pmin == -1 && kh_exist(read_pairs, k) ) { //job done, delete entry
1120                char *key = (char *)kh_key(read_pairs, k);
1121                pair_t *val = kh_val(read_pairs, k);
1122                if ( val) {
1123                    free(val->chunks);
1124                    free(val);
1125                }
1126                free(key);
1127                kh_del(qn2pair, read_pairs, k);
1128                stats->pair_count--;
1129                return;
1130            }
1131
1132            int i;
1133            for (i=0; i<pc->n; i++) {
1134                if ( pmin >= pc->chunks[i].end )
1135                    continue;
1136
1137                if ( pmax <= pc->chunks[i].beg ) //no overlap
1138                    break;
1139
1140                if ( pmin < pc->chunks[i].beg ) { //overlap at the beginning
1141                    round_buffer_insert_read(&(stats->cov_rbuf), pmin, pc->chunks[i].beg);
1142                    pmin = pc->chunks[i].beg;
1143                }
1144
1145                if ( pmax <= pc->chunks[i].end ) { //completely contained
1146                    stats->nbases_mapped_cigar -= (pmax - pmin);
1147                    return;
1148                } else {                           //overlap at the end
1149                    stats->nbases_mapped_cigar -= (pc->chunks[i].end - pmin);
1150                    pmin = pc->chunks[i].end;
1151                }
1152            }
1153        }
1154    }
1155    round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
1156}
1157
1158void collect_stats(bam1_t *bam_line, stats_t *stats, khash_t(qn2pair) *read_pairs)
1159{
1160    if ( !is_in_regions(bam_line,stats) )
1161        return;
1162    if ( stats->rg_hash )
1163    {
1164        const uint8_t *rg = bam_aux_get(bam_line, "RG");
1165        if ( !rg ) return;  // certain read groups were requested but this record has none
1166        khint_t k = kh_get(rg, stats->rg_hash, (const char*)(rg + 1));
1167        if ( k == kh_end((kh_rg_t *)stats->rg_hash) ) return;
1168    }
1169    if ( stats->info->flag_require && (bam_line->core.flag & stats->info->flag_require)!=stats->info->flag_require )
1170    {
1171        stats->nreads_filtered++;
1172        return;
1173    }
1174    if ( stats->info->flag_filter && (bam_line->core.flag & stats->info->flag_filter) )
1175    {
1176        stats->nreads_filtered++;
1177        return;
1178    }
1179    if ( stats->info->filter_readlen!=-1 && bam_line->core.l_qseq!=stats->info->filter_readlen )
1180        return;
1181
1182    update_checksum(bam_line, stats);
1183
1184    // Secondary reads don't count for most stats purposes
1185    if ( bam_line->core.flag & BAM_FSECONDARY )
1186    {
1187        stats->nreads_secondary++;
1188        return;
1189    }
1190
1191    if ( bam_line->core.flag & BAM_FSUPPLEMENTARY )
1192    {
1193        stats->nreads_supplementary++;
1194    }
1195
1196    // If line has no sequence cannot continue
1197    int seq_len = bam_line->core.l_qseq;
1198    if ( !seq_len ) return;
1199
1200    if ( IS_DUP(bam_line) )
1201    {
1202        stats->total_len_dup += seq_len;
1203        stats->nreads_dup++;
1204    }
1205
1206    uint32_t order = IS_PAIRED(bam_line) ? (IS_READ1(bam_line) ? READ_ORDER_FIRST : 0) + (IS_READ2(bam_line) ? READ_ORDER_LAST : 0) : READ_ORDER_FIRST;
1207
1208    int read_len = unclipped_length(bam_line);
1209    if ( read_len >= stats->nbases )
1210        realloc_buffers(stats,read_len);
1211    // Update max_len observed
1212    if ( stats->max_len<read_len )
1213        stats->max_len = read_len;
1214    if ( order == READ_ORDER_FIRST && stats->max_len_1st < read_len )
1215        stats->max_len_1st = read_len;
1216    if ( order == READ_ORDER_LAST && stats->max_len_2nd < read_len )
1217        stats->max_len_2nd = read_len;
1218    if ( ( bam_line->core.flag & (BAM_FUNMAP|BAM_FSECONDARY|BAM_FSUPPLEMENTARY|BAM_FQCFAIL|BAM_FDUP) ) == 0 )
1219        stats->mapping_qualities[bam_line->core.qual]++;
1220
1221    int i;
1222    int gc_count = 0;
1223
1224    // These stats should only be calculated for the original reads ignoring supplementary artificial reads
1225    // otherwise we'll accidentally double count
1226    if ( IS_ORIGINAL(bam_line) ) {
1227        stats->read_lengths[read_len]++;
1228        if ( order == READ_ORDER_FIRST ) stats->read_lengths_1st[read_len]++;
1229        if ( order == READ_ORDER_LAST ) stats->read_lengths_2nd[read_len]++;
1230        collect_orig_read_stats(bam_line, stats, &gc_count);
1231    }
1232
1233    // Look at the flags and increment appropriate counters (mapped, paired, etc)
1234    if ( IS_UNMAPPED(bam_line) ) return;
1235
1236    count_indels(stats, bam_line);
1237
1238    if ( IS_PAIRED_AND_MAPPED(bam_line) && IS_ORIGINAL(bam_line) )
1239    {
1240        // The insert size is tricky, because for long inserts the libraries are
1241        // prepared differently and the pairs point in other direction. BWA does
1242        // not set the paired flag for them.  Similar thing is true also for 454
1243        // reads. Mates mapped to different chromosomes have isize==0.
1244        int32_t isize = bam_line->core.isize;
1245        if ( isize<0 ) isize = -isize;
1246        if ( stats->info->nisize > 0 && isize > stats->info->nisize )
1247            isize = stats->info->nisize;
1248        if ( isize>0 || bam_line->core.tid==bam_line->core.mtid )
1249        {
1250            hts_pos_t pos_fst = bam_line->core.mpos - bam_line->core.pos;
1251            int is_fst  = IS_READ1(bam_line) ? 1 : -1;
1252            int is_fwd  = IS_REVERSE(bam_line) ? -1 : 1;
1253            int is_mfwd = IS_MATE_REVERSE(bam_line) ? -1 : 1;
1254
1255            if ( is_fwd*is_mfwd>0 )
1256                stats->isize->inc_other(stats->isize->data, isize);
1257            else if ( is_fst*pos_fst>=0 )
1258            {
1259                if ( is_fst*is_fwd>0 )
1260                    stats->isize->inc_inward(stats->isize->data, isize);
1261                else
1262                    stats->isize->inc_outward(stats->isize->data, isize);
1263            }
1264            else if ( is_fst*pos_fst<0 )
1265            {
1266                if ( is_fst*is_fwd>0 )
1267                    stats->isize->inc_outward(stats->isize->data, isize);
1268                else
1269                    stats->isize->inc_inward(stats->isize->data, isize);
1270            }
1271        }
1272    }
1273
1274    // Number of mismatches
1275    uint8_t *nm = bam_aux_get(bam_line,"NM");
1276    if (nm)
1277        stats->nmismatches += bam_aux2i(nm);
1278
1279    // Number of mapped bases from cigar
1280    if ( bam_line->core.n_cigar == 0)
1281        error("FIXME: mapped read with no cigar?\n");
1282    int readlen=seq_len;
1283    if ( stats->regions )
1284    {
1285        // Count only on-target bases
1286        hts_pos_t iref = bam_line->core.pos + 1;
1287        for (i=0; i<bam_line->core.n_cigar; i++)
1288        {
1289            int cig  = bam_cigar_op(bam_get_cigar(bam_line)[i]);
1290            int ncig = bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1291            if ( !ncig ) continue;  // curiously, this can happen: 0D
1292            if ( cig==BAM_CDEL ) readlen += ncig;
1293            else if ( cig==BAM_CMATCH || cig==BAM_CEQUAL || cig==BAM_CDIFF )
1294            {
1295                if ( iref < stats->reg_from ) ncig -= stats->reg_from-iref;
1296                else if ( iref+ncig-1 > stats->reg_to ) ncig -= iref+ncig-1 - stats->reg_to;
1297                if ( ncig<0 ) ncig = 0;
1298                stats->nbases_mapped_cigar += ncig;
1299                iref += bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1300            }
1301            else if ( cig==BAM_CINS )
1302            {
1303                iref += ncig;
1304                if ( iref>=stats->reg_from && iref<=stats->reg_to )
1305                    stats->nbases_mapped_cigar += ncig;
1306            }
1307        }
1308    }
1309    else
1310    {
1311        // Count the whole read
1312        for (i=0; i<bam_line->core.n_cigar; i++)
1313        {
1314            int cig  = bam_cigar_op(bam_get_cigar(bam_line)[i]);
1315            if ( cig==BAM_CMATCH || cig==BAM_CINS || cig==BAM_CEQUAL || cig==BAM_CDIFF )
1316                stats->nbases_mapped_cigar += bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1317            if ( cig==BAM_CDEL )
1318                readlen += bam_cigar_oplen(bam_get_cigar(bam_line)[i]);
1319        }
1320    }
1321
1322    if ( stats->tid==bam_line->core.tid && bam_line->core.pos<stats->pos )
1323        stats->is_sorted = 0;
1324    stats->pos = bam_line->core.pos;
1325
1326    if ( stats->is_sorted )
1327    {
1328        if ( stats->tid==-1 || stats->tid!=bam_line->core.tid ) {
1329            round_buffer_flush(stats, -1);
1330        }
1331
1332        //cleanup the pair hash table to free memory
1333        stats->last_read_flush++;
1334        if ( stats->pair_count > DEFAULT_PAIR_MAX && stats->last_read_flush > DEFAULT_PAIR_MAX) {
1335            stats->pair_count -= cleanup_overlaps(read_pairs, bam_line->core.pos);
1336            stats->last_read_flush = 0;
1337        }
1338
1339        if ( stats->last_pair_tid != bam_line->core.tid) {
1340            stats->pair_count -= cleanup_overlaps(read_pairs, INT64_MAX-1);
1341            stats->last_pair_tid = bam_line->core.tid;
1342            stats->last_read_flush = 0;
1343        }
1344
1345        // Mismatches per cycle and GC-depth graph. For simplicity, reads overlapping GCD bins
1346        //  are not splitted which results in up to seq_len-1 overlaps. The default bin size is
1347        //  20kbp, so the effect is negligible.
1348        if ( stats->info->fai )
1349        {
1350            hts_pos_t inc_ref = 0;
1351            int inc_gcd = 0;
1352            // First pass or new chromosome, or read goes beyond the rseq buffer
1353            if ( stats->rseq_pos==-1 || stats->tid != bam_line->core.tid
1354                 || stats->rseq_pos+stats->nrseq_buf < bam_line->core.pos+readlen) {
1355                inc_ref=bam_line->core.pos+readlen;
1356                inc_gcd=1;
1357            }
1358            // Read overlaps the next gcd bin
1359            else if ( stats->gcd_pos+stats->info->gcd_bin_size < bam_line->core.pos+readlen )
1360            {
1361                inc_gcd = 1;
1362                if ( stats->rseq_pos+stats->nrseq_buf < bam_line->core.pos+stats->info->gcd_bin_size ) inc_ref = bam_line->core.pos+stats->info->gcd_bin_size;
1363            }
1364            if ( inc_gcd )
1365            {
1366                stats->igcd++;
1367                if ( stats->igcd >= stats->ngcd )
1368                    realloc_gcd_buffer(stats, readlen);
1369                if ( inc_ref )
1370                    read_ref_seq(stats, bam_line->core.tid,
1371                                 bam_line->core.pos, inc_ref);
1372                stats->gcd_pos = bam_line->core.pos;
1373                stats->gcd[ stats->igcd ].gc = fai_gc_content(stats, stats->gcd_pos, stats->info->gcd_bin_size);
1374            }
1375
1376            count_mismatches_per_cycle(stats,bam_line,read_len);
1377        }
1378        // No reference and first pass, new chromosome or sequence going beyond the end of the gcd bin
1379        else if ( stats->gcd_pos==-1 || stats->tid != bam_line->core.tid || bam_line->core.pos - stats->gcd_pos > stats->info->gcd_bin_size )
1380        {
1381            // First pass or a new chromosome
1382            stats->tid     = bam_line->core.tid;
1383            stats->gcd_pos = bam_line->core.pos;
1384            stats->igcd++;
1385            if ( stats->igcd >= stats->ngcd )
1386                realloc_gcd_buffer(stats, readlen);
1387        }
1388        stats->gcd[ stats->igcd ].depth++;
1389        // When no reference sequence is given, approximate the GC from the read (much shorter window, but otherwise OK)
1390        if ( !stats->info->fai )
1391            stats->gcd[ stats->igcd ].gc += (float) gc_count / seq_len;
1392
1393        // Coverage distribution graph
1394        round_buffer_flush(stats,bam_line->core.pos);
1395        if ( stats->regions ) {
1396            hts_pos_t p = bam_line->core.pos, pnew, pmin = 0, pmax = 0;
1397            uint32_t j = 0;
1398            i = 0;
1399            while ( j < bam_line->core.n_cigar && i < stats->nchunks ) {
1400                int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
1401                int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
1402                switch(op) {
1403                case BAM_CMATCH:
1404                case BAM_CEQUAL:
1405                case BAM_CDIFF:
1406                    pmin = MAX(p, stats->chunks[i].beg-1); // 0 based
1407                    pmax = MIN(p+oplen, stats->chunks[i].end); // 1 based
1408                    if ( pmax > pmin ) {
1409                        if ( stats->info->remove_overlaps )
1410                            remove_overlaps(bam_line, read_pairs, stats, pmin, pmax);
1411                        else
1412                            round_buffer_insert_read(&(stats->cov_rbuf), pmin, pmax);
1413                    }
1414                    break;
1415                case BAM_CDEL:
1416                    break;
1417                }
1418                pnew = p + (bam_cigar_type(op)&2 ? oplen : 0); // consumes reference
1419
1420                if ( pnew >= stats->chunks[i].end ) {
1421                    // go to the next chunk
1422                    i++;
1423                } else {
1424                    // go to the next CIGAR op
1425                    j++;
1426                    p = pnew;
1427                }
1428            }
1429        } else {
1430            hts_pos_t p = bam_line->core.pos;
1431            uint32_t j;
1432            for (j = 0; j < bam_line->core.n_cigar; j++) {
1433                int op = bam_cigar_op(bam_get_cigar(bam_line)[j]);
1434                int oplen = bam_cigar_oplen(bam_get_cigar(bam_line)[j]);
1435                switch(op) {
1436                case BAM_CMATCH:
1437                case BAM_CEQUAL:
1438                case BAM_CDIFF:
1439                    if ( stats->info->remove_overlaps )
1440                        remove_overlaps(bam_line, read_pairs, stats, p, p+oplen);
1441                    else
1442                        round_buffer_insert_read(&(stats->cov_rbuf), p, p+oplen);
1443                    break;
1444                case BAM_CDEL:
1445                    break;
1446                }
1447                p += bam_cigar_type(op)&2 ? oplen : 0; // consumes reference
1448            }
1449        }
1450        if ( stats->info->remove_overlaps )
1451           remove_overlaps(bam_line, read_pairs, stats, -1LL, -1LL); //remove the line from the hash table
1452    }
1453}
1454
1455// Sort by GC and depth
1456#define GCD_t(x) ((gc_depth_t *)x)
1457static int gcd_cmp(const void *a, const void *b)
1458{
1459    if ( GCD_t(a)->gc < GCD_t(b)->gc ) return -1;
1460    if ( GCD_t(a)->gc > GCD_t(b)->gc ) return 1;
1461    if ( GCD_t(a)->depth < GCD_t(b)->depth ) return -1;
1462    if ( GCD_t(a)->depth > GCD_t(b)->depth ) return 1;
1463    return 0;
1464}
1465#undef GCD_t
1466
1467float gcd_percentile(gc_depth_t *gcd, int N, int p)
1468{
1469    float n,d;
1470    int k;
1471
1472    n = (float)p*(N+1)/100;
1473    k = n;
1474    if ( k<=0 )
1475        return gcd[0].depth;
1476    if ( k>=N )
1477        return gcd[N-1].depth;
1478
1479    d = n - k;
1480    return gcd[k-1].depth + d*(gcd[k].depth - gcd[k-1].depth);
1481}
1482
1483void output_stats(FILE *to, stats_t *stats, int sparse)
1484{
1485    // Calculate average insert size and standard deviation (from the main bulk data only)
1486    int isize, ibulk=0, icov, imapq=0;
1487    uint64_t nisize=0, nisize_inward=0, nisize_outward=0, nisize_other=0, cov_sum=0;
1488    double bulk=0, avg_isize=0, sd_isize=0;
1489    for (isize=0; isize<stats->isize->nitems(stats->isize->data); isize++)
1490    {
1491        // Each pair was counted twice
1492        stats->isize->set_inward(stats->isize->data, isize, stats->isize->inward(stats->isize->data, isize) * 0.5);
1493        stats->isize->set_outward(stats->isize->data, isize, stats->isize->outward(stats->isize->data, isize) * 0.5);
1494        stats->isize->set_other(stats->isize->data, isize, stats->isize->other(stats->isize->data, isize) * 0.5);
1495
1496        nisize_inward += stats->isize->inward(stats->isize->data, isize);
1497        nisize_outward += stats->isize->outward(stats->isize->data, isize);
1498        nisize_other += stats->isize->other(stats->isize->data, isize);
1499        nisize += stats->isize->inward(stats->isize->data, isize) + stats->isize->outward(stats->isize->data, isize) + stats->isize->other(stats->isize->data, isize);
1500    }
1501
1502    for (isize=0; isize<stats->isize->nitems(stats->isize->data); isize++)
1503    {
1504        uint64_t num = stats->isize->inward(stats->isize->data, isize) +  stats->isize->outward(stats->isize->data, isize) + stats->isize->other(stats->isize->data, isize);
1505        if (num > 0) ibulk = isize + 1;
1506        bulk += num;
1507        avg_isize += isize * (stats->isize->inward(stats->isize->data, isize) +  stats->isize->outward(stats->isize->data, isize) + stats->isize->other(stats->isize->data, isize));
1508
1509        if ( bulk/nisize > stats->info->isize_main_bulk )
1510        {
1511            ibulk  = isize+1;
1512            nisize = bulk;
1513            break;
1514        }
1515    }
1516    avg_isize /= nisize ? nisize : 1;
1517    for (isize=1; isize<ibulk; isize++)
1518        sd_isize += (stats->isize->inward(stats->isize->data, isize) + stats->isize->outward(stats->isize->data, isize) +stats->isize->other(stats->isize->data, isize)) * (isize-avg_isize)*(isize-avg_isize) / (nisize ? nisize : 1);
1519    sd_isize = sqrt(sd_isize);
1520
1521    fprintf(to, "# This file was produced by samtools stats (%s+htslib-%s) and can be plotted using plot-bamstats\n", samtools_version(), hts_version());
1522    if( stats->split_name != NULL ){
1523        fprintf(to, "# This file contains statistics only for reads with tag: %s=%s\n", stats->info->split_tag, stats->split_name);
1524    }
1525    else{
1526        fprintf(to, "# This file contains statistics for all reads.\n");
1527    }
1528    fprintf(to, "# The command line was:  %s",stats->info->argv[0]);
1529    int i;
1530    for (i=1; i<stats->info->argc; i++)
1531        fprintf(to, " %s", stats->info->argv[i]);
1532    fprintf(to, "\n");
1533    fprintf(to, "# CHK, Checksum\t[2]Read Names\t[3]Sequences\t[4]Qualities\n");
1534    fprintf(to, "# CHK, CRC32 of reads which passed filtering followed by addition (32bit overflow)\n");
1535    fprintf(to, "CHK\t%08x\t%08x\t%08x\n", stats->checksum.names,stats->checksum.reads,stats->checksum.quals);
1536    fprintf(to, "# Summary Numbers. Use `grep ^SN | cut -f 2-` to extract this part.\n");
1537    fprintf(to, "SN\traw total sequences:\t%ld\t# excluding supplementary and secondary reads\n", (long)(stats->nreads_filtered+stats->nreads_1st+stats->nreads_2nd+stats->nreads_other));  // not counting excluded seqs (and none of the below)
1538    fprintf(to, "SN\tfiltered sequences:\t%ld\n", (long)stats->nreads_filtered);
1539    fprintf(to, "SN\tsequences:\t%ld\n", (long)(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other));
1540    fprintf(to, "SN\tis sorted:\t%d\n", stats->is_sorted ? 1 : 0);
1541    fprintf(to, "SN\t1st fragments:\t%ld\n", (long)stats->nreads_1st);
1542    fprintf(to, "SN\tlast fragments:\t%ld\n", (long)stats->nreads_2nd);
1543    fprintf(to, "SN\treads mapped:\t%ld\n", (long)(stats->nreads_paired_and_mapped+stats->nreads_single_mapped));
1544    fprintf(to, "SN\treads mapped and paired:\t%ld\t# paired-end technology bit set + both mates mapped\n", (long)stats->nreads_paired_and_mapped);
1545    fprintf(to, "SN\treads unmapped:\t%ld\n", (long)stats->nreads_unmapped);
1546    fprintf(to, "SN\treads properly paired:\t%ld\t# proper-pair bit set\n", (long)stats->nreads_properly_paired);
1547    fprintf(to, "SN\treads paired:\t%ld\t# paired-end technology bit set\n", (long)stats->nreads_paired_tech);
1548    fprintf(to, "SN\treads duplicated:\t%ld\t# PCR or optical duplicate bit set\n", (long)stats->nreads_dup);
1549    fprintf(to, "SN\treads MQ0:\t%ld\t# mapped and MQ=0\n", (long)stats->nreads_mq0);
1550    fprintf(to, "SN\treads QC failed:\t%ld\n", (long)stats->nreads_QCfailed);
1551    fprintf(to, "SN\tnon-primary alignments:\t%ld\n", (long)stats->nreads_secondary);
1552    fprintf(to, "SN\tsupplementary alignments:\t%ld\n", (long)stats->nreads_supplementary);
1553    fprintf(to, "SN\ttotal length:\t%ld\t# ignores clipping\n", (long)stats->total_len);
1554    fprintf(to, "SN\ttotal first fragment length:\t%ld\t# ignores clipping\n", (long)stats->total_len_1st);
1555    fprintf(to, "SN\ttotal last fragment length:\t%ld\t# ignores clipping\n", (long)stats->total_len_2nd);
1556    fprintf(to, "SN\tbases mapped:\t%ld\t# ignores clipping\n", (long)stats->nbases_mapped);                 // the length of the whole read goes here, including soft-clips etc.
1557    fprintf(to, "SN\tbases mapped (cigar):\t%ld\t# more accurate\n", (long)stats->nbases_mapped_cigar);   // only matched and inserted bases are counted here
1558    fprintf(to, "SN\tbases trimmed:\t%ld\n", (long)stats->nbases_trimmed);
1559    fprintf(to, "SN\tbases duplicated:\t%ld\n", (long)stats->total_len_dup);
1560    fprintf(to, "SN\tmismatches:\t%ld\t# from NM fields\n", (long)stats->nmismatches);
1561    fprintf(to, "SN\terror rate:\t%e\t# mismatches / bases mapped (cigar)\n", stats->nbases_mapped_cigar ? (float)stats->nmismatches/stats->nbases_mapped_cigar : 0);
1562    float avg_read_length = (stats->nreads_1st +
1563                             stats->nreads_2nd +
1564                             stats->nreads_other)
1565        ? (float)stats->total_len / (stats->nreads_1st +
1566                                     stats->nreads_2nd +
1567                                     stats->nreads_other)
1568        : 0;
1569    fprintf(to, "SN\taverage length:\t%.0f\n", avg_read_length);
1570    fprintf(to, "SN\taverage first fragment length:\t%.0f\n", stats->nreads_1st? (float)stats->total_len_1st/stats->nreads_1st:0);
1571    fprintf(to, "SN\taverage last fragment length:\t%.0f\n", stats->nreads_2nd? (float)stats->total_len_2nd/stats->nreads_2nd:0);
1572    fprintf(to, "SN\tmaximum length:\t%d\n", stats->max_len);
1573    fprintf(to, "SN\tmaximum first fragment length:\t%d\n", stats->max_len_1st);
1574    fprintf(to, "SN\tmaximum last fragment length:\t%d\n", stats->max_len_2nd);
1575    fprintf(to, "SN\taverage quality:\t%.1f\n", stats->total_len?stats->sum_qual/stats->total_len:0);
1576    fprintf(to, "SN\tinsert size average:\t%.1f\n", avg_isize);
1577    fprintf(to, "SN\tinsert size standard deviation:\t%.1f\n", sd_isize);
1578    fprintf(to, "SN\tinward oriented pairs:\t%ld\n", (long)nisize_inward);
1579    fprintf(to, "SN\toutward oriented pairs:\t%ld\n", (long)nisize_outward);
1580    fprintf(to, "SN\tpairs with other orientation:\t%ld\n", (long)nisize_other);
1581    fprintf(to, "SN\tpairs on different chromosomes:\t%ld\n", (long)stats->nreads_anomalous/2);
1582    fprintf(to, "SN\tpercentage of properly paired reads (%%):\t%.1f\n", (stats->nreads_1st+stats->nreads_2nd+stats->nreads_other)? (float)(100*stats->nreads_properly_paired)/(stats->nreads_1st+stats->nreads_2nd+stats->nreads_other):0);
1583    if ( stats->target_count ) {
1584        fprintf(to, "SN\tbases inside the target:\t%" PRIu64 "\n", stats->target_count);
1585        for (icov=stats->info->cov_threshold+1; icov<stats->ncov; icov++)
1586            cov_sum += stats->cov[icov];
1587        fprintf(to, "SN\tpercentage of target genome with coverage > %d (%%):\t%.2f\n", stats->info->cov_threshold, (float)(100*cov_sum)/stats->target_count);
1588    }
1589
1590    int ibase,iqual;
1591    if ( stats->max_len<stats->nbases ) stats->max_len++;
1592    if ( stats->max_qual+1<stats->nquals ) stats->max_qual++;
1593    fprintf(to, "# First Fragment Qualities. Use `grep ^FFQ | cut -f 2-` to extract this part.\n");
1594    fprintf(to, "# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n");
1595    for (ibase=0; ibase<stats->max_len_1st; ibase++)
1596    {
1597        fprintf(to, "FFQ\t%d",ibase+1);
1598        for (iqual=0; iqual<=stats->max_qual; iqual++)
1599        {
1600            fprintf(to, "\t%ld", (long)stats->quals_1st[ibase*stats->nquals+iqual]);
1601        }
1602        fprintf(to, "\n");
1603    }
1604    fprintf(to, "# Last Fragment Qualities. Use `grep ^LFQ | cut -f 2-` to extract this part.\n");
1605    fprintf(to, "# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n");
1606    for (ibase=0; ibase<stats->max_len_2nd; ibase++)
1607    {
1608        fprintf(to, "LFQ\t%d",ibase+1);
1609        for (iqual=0; iqual<=stats->max_qual; iqual++)
1610        {
1611            fprintf(to, "\t%ld", (long)stats->quals_2nd[ibase*stats->nquals+iqual]);
1612        }
1613        fprintf(to, "\n");
1614    }
1615    if ( stats->mpc_buf )
1616    {
1617        fprintf(to, "# Mismatches per cycle and quality. Use `grep ^MPC | cut -f 2-` to extract this part.\n");
1618        fprintf(to, "# Columns correspond to qualities, rows to cycles. First column is the cycle number, second\n");
1619        fprintf(to, "# is the number of N's and the rest is the number of mismatches\n");
1620        for (ibase=0; ibase<stats->max_len; ibase++)
1621        {
1622            fprintf(to, "MPC\t%d",ibase+1);
1623            for (iqual=0; iqual<=stats->max_qual; iqual++)
1624            {
1625                fprintf(to, "\t%ld", (long)stats->mpc_buf[ibase*stats->nquals+iqual]);
1626            }
1627            fprintf(to, "\n");
1628        }
1629    }
1630    fprintf(to, "# GC Content of first fragments. Use `grep ^GCF | cut -f 2-` to extract this part.\n");
1631    int ibase_prev = 0;
1632    for (ibase=0; ibase<stats->ngc; ibase++)
1633    {
1634        if ( stats->gc_1st[ibase]==stats->gc_1st[ibase_prev] ) continue;
1635        fprintf(to, "GCF\t%.2f\t%ld\n", (ibase+ibase_prev)*0.5*100./(stats->ngc-1), (long)stats->gc_1st[ibase_prev]);
1636        ibase_prev = ibase;
1637    }
1638    fprintf(to, "# GC Content of last fragments. Use `grep ^GCL | cut -f 2-` to extract this part.\n");
1639    ibase_prev = 0;
1640    for (ibase=0; ibase<stats->ngc; ibase++)
1641    {
1642        if ( stats->gc_2nd[ibase]==stats->gc_2nd[ibase_prev] ) continue;
1643        fprintf(to, "GCL\t%.2f\t%ld\n", (ibase+ibase_prev)*0.5*100./(stats->ngc-1), (long)stats->gc_2nd[ibase_prev]);
1644        ibase_prev = ibase;
1645    }
1646    fprintf(to, "# ACGT content per cycle. Use `grep ^GCC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
1647    for (ibase=0; ibase<stats->max_len; ibase++)
1648    {
1649        acgtno_count_t *acgtno_count_1st = &(stats->acgtno_cycles_1st[ibase]);
1650        acgtno_count_t *acgtno_count_2nd = &(stats->acgtno_cycles_2nd[ibase]);
1651        uint64_t acgt_sum = acgtno_count_1st->a + acgtno_count_1st->c + acgtno_count_1st->g + acgtno_count_1st->t +
1652                acgtno_count_2nd->a + acgtno_count_2nd->c + acgtno_count_2nd->g + acgtno_count_2nd->t;
1653        if ( ! acgt_sum ) continue;
1654        fprintf(to, "GCC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1655                100.*(acgtno_count_1st->a + acgtno_count_2nd->a)/acgt_sum,
1656                100.*(acgtno_count_1st->c + acgtno_count_2nd->c)/acgt_sum,
1657                100.*(acgtno_count_1st->g + acgtno_count_2nd->g)/acgt_sum,
1658                100.*(acgtno_count_1st->t + acgtno_count_2nd->t)/acgt_sum,
1659                100.*(acgtno_count_1st->n + acgtno_count_2nd->n)/acgt_sum,
1660                100.*(acgtno_count_1st->other + acgtno_count_2nd->other)/acgt_sum);
1661    }
1662    fprintf(to, "# ACGT content per cycle, read oriented. Use `grep ^GCT | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]\n");
1663    for (ibase=0; ibase<stats->max_len; ibase++)
1664    {
1665        acgtno_count_t *acgtno_count = &(stats->acgtno_revcomp[ibase]);
1666        uint64_t acgt_sum = acgtno_count->a + acgtno_count->c + acgtno_count->g + acgtno_count->t;
1667        if ( ! acgt_sum ) continue;
1668        fprintf(to, "GCT\t%d\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1669                100.*(acgtno_count->a)/acgt_sum,
1670                100.*(acgtno_count->c)/acgt_sum,
1671                100.*(acgtno_count->g)/acgt_sum,
1672                100.*(acgtno_count->t)/acgt_sum);
1673    }
1674
1675    uint64_t tA=0, tC=0, tG=0, tT=0, tN=0;
1676    fprintf(to, "# ACGT content per cycle for first fragments. Use `grep ^FBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
1677    for (ibase=0; ibase<stats->max_len; ibase++)
1678    {
1679        acgtno_count_t *acgtno_count_1st = &(stats->acgtno_cycles_1st[ibase]);
1680        uint64_t acgt_sum_1st = acgtno_count_1st->a + acgtno_count_1st->c + acgtno_count_1st->g + acgtno_count_1st->t;
1681        tA += acgtno_count_1st->a;
1682        tC += acgtno_count_1st->c;
1683        tG += acgtno_count_1st->g;
1684        tT += acgtno_count_1st->t;
1685        tN += acgtno_count_1st->n;
1686
1687        if ( acgt_sum_1st )
1688            fprintf(to, "FBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1689                    100.*acgtno_count_1st->a/acgt_sum_1st,
1690                    100.*acgtno_count_1st->c/acgt_sum_1st,
1691                    100.*acgtno_count_1st->g/acgt_sum_1st,
1692                    100.*acgtno_count_1st->t/acgt_sum_1st,
1693                    100.*acgtno_count_1st->n/acgt_sum_1st,
1694                    100.*acgtno_count_1st->other/acgt_sum_1st);
1695
1696    }
1697    fprintf(to, "# ACGT raw counters for first fragments. Use `grep ^FTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
1698    fprintf(to, "FTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
1699    tA=0, tC=0, tG=0, tT=0, tN=0;
1700    fprintf(to, "# ACGT content per cycle for last fragments. Use `grep ^LBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N and O counts as a percentage of all A/C/G/T bases [%%]\n");
1701    for (ibase=0; ibase<stats->max_len; ibase++)
1702    {
1703        acgtno_count_t *acgtno_count_2nd = &(stats->acgtno_cycles_2nd[ibase]);
1704        uint64_t acgt_sum_2nd = acgtno_count_2nd->a + acgtno_count_2nd->c + acgtno_count_2nd->g + acgtno_count_2nd->t;
1705        tA += acgtno_count_2nd->a;
1706        tC += acgtno_count_2nd->c;
1707        tG += acgtno_count_2nd->g;
1708        tT += acgtno_count_2nd->t;
1709        tN += acgtno_count_2nd->n;
1710
1711        if ( acgt_sum_2nd )
1712            fprintf(to, "LBC\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", ibase+1,
1713                    100.*acgtno_count_2nd->a/acgt_sum_2nd,
1714                    100.*acgtno_count_2nd->c/acgt_sum_2nd,
1715                    100.*acgtno_count_2nd->g/acgt_sum_2nd,
1716                    100.*acgtno_count_2nd->t/acgt_sum_2nd,
1717                    100.*acgtno_count_2nd->n/acgt_sum_2nd,
1718                    100.*acgtno_count_2nd->other/acgt_sum_2nd);
1719
1720    }
1721    fprintf(to, "# ACGT raw counters for last fragments. Use `grep ^LTC | cut -f 2-` to extract this part. The columns are: A,C,G,T,N base counters\n");
1722    fprintf(to, "LTC\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\t%" PRId64 "\n", tA, tC, tG, tT, tN);
1723
1724    int tag;
1725    for (tag=0; tag<stats->ntags; tag++) {
1726        if (stats->tags_barcode[tag].nbases) {
1727            fprintf(to, "# ACGT content per cycle for barcodes. Use `grep ^%sC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [%%]; and N counts as a percentage of all A/C/G/T bases [%%]\n",
1728                    stats->tags_barcode[tag].tag_name);
1729            for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
1730            {
1731                if (ibase == stats->tags_barcode[tag].tag_sep)
1732                    continue;
1733
1734                acgtno_count_t *acgtno_count = stats->acgtno_barcode + stats->tags_barcode[tag].offset + ibase;
1735                uint64_t acgt_sum = acgtno_count->a + acgtno_count->c + acgtno_count->g + acgtno_count->t;
1736
1737                if ( acgt_sum )
1738                    fprintf(to, "%sC%d\t%d\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", stats->tags_barcode[tag].tag_name,
1739                            stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
1740                            stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep,
1741                                    100.*acgtno_count->a/acgt_sum,
1742                                    100.*acgtno_count->c/acgt_sum,
1743                                    100.*acgtno_count->g/acgt_sum,
1744                                    100.*acgtno_count->t/acgt_sum,
1745                                    100.*acgtno_count->n/acgt_sum);
1746            }
1747
1748            fprintf(to, "# Barcode Qualities. Use `grep ^%sQ | cut -f 2-` to extract this part.\n", stats->tags_barcode[tag].qual_name);
1749            fprintf(to, "# Columns correspond to qualities and rows to barcode cycles. First column is the cycle number.\n");
1750            for (ibase=0; ibase<stats->tags_barcode[tag].nbases; ibase++)
1751            {
1752                if (ibase == stats->tags_barcode[tag].tag_sep)
1753                    continue;
1754
1755                fprintf(to, "%sQ%d\t%d", stats->tags_barcode[tag].qual_name,
1756                        stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? 1 : 2,
1757                        stats->tags_barcode[tag].tag_sep < 0 || ibase < stats->tags_barcode[tag].tag_sep ? ibase+1 : ibase-stats->tags_barcode[tag].tag_sep);
1758                for (iqual=0; iqual<=stats->tags_barcode[tag].max_qual; iqual++)
1759                {
1760                    fprintf(to, "\t%ld", (long)stats->quals_barcode[(stats->tags_barcode[tag].offset + ibase)*stats->nquals+iqual]);
1761                }
1762                fprintf(to, "\n");
1763            }
1764        }
1765    }
1766
1767    fprintf(to, "# Insert sizes. Use `grep ^IS | cut -f 2-` to extract this part. The columns are: insert size, pairs total, inward oriented pairs, outward oriented pairs, other pairs\n");
1768    for (isize=0; isize<ibulk; isize++) {
1769        long in = (long)(stats->isize->inward(stats->isize->data, isize));
1770        long out = (long)(stats->isize->outward(stats->isize->data, isize));
1771        long other = (long)(stats->isize->other(stats->isize->data, isize));
1772        if (!sparse || in + out + other > 0) {
1773            fprintf(to, "IS\t%d\t%ld\t%ld\t%ld\t%ld\n", isize,  in+out+other,
1774                in , out, other);
1775        }
1776    }
1777
1778    fprintf(to, "# Read lengths. Use `grep ^RL | cut -f 2-` to extract this part. The columns are: read length, count\n");
1779    int ilen;
1780    for (ilen=0; ilen<stats->max_len; ilen++)
1781    {
1782        if ( stats->read_lengths[ilen+1]>0 )
1783            fprintf(to, "RL\t%d\t%ld\n", ilen+1, (long)stats->read_lengths[ilen+1]);
1784    }
1785
1786    fprintf(to, "# Read lengths - first fragments. Use `grep ^FRL | cut -f 2-` to extract this part. The columns are: read length, count\n");
1787    for (ilen=0; ilen<stats->max_len_1st; ilen++)
1788    {
1789        if ( stats->read_lengths_1st[ilen+1]>0 )
1790            fprintf(to, "FRL\t%d\t%ld\n", ilen+1, (long)stats->read_lengths_1st[ilen+1]);
1791    }
1792
1793    fprintf(to, "# Read lengths - last fragments. Use `grep ^LRL | cut -f 2-` to extract this part. The columns are: read length, count\n");
1794    for (ilen=0; ilen<stats->max_len_2nd; ilen++)
1795    {
1796        if ( stats->read_lengths_2nd[ilen+1]>0 )
1797            fprintf(to, "LRL\t%d\t%ld\n", ilen+1, (long)stats->read_lengths_2nd[ilen+1]);
1798    }
1799
1800    fprintf(to, "# Mapping qualities for reads !(UNMAP|SECOND|SUPPL|QCFAIL|DUP). Use `grep ^MAPQ | cut -f 2-` to extract this part. The columns are: mapq, count\n");
1801    for (imapq=0; imapq < 256; imapq++)
1802    {
1803        if ( stats->mapping_qualities[imapq]>0 )
1804            fprintf(to, "MAPQ\t%d\t%ld\n", imapq, (long)stats->mapping_qualities[imapq]);
1805    }
1806
1807    fprintf(to, "# Indel distribution. Use `grep ^ID | cut -f 2-` to extract this part. The columns are: length, number of insertions, number of deletions\n");
1808
1809    for (ilen=0; ilen<stats->nindels; ilen++)
1810    {
1811        if ( stats->insertions[ilen]>0 || stats->deletions[ilen]>0 )
1812            fprintf(to, "ID\t%d\t%ld\t%ld\n", ilen+1, (long)stats->insertions[ilen], (long)stats->deletions[ilen]);
1813    }
1814
1815    fprintf(to, "# Indels per cycle. Use `grep ^IC | cut -f 2-` to extract this part. The columns are: cycle, number of insertions (fwd), .. (rev) , number of deletions (fwd), .. (rev)\n");
1816    for (ilen=0; ilen<=stats->nbases; ilen++)
1817    {
1818        // For deletions we print the index of the cycle before the deleted base (1-based) and for insertions
1819        //  the index of the cycle of the first inserted base (also 1-based)
1820        if ( stats->ins_cycles_1st[ilen]>0 || stats->ins_cycles_2nd[ilen]>0 || stats->del_cycles_1st[ilen]>0 || stats->del_cycles_2nd[ilen]>0 )
1821            fprintf(to, "IC\t%d\t%ld\t%ld\t%ld\t%ld\n", ilen+1, (long)stats->ins_cycles_1st[ilen], (long)stats->ins_cycles_2nd[ilen], (long)stats->del_cycles_1st[ilen], (long)stats->del_cycles_2nd[ilen]);
1822    }
1823
1824    fprintf(to, "# Coverage distribution. Use `grep ^COV | cut -f 2-` to extract this part.\n");
1825    if  ( stats->cov[0] )
1826        fprintf(to, "COV\t[<%d]\t%d\t%ld\n",stats->info->cov_min,stats->info->cov_min-1, (long)stats->cov[0]);
1827    for (icov=1; icov<stats->ncov-1; icov++)
1828        if ( stats->cov[icov] )
1829            fprintf(to, "COV\t[%d-%d]\t%d\t%ld\n",stats->info->cov_min + (icov-1)*stats->info->cov_step, stats->info->cov_min + icov*stats->info->cov_step-1,stats->info->cov_min + icov*stats->info->cov_step-1, (long)stats->cov[icov]);
1830    if ( stats->cov[stats->ncov-1] )
1831        fprintf(to, "COV\t[%d<]\t%d\t%ld\n",stats->info->cov_min + (stats->ncov-2)*stats->info->cov_step-1,stats->info->cov_min + (stats->ncov-2)*stats->info->cov_step-1, (long)stats->cov[stats->ncov-1]);
1832
1833    // Calculate average GC content, then sort by GC and depth
1834    fprintf(to, "# GC-depth. Use `grep ^GCD | cut -f 2-` to extract this part. The columns are: GC%%, unique sequence percentiles, 10th, 25th, 50th, 75th and 90th depth percentile\n");
1835    uint32_t igcd;
1836    for (igcd=0; igcd<stats->igcd; igcd++)
1837    {
1838        if ( stats->info->fai )
1839            stats->gcd[igcd].gc = rint(100. * stats->gcd[igcd].gc);
1840        else
1841            if ( stats->gcd[igcd].depth )
1842                stats->gcd[igcd].gc = rint(100. * stats->gcd[igcd].gc / stats->gcd[igcd].depth);
1843    }
1844    if ( stats->ngcd )
1845        qsort(stats->gcd, stats->igcd+1, sizeof(gc_depth_t), gcd_cmp);
1846    igcd = 0;
1847    while ( igcd < stats->igcd )
1848    {
1849        // Calculate percentiles (10,25,50,75,90th) for the current GC content and print
1850        uint32_t nbins=0, itmp=igcd;
1851        float gc = stats->gcd[igcd].gc;
1852        while ( itmp<stats->igcd && fabs(stats->gcd[itmp].gc-gc)<0.1 )
1853        {
1854            nbins++;
1855            itmp++;
1856        }
1857        fprintf(to, "GCD\t%.1f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n", gc, (igcd+nbins+1)*100./(stats->igcd+1),
1858                gcd_percentile(&(stats->gcd[igcd]),nbins,10) *avg_read_length/stats->info->gcd_bin_size,
1859                gcd_percentile(&(stats->gcd[igcd]),nbins,25) *avg_read_length/stats->info->gcd_bin_size,
1860                gcd_percentile(&(stats->gcd[igcd]),nbins,50) *avg_read_length/stats->info->gcd_bin_size,
1861                gcd_percentile(&(stats->gcd[igcd]),nbins,75) *avg_read_length/stats->info->gcd_bin_size,
1862                gcd_percentile(&(stats->gcd[igcd]),nbins,90) *avg_read_length/stats->info->gcd_bin_size
1863              );
1864        igcd += nbins;
1865    }
1866}
1867
1868static void init_regions(stats_t *stats, const char *file, stats_info_t* info)
1869{
1870    FILE *fp = fopen(file,"r");
1871    if ( !fp ) error("%s: %s\n",file,strerror(errno));
1872
1873    kstring_t line = { 0, 0, NULL };
1874    int warned = 0, r, p, new_p;
1875    int prev_tid=-1;
1876    hts_pos_t prev_pos=-1LL;
1877    while (line.l = 0, kgetline(&line, (kgets_func *)fgets, fp) >= 0)
1878    {
1879        if ( line.s[0] == '#' ) continue;
1880
1881        int i = 0;
1882        while ( i<line.l && !isspace(line.s[i]) ) i++;
1883        if ( i>=line.l ) error("Could not parse the file: %s [%s]\n", file, line.s);
1884        line.s[i] = '\0';
1885
1886        int tid = bam_name2id(stats->info->sam_header, line.s);
1887        if ( tid < 0 )
1888        {
1889            if ( !warned )
1890                fprintf(stderr,"Warning: Some sequences not present in the BAM, e.g. \"%s\". This message is printed only once.\n", line.s);
1891            warned = 1;
1892            continue;
1893        }
1894
1895        if ( tid >= stats->nregions )
1896        {
1897            if(!(stats->regions = realloc(stats->regions,sizeof(regions_t)*(tid+REG_INC))))
1898                error("Could not allocate memory for region.\n");
1899
1900            int j;
1901            for (j=stats->nregions; j<tid+REG_INC; j++)
1902            {
1903                stats->regions[j].npos = stats->regions[j].mpos = stats->regions[j].cpos = 0;
1904                stats->regions[j].pos = NULL;
1905            }
1906            stats->nregions = tid+REG_INC;
1907        }
1908        int npos = stats->regions[tid].npos;
1909        if ( npos >= stats->regions[tid].mpos )
1910        {
1911            stats->regions[tid].mpos = npos+POS_INC;
1912            if (!(stats->regions[tid].pos = realloc(stats->regions[tid].pos, sizeof(hts_pair_pos_t)*stats->regions[tid].mpos)))
1913                error("Could not allocate memory for interval.\n");
1914        }
1915
1916        if ( (sscanf(&line.s[i+1],"%"SCNd64" %"SCNd64, &stats->regions[tid].pos[npos].beg, &stats->regions[tid].pos[npos].end))!=2 ) error("Could not parse the region [%s]\n", &line.s[i+1]);
1917        if ( prev_tid==-1 || prev_tid!=tid )
1918        {
1919            prev_tid = tid;
1920            prev_pos = stats->regions[tid].pos[npos].beg;
1921        }
1922        if ( prev_pos>stats->regions[tid].pos[npos].beg )
1923            error("The positions are not in chromosomal order (%s:%"PRIhts_pos" comes after %"PRIhts_pos")\n", line.s, stats->regions[tid].pos[npos].beg, prev_pos);
1924        stats->regions[tid].npos++;
1925        if ( stats->regions[tid].npos > stats->nchunks )
1926            stats->nchunks = stats->regions[tid].npos;
1927    }
1928    free(line.s);
1929    if ( !stats->regions ) error("Unable to map the -t sequences to the BAM sequences.\n");
1930    fclose(fp);
1931
1932    // sort region intervals and remove duplicates
1933    for (r = 0; r < stats->nregions; r++) {
1934        regions_t *reg = &stats->regions[r];
1935        if ( reg->npos > 1 ) {
1936            qsort(reg->pos, reg->npos, sizeof(hts_pair_pos_t), regions_lt);
1937            for (new_p = 0, p = 1; p < reg->npos; p++) {
1938                if ( reg->pos[new_p].end < reg->pos[p].beg )
1939                    reg->pos[++new_p] = reg->pos[p];
1940                else if ( reg->pos[new_p].end < reg->pos[p].end )
1941                    reg->pos[new_p].end = reg->pos[p].end;
1942            }
1943            reg->npos = ++new_p;
1944        }
1945        for (p = 0; p < reg->npos; p++) {
1946            if (reg->pos[p].end < HTS_POS_MAX) {
1947                stats->target_count += (reg->pos[p].end - reg->pos[p].beg + 1);
1948            } else {
1949                uint64_t hdr_end = sam_hdr_tid2len(info->sam_header, r);
1950                if (hdr_end)
1951                    stats->target_count += (hdr_end - reg->pos[p].beg + 1);
1952            }
1953        }
1954    }
1955
1956    if (!(stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t))))
1957        error("Could not allocate memory for chunk.\n");
1958}
1959
1960void destroy_regions(stats_t *stats)
1961{
1962    int i;
1963    for (i=0; i<stats->nregions; i++)
1964    {
1965        if ( !stats->regions[i].mpos ) continue;
1966        free(stats->regions[i].pos);
1967    }
1968    if ( stats->regions ) free(stats->regions);
1969    if ( stats->chunks ) free(stats->chunks);
1970}
1971
1972void reset_regions(stats_t *stats)
1973{
1974    int i;
1975    for (i=0; i<stats->nregions; i++)
1976        stats->regions[i].cpos = 0;
1977}
1978
1979int is_in_regions(bam1_t *bam_line, stats_t *stats)
1980{
1981    if ( !stats->regions ) return 1;
1982
1983    if ( bam_line->core.tid >= stats->nregions || bam_line->core.tid<0 ) return 0;
1984    if ( !stats->is_sorted ) error("The BAM must be sorted in order for -t to work.\n");
1985
1986    regions_t *reg = &stats->regions[bam_line->core.tid];
1987    if ( reg->cpos==reg->npos ) return 0;       // done for this chr
1988
1989    // Find a matching interval or skip this read. No splicing of reads is done, no indels or soft clips considered,
1990    //  even small overlap is enough to include the read in the stats.
1991    int i = reg->cpos;
1992    while ( i<reg->npos && reg->pos[i].end<=bam_line->core.pos ) i++;
1993    if ( i>=reg->npos ) { reg->cpos = reg->npos; return 0; }
1994    int64_t endpos = bam_endpos(bam_line);
1995    if ( endpos < reg->pos[i].beg ) return 0;
1996
1997    //found a read overlapping a region
1998    reg->cpos = i;
1999    stats->reg_from = reg->pos[i].beg;
2000    stats->reg_to   = reg->pos[i].end;
2001
2002    //now find all the overlapping chunks
2003    stats->nchunks = 0;
2004    while (i < reg->npos) {
2005        if (bam_line->core.pos < reg->pos[i].end && endpos >= reg->pos[i].beg) {
2006            stats->chunks[stats->nchunks].beg = MAX(bam_line->core.pos+1, reg->pos[i].beg);
2007            stats->chunks[stats->nchunks].end = MIN(endpos, reg->pos[i].end);
2008            stats->nchunks++;
2009        }
2010        i++;
2011    }
2012
2013    return 1;
2014}
2015
2016int replicate_regions(stats_t *stats, hts_itr_multi_t *iter, stats_info_t *info) {
2017    if ( !stats || !iter)
2018        return 1;
2019
2020    int i, j, tid;
2021    stats->nregions = iter->n_reg;
2022    stats->regions = calloc(stats->nregions, sizeof(regions_t));
2023    stats->chunks = calloc(stats->nchunks, sizeof(hts_pair_pos_t));
2024    if ( !stats->regions || !stats->chunks )
2025        return 1;
2026
2027    for (i = 0; i < iter->n_reg; i++) {
2028        tid = iter->reg_list[i].tid;
2029        if ( tid < 0 )
2030            continue;
2031
2032        if ( tid >= stats->nregions ) {
2033            regions_t *tmp = realloc(stats->regions, (tid+10) * sizeof(regions_t));
2034            if ( !tmp )
2035                return 1;
2036            stats->regions = tmp;
2037            memset(stats->regions + stats->nregions, 0,
2038                   (tid+10-stats->nregions) * sizeof(regions_t));
2039            stats->nregions = tid+10;
2040        }
2041
2042        stats->regions[tid].mpos = stats->regions[tid].npos = iter->reg_list[i].count;
2043        stats->regions[tid].pos = calloc(stats->regions[tid].mpos, sizeof(hts_pair_pos_t));
2044        if ( !stats->regions[tid].pos )
2045            return 1;
2046
2047        for (j = 0; j < stats->regions[tid].npos; j++) {
2048            stats->regions[tid].pos[j].beg = iter->reg_list[i].intervals[j].beg+1;
2049            stats->regions[tid].pos[j].end = iter->reg_list[i].intervals[j].end;
2050            if (stats->regions[tid].pos[j].end < HTS_POS_MAX) {
2051                stats->target_count += (stats->regions[tid].pos[j].end - stats->regions[tid].pos[j].beg + 1);
2052            } else {
2053                uint64_t hdr_end = sam_hdr_tid2len(info->sam_header, tid);
2054                if (hdr_end)
2055                    stats->target_count += (hdr_end - stats->regions[tid].pos[j].beg + 1);
2056            }
2057        }
2058    }
2059
2060    return 0;
2061}
2062
2063static void init_group_id(stats_t *stats, stats_info_t *info, const char *id)
2064{
2065    stats->rg_hash = kh_init(rg);
2066    if (!stats->rg_hash) error("Could not initialise RG set\n");
2067    sam_hdr_t *hdr = info->sam_header;
2068    const char *key;
2069    kstring_t sm = KS_INITIALIZE;
2070    int i, ret, nrg = sam_hdr_count_lines(hdr, "RG");
2071    if (nrg < 0) error("Could not parse header\n");
2072
2073    for (i=0; i<nrg; i++) {
2074        key = sam_hdr_line_name(hdr, "RG", i);
2075        if (!strcmp(key, id)) {
2076            kh_put(rg, stats->rg_hash, key, &ret);
2077            if (ret == -1) { ks_free(&sm); error("Could not add key \"%s\" to RG set\n", key); }
2078        } else { /* Check for SM name, as per manual */
2079            if (!sam_hdr_find_tag_pos(hdr, "RG", i, "SM", &sm)) {
2080                if (!strcmp(ks_c_str(&sm), id)) {
2081                    kh_put(rg, stats->rg_hash, key, &ret);
2082                    if (ret == -1) { ks_free(&sm); error("Could not add key \"%s\" to RG set\n", key); }
2083                }
2084            }
2085        }
2086    }
2087
2088    ks_free(&sm);
2089}
2090
2091
2092static void  HTS_FORMAT(HTS_PRINTF_FMT, 1, 2) HTS_NORETURN
2093error(const char *format, ...)
2094{
2095    if ( !format )
2096    {
2097        printf("About: The program collects statistics from BAM files. The output can be visualized using plot-bamstats.\n");
2098        printf("Usage: samtools stats [OPTIONS] file.bam\n");
2099        printf("       samtools stats [OPTIONS] file.bam chr:from-to\n");
2100        printf("Options:\n");
2101        printf("    -c, --coverage <int>,<int>,<int>    Coverage distribution min,max,step [1,1000,1]\n");
2102        printf("    -d, --remove-dups                   Exclude from statistics reads marked as duplicates\n");
2103        printf("    -X, --customized-index-file         Use a customized index file\n");
2104        printf("    -f, --required-flag  <str|int>      Required flag, 0 for unset. See also `samtools flags` [0]\n");
2105        printf("    -F, --filtering-flag <str|int>      Filtering flag, 0 for unset. See also `samtools flags` [0]\n");
2106        printf("        --GC-depth <float>              the size of GC-depth bins (decreasing bin size increases memory requirement) [2e4]\n");
2107        printf("    -h, --help                          This help message\n");
2108        printf("    -i, --insert-size <int>             Maximum insert size [8000]\n");
2109        printf("    -I, --id <string>                   Include only listed read group or sample name\n");
2110        printf("    -l, --read-length <int>             Include in the statistics only reads with the given read length [-1]\n");
2111        printf("    -m, --most-inserts <float>          Report only the main part of inserts [0.99]\n");
2112        printf("    -P, --split-prefix <str>            Path or string prefix for filepaths output by -S (default is input filename)\n");
2113        printf("    -q, --trim-quality <int>            The BWA trimming parameter [0]\n");
2114        printf("    -r, --ref-seq <file>                Reference sequence (required for GC-depth and mismatches-per-cycle calculation).\n");
2115        printf("    -s, --sam                           Ignored (input format is auto-detected).\n");
2116        printf("    -S, --split <tag>                   Also write statistics to separate files split by tagged field.\n");
2117        printf("    -t, --target-regions <file>         Do stats in these regions only. Tab-delimited file chr,from,to, 1-based, inclusive.\n");
2118        printf("    -x, --sparse                        Suppress outputting IS rows where there are no insertions.\n");
2119        printf("    -p, --remove-overlaps               Remove overlaps of paired-end reads from coverage and base count computations.\n");
2120        printf("    -g, --cov-threshold <int>           Only bases with coverage above this value will be included in the target percentage computation [0]\n");
2121        sam_global_opt_help(stdout, "-.--.@-.");
2122        printf("\n");
2123    }
2124    else
2125    {
2126        va_list ap;
2127        va_start(ap, format);
2128        vfprintf(stderr, format, ap);
2129        va_end(ap);
2130    }
2131    exit(1);
2132}
2133
2134void cleanup_stats_info(stats_info_t* info){
2135    if (info->fai) fai_destroy(info->fai);
2136    if (info->sam) sam_close(info->sam);
2137    free(info);
2138}
2139
2140void cleanup_stats(stats_t* stats)
2141{
2142    free(stats->cov_rbuf.buffer); free(stats->cov);
2143    free(stats->quals_1st); free(stats->quals_2nd);
2144    free(stats->gc_1st); free(stats->gc_2nd);
2145    stats->isize->isize_free(stats->isize->data);
2146    free(stats->isize);
2147    free(stats->gcd);
2148    free(stats->rseq_buf);
2149    free(stats->mpc_buf);
2150    free(stats->acgtno_cycles_1st);
2151    free(stats->acgtno_cycles_2nd);
2152    free(stats->acgtno_revcomp);
2153    free(stats->read_lengths);
2154    free(stats->read_lengths_1st);
2155    free(stats->read_lengths_2nd);
2156    free(stats->insertions);
2157    free(stats->deletions);
2158    free(stats->ins_cycles_1st);
2159    free(stats->ins_cycles_2nd);
2160    free(stats->del_cycles_1st);
2161    free(stats->del_cycles_2nd);
2162    if (stats->acgtno_barcode) free(stats->acgtno_barcode);
2163    if (stats->quals_barcode) free(stats->quals_barcode);
2164    free(stats->tags_barcode);
2165    destroy_regions(stats);
2166    if ( stats->rg_hash ) kh_destroy(rg, stats->rg_hash);
2167    free(stats->split_name);
2168    free(stats->mapping_qualities);
2169    free(stats);
2170}
2171
2172void output_split_stats(khash_t(c2stats) *split_hash, char* bam_fname, int sparse)
2173{
2174    int i = 0;
2175    kstring_t output_filename = { 0, 0, NULL };
2176    stats_t *curr_stats = NULL;
2177    for(i = kh_begin(split_hash); i != kh_end(split_hash); ++i){
2178        if(!kh_exist(split_hash, i)) continue;
2179        curr_stats = kh_value(split_hash, i);
2180        round_buffer_flush(curr_stats, -1);
2181
2182        output_filename.l = 0;
2183        if (curr_stats->info->split_prefix)
2184            kputs(curr_stats->info->split_prefix, &output_filename);
2185        else
2186            kputs(bam_fname, &output_filename);
2187        kputc('_', &output_filename);
2188        kputs(curr_stats->split_name, &output_filename);
2189        kputs(".bamstat", &output_filename);
2190
2191        FILE *to = fopen(output_filename.s, "w");
2192        if(to == NULL){
2193            error("Could not open '%s' for writing.\n", output_filename.s);
2194        }
2195        output_stats(to, curr_stats, sparse);
2196        fclose(to);
2197    }
2198
2199    free(output_filename.s);
2200}
2201
2202void destroy_split_stats(khash_t(c2stats) *split_hash)
2203{
2204    if (!split_hash)
2205        return;
2206
2207    int i = 0;
2208    stats_t *curr_stats = NULL;
2209    for(i = kh_begin(split_hash); i != kh_end(split_hash); ++i){
2210        if(!kh_exist(split_hash, i)) continue;
2211        curr_stats = kh_value(split_hash, i);
2212        cleanup_stats(curr_stats);
2213    }
2214    kh_destroy(c2stats, split_hash);
2215}
2216
2217stats_info_t* stats_info_init(int argc, char *argv[])
2218{
2219    stats_info_t* info = calloc(1, sizeof(stats_info_t));
2220    if (!info) {
2221        return NULL;
2222    }
2223
2224    info->nisize = 8000;
2225    info->isize_main_bulk = 0.99;   // There are always outliers at the far end
2226    info->gcd_bin_size = 20e3;
2227    info->cov_min  = 1;
2228    info->cov_max  = 1000;
2229    info->cov_step = 1;
2230    info->filter_readlen = -1;
2231    info->argc = argc;
2232    info->argv = argv;
2233    info->remove_overlaps = 0;
2234    info->cov_threshold = 0;
2235
2236    return info;
2237}
2238
2239int init_stat_info_fname(stats_info_t* info, const char* bam_fname, const htsFormat* in_fmt)
2240{
2241    // .. bam
2242    samFile* sam;
2243    if ((sam = sam_open_format(bam_fname, "r", in_fmt)) == 0) {
2244        print_error_errno("stats", "failed to open \"%s\"", bam_fname);
2245        return 1;
2246    }
2247    info->sam = sam;
2248    info->sam_header = sam_hdr_read(sam);
2249    if (info->sam_header == NULL) {
2250        print_error("stats", "failed to read header for \"%s\"", bam_fname);
2251        return 1;
2252    }
2253    return 0;
2254}
2255
2256stats_t* stats_init(void)
2257{
2258    stats_t *stats = calloc(1,sizeof(stats_t));
2259    if (!stats)
2260        return NULL;
2261
2262    stats->ngc    = 200;
2263    stats->nquals = 256;
2264    stats->nbases = 300;
2265    stats->rseq_pos     = -1;
2266    stats->tid = -1;
2267    stats->gcd_pos = -1LL;
2268    stats->igcd = 0;
2269    stats->is_sorted = 1;
2270    stats->nindels = stats->nbases;
2271    stats->split_name = NULL;
2272    stats->nchunks = 0;
2273    stats->pair_count = 0;
2274    stats->last_pair_tid = -2;
2275    stats->last_read_flush = 0;
2276    stats->target_count = 0;
2277
2278    return stats;
2279}
2280
2281static int init_barcode_tags(stats_t* stats) {
2282    stats->ntags = 4;
2283    stats->tags_barcode = calloc(stats->ntags, sizeof(barcode_info_t));
2284    if (!stats->tags_barcode)
2285        return -1;
2286    stats->tags_barcode[0] = (barcode_info_t){"BC", "QT", 0, -1, -1, 0};
2287    stats->tags_barcode[1] = (barcode_info_t){"CR", "CY", 0, -1, -1, 0};
2288    stats->tags_barcode[2] = (barcode_info_t){"OX", "BZ", 0, -1, -1, 0};
2289    stats->tags_barcode[3] = (barcode_info_t){"RX", "QX", 0, -1, -1, 0};
2290    return 0;
2291}
2292
2293static void init_stat_structs(stats_t* stats, stats_info_t* info, const char* group_id, const char* targets)
2294{
2295    // Give stats_t a pointer to the info struct
2296    // This saves us having to pass the stats_info_t to every function
2297    stats->info = info;
2298
2299    // Init structures
2300    //  .. coverage bins and round buffer
2301    if ( info->cov_step > info->cov_max - info->cov_min + 1 )
2302    {
2303        info->cov_step = info->cov_max - info->cov_min;
2304        if ( info->cov_step <= 0 )
2305            info->cov_step = 1;
2306    }
2307    stats->ncov = 3 + (info->cov_max-info->cov_min) / info->cov_step;
2308    info->cov_max = info->cov_min + ((info->cov_max-info->cov_min)/info->cov_step +1)*info->cov_step - 1;
2309    stats->cov = calloc(sizeof(uint64_t),stats->ncov);
2310    if (!stats->cov) goto nomem;
2311    stats->cov_rbuf.size = stats->nbases*5;
2312    stats->cov_rbuf.buffer = calloc(sizeof(int32_t),stats->cov_rbuf.size);
2313    if (!stats->cov_rbuf.buffer) goto nomem;
2314    if ( group_id ) init_group_id(stats, info, group_id);
2315    // .. arrays
2316    stats->quals_1st      = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
2317    if (!stats->quals_1st) goto nomem;
2318    stats->quals_2nd      = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
2319    if (!stats->quals_2nd) goto nomem;
2320    stats->gc_1st         = calloc(stats->ngc,sizeof(uint64_t));
2321    if (!stats->gc_1st) goto nomem;
2322    stats->gc_2nd         = calloc(stats->ngc,sizeof(uint64_t));
2323    if (!stats->gc_2nd) goto nomem;
2324    stats->isize          = init_isize_t(info->nisize ?info->nisize+1 :0);
2325    if (!stats->isize) goto nomem;
2326    stats->gcd            = calloc(stats->ngcd,sizeof(gc_depth_t));
2327    if (!stats->gcd) goto nomem;
2328    if (info->fai) {
2329        stats->mpc_buf    = calloc(stats->nquals*stats->nbases,sizeof(uint64_t));
2330        if (!stats->mpc_buf) goto nomem;
2331    } else {
2332        stats->mpc_buf    = NULL;
2333    }
2334    stats->acgtno_cycles_1st  = calloc(stats->nbases,sizeof(acgtno_count_t));
2335    if (!stats->acgtno_cycles_1st) goto nomem;
2336    stats->acgtno_cycles_2nd  = calloc(stats->nbases,sizeof(acgtno_count_t));
2337    if (!stats->acgtno_cycles_2nd) goto nomem;
2338    stats->acgtno_revcomp  = calloc(stats->nbases,sizeof(acgtno_count_t));
2339    if (!stats->acgtno_revcomp) goto nomem;
2340    stats->read_lengths   = calloc(stats->nbases,sizeof(uint64_t));
2341    if (!stats->read_lengths)     goto nomem;
2342    stats->read_lengths_1st   = calloc(stats->nbases,sizeof(uint64_t));
2343    if (!stats->read_lengths_1st) goto nomem;
2344    stats->read_lengths_2nd   = calloc(stats->nbases,sizeof(uint64_t));
2345    if (!stats->read_lengths_2nd) goto nomem;
2346    stats->insertions     = calloc(stats->nbases,sizeof(uint64_t));
2347    if (!stats->insertions) goto nomem;
2348    stats->deletions      = calloc(stats->nbases,sizeof(uint64_t));
2349    if (!stats->deletions)  goto nomem;
2350    stats->ins_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
2351    if (!stats->ins_cycles_1st) goto nomem;
2352    stats->ins_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
2353    if (!stats->ins_cycles_2nd) goto nomem;
2354    stats->del_cycles_1st = calloc(stats->nbases+1,sizeof(uint64_t));
2355    if (!stats->del_cycles_1st) goto nomem;
2356    stats->del_cycles_2nd = calloc(stats->nbases+1,sizeof(uint64_t));
2357    if (!stats->del_cycles_2nd) goto nomem;
2358    stats->mapping_qualities = calloc(256,sizeof(uint64_t));
2359    if(!stats->mapping_qualities) goto nomem;
2360    if (init_barcode_tags(stats) < 0)
2361        goto nomem;
2362    realloc_rseq_buffer(stats);
2363    if ( targets )
2364        init_regions(stats, targets, info);
2365    return;
2366 nomem:
2367    error("Out of memory");
2368}
2369
2370static stats_t* get_curr_split_stats(bam1_t* bam_line, khash_t(c2stats)* split_hash, stats_info_t* info, char* targets)
2371{
2372    stats_t *curr_stats = NULL;
2373    const uint8_t *tag_val = bam_aux_get(bam_line, info->split_tag);
2374    if(tag_val == 0){
2375        error("Tag '%s' not found in bam_line.\n", info->split_tag);
2376    }
2377    char* split_name = strdup(bam_aux2Z(tag_val));
2378
2379    // New stats object, under split
2380    khiter_t k = kh_get(c2stats, split_hash, split_name);
2381    if(k == kh_end(split_hash)){
2382        curr_stats = stats_init(); // mallocs new instance
2383        if (!curr_stats) {
2384            error("Couldn't allocate split stats");
2385        }
2386        init_stat_structs(curr_stats, info, NULL, targets);
2387        curr_stats->split_name = split_name;
2388
2389        // Record index in hash
2390        int ret = 0;
2391        khiter_t iter = kh_put(c2stats, split_hash, split_name, &ret);
2392        if( ret < 0 ){
2393            error("Failed to insert key '%s' into split_hash", split_name);
2394        }
2395        kh_val(split_hash, iter) = curr_stats; // store pointer to stats
2396    }
2397    else{
2398        curr_stats = kh_value(split_hash, k);
2399        free(split_name); // don't need to hold on to this if it wasn't new
2400    }
2401    return curr_stats;
2402}
2403
2404int main_stats(int argc, char *argv[])
2405{
2406    char *targets = NULL;
2407    char *bam_fname = NULL;
2408    char *bam_idx_fname = NULL;
2409    char *group_id = NULL;
2410    int sparse = 0, has_index_file = 0, ret = 1;
2411    sam_global_args ga = SAM_GLOBAL_ARGS_INIT;
2412
2413    stats_info_t *info = stats_info_init(argc, argv);
2414    if (!info) {
2415        fprintf(stderr, "Could not allocate memory for info.\n");
2416        return 1;
2417    }
2418
2419    static const struct option loptions[] =
2420    {
2421        SAM_OPT_GLOBAL_OPTIONS('-', 0, '-', '-', 0, '@'),
2422        {"help", no_argument, NULL, 'h'},
2423        {"remove-dups", no_argument, NULL, 'd'},
2424        {"sam", no_argument, NULL, 's'},
2425        {"customized-index-file", required_argument, NULL, 'X'},
2426        {"ref-seq", required_argument, NULL, 'r'},
2427        {"coverage", required_argument, NULL, 'c'},
2428        {"read-length", required_argument, NULL, 'l'},
2429        {"insert-size", required_argument, NULL, 'i'},
2430        {"most-inserts", required_argument, NULL, 'm'},
2431        {"trim-quality", required_argument, NULL, 'q'},
2432        {"target-regions", required_argument, NULL, 't'},
2433        {"required-flag", required_argument, NULL, 'f'},
2434        {"filtering-flag", required_argument, NULL, 'F'},
2435        {"id", required_argument, NULL, 'I'},
2436        {"GC-depth", required_argument, NULL, 1},
2437        {"sparse", no_argument, NULL, 'x'},
2438        {"split", required_argument, NULL, 'S'},
2439        {"split-prefix", required_argument, NULL, 'P'},
2440        {"remove-overlaps", no_argument, NULL, 'p'},
2441        {"cov-threshold", required_argument, NULL, 'g'},
2442        {NULL, 0, NULL, 0}
2443    };
2444    int opt, tmp_flag;
2445
2446    while ( (opt=getopt_long(argc,argv,"?hdsXxpr:c:l:i:t:m:q:f:F:g:I:S:P:@:",loptions,NULL))>0 )
2447    {
2448        switch (opt)
2449        {
2450            case 'f':
2451                tmp_flag = bam_str2flag(optarg);
2452
2453                if (tmp_flag < 0) {
2454                    print_error("stats", "Unknown flag '%s'", optarg);
2455                    return 1;
2456                }
2457
2458                info->flag_require = tmp_flag;
2459                break;
2460
2461            case 'F':
2462                tmp_flag = bam_str2flag(optarg);
2463
2464                if (tmp_flag < 0) {
2465                    print_error("stats", "Unknown flag '%s'", optarg);
2466                    return 1;
2467                }
2468
2469                info->flag_filter |= tmp_flag;
2470                break;
2471
2472            case 'd': info->flag_filter |= BAM_FDUP; break;
2473            case 'X': has_index_file = 1; break;
2474            case 's': break;
2475            case 'r': info->fai = fai_load(optarg);
2476                      if (info->fai==NULL)
2477                          error("Could not load faidx: %s\n", optarg);
2478                      break;
2479            case  1 : info->gcd_bin_size = atof(optarg); break;
2480            case 'c': if ( sscanf(optarg,"%d,%d,%d",&info->cov_min,&info->cov_max,&info->cov_step)!= 3 )
2481                          error("Unable to parse -c %s\n", optarg);
2482                      break;
2483            case 'l': info->filter_readlen = atoi(optarg); break;
2484            case 'i': info->nisize = atoi(optarg); break;
2485            case 'm': info->isize_main_bulk = atof(optarg); break;
2486            case 'q': info->trim_qual = atoi(optarg); break;
2487            case 't': targets = optarg; break;
2488            case 'I': group_id = optarg; break;
2489            case 'x': sparse = 1; break;
2490            case 'S': info->split_tag = optarg; break;
2491            case 'P': info->split_prefix = optarg; break;
2492            case 'p': info->remove_overlaps = 1; break;
2493            case 'g': info->cov_threshold = atoi(optarg);
2494                      if ( info->cov_threshold < 0 || info->cov_threshold == INT_MAX )
2495                          error("Unsupported value for coverage threshold %d\n", info->cov_threshold);
2496                      break;
2497            case '?':
2498            case 'h': error(NULL);
2499            /* no break */
2500            default:
2501                if (parse_sam_global_opt(opt, optarg, loptions, &ga) != 0)
2502                    error("Unknown argument: %s\n", optarg);
2503                break;
2504        }
2505    }
2506
2507    bam_fname = argv[optind++];
2508    if ( !bam_fname )
2509    {
2510        if ( isatty(STDIN_FILENO) )
2511            error(NULL);
2512        bam_fname = "-";
2513    }
2514
2515    if (init_stat_info_fname(info, bam_fname, &ga.in)) {
2516        cleanup_stats_info(info);
2517        return 1;
2518    }
2519
2520    if (has_index_file && !(bam_idx_fname = argv[optind++])) {
2521        fprintf(stderr, "No index file provided\n");
2522        cleanup_stats_info(info);
2523        return 1;
2524    }
2525
2526    if (ga.nthreads > 0)
2527        hts_set_threads(info->sam, ga.nthreads);
2528
2529    stats_t *all_stats = stats_init();
2530    if (!all_stats) {
2531        fprintf(stderr, "Could not allocate memory for stats.\n");
2532        cleanup_stats_info(info);
2533        return 1;
2534    }
2535    stats_t *curr_stats = NULL;
2536    init_stat_structs(all_stats, info, group_id, targets);
2537    // Init
2538    // .. hash
2539    khash_t(c2stats)* split_hash = kh_init(c2stats);
2540    if (!split_hash) goto cleanup_all_stats;
2541
2542    khash_t(qn2pair)* read_pairs = kh_init(qn2pair);
2543    if (!read_pairs) goto cleanup_split_hash;
2544
2545    // Collect statistics
2546    bam1_t *bam_line = bam_init1();
2547    if (!bam_line) goto cleanup_read_pairs;
2548
2549    if (optind < argc) {
2550        // Region:interval arguments in the command line
2551        hts_idx_t *bam_idx = NULL;
2552        if (has_index_file) {
2553            bam_idx = sam_index_load2(info->sam, bam_fname, bam_idx_fname);
2554        } else {
2555            // If an index filename has not been specified, look alongside the alignment file
2556            bam_idx = sam_index_load(info->sam, bam_fname);
2557        }
2558
2559        if (bam_idx) {
2560            hts_itr_multi_t *iter = sam_itr_regarray(bam_idx, info->sam_header, &argv[optind], argc - optind);
2561            if (iter) {
2562                if (!targets) {
2563                    all_stats->nchunks = argc-optind;
2564                    if (replicate_regions(all_stats, iter, info))
2565                        fprintf(stderr, "Replications of the regions failed\n");
2566                }
2567
2568                if ( all_stats->nregions && all_stats->regions ) {
2569                    while ((ret = sam_itr_next(info->sam, iter, bam_line)) >= 0) {
2570                        if (info->split_tag) {
2571                            curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
2572                            collect_stats(bam_line, curr_stats, read_pairs);
2573                        }
2574                        collect_stats(bam_line, all_stats, read_pairs);
2575                    }
2576
2577                    if (ret < -1) {
2578                        fprintf(stderr, "Failure while running the iterator\n");
2579                        hts_itr_multi_destroy(iter);
2580                        hts_idx_destroy(bam_idx);
2581                        goto cleanup;
2582                    }
2583                }
2584                hts_itr_multi_destroy(iter);
2585            } else {
2586                fprintf(stderr, "Multi-region iterator could not be created\n");
2587                hts_idx_destroy(bam_idx);
2588                goto cleanup;
2589            }
2590            hts_idx_destroy(bam_idx);
2591        } else {
2592            if (has_index_file)
2593                fprintf(stderr, "Invalid index file '%s'\n", bam_idx_fname);
2594            fprintf(stderr, "Random alignment retrieval only works for indexed files\n");
2595            goto cleanup;
2596        }
2597    } else {
2598        if ( info->cov_threshold > 0 && !targets ) {
2599            fprintf(stderr, "Coverage percentage calculation requires a list of target regions\n");
2600            goto cleanup;
2601        }
2602
2603        // Stream through the entire BAM ignoring off-target regions if -t is given
2604        while ((ret = sam_read1(info->sam, info->sam_header, bam_line)) >= 0) {
2605            if (info->split_tag) {
2606                curr_stats = get_curr_split_stats(bam_line, split_hash, info, targets);
2607                collect_stats(bam_line, curr_stats, read_pairs);
2608            }
2609            collect_stats(bam_line, all_stats, read_pairs);
2610        }
2611
2612        if (ret < -1) {
2613            fprintf(stderr, "Failure while decoding file\n");
2614            goto cleanup;
2615        }
2616    }
2617
2618    round_buffer_flush(all_stats, -1);
2619    output_stats(stdout, all_stats, sparse);
2620    if (info->split_tag)
2621        output_split_stats(split_hash, bam_fname, sparse);
2622
2623    ret = 0;
2624cleanup:
2625    bam_destroy1(bam_line);
2626    sam_hdr_destroy(info->sam_header);
2627    sam_global_args_free(&ga);
2628
2629cleanup_read_pairs:
2630    cleanup_overlaps(read_pairs, INT64_MAX);
2631cleanup_split_hash:
2632    destroy_split_stats(split_hash);
2633cleanup_all_stats:
2634    cleanup_stats(all_stats);
2635    cleanup_stats_info(info);
2636
2637    return ret;
2638}