Source code

Download source archive ↓

upstream-dependencies/samtools/bam_reheader.c

1/*  bam_reheader.c -- reheader subcommand.
2
3    Copyright (C) 2010 Broad Institute.
4    Copyright (C) 2012-2019 Genome Research Ltd.
5
6    Author: Heng Li <lh3@sanger.ac.uk>
7
8Permission is hereby granted, free of charge, to any person obtaining a copy
9of this software and associated documentation files (the "Software"), to deal
10in the Software without restriction, including without limitation the rights
11to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12copies of the Software, and to permit persons to whom the Software is
13furnished to do so, subject to the following conditions:
14
15The above copyright notice and this permission notice shall be included in
16all copies or substantial portions of the Software.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE.  */
25
26#include <config.h>
27
28#include <stdio.h>
29#include <stdlib.h>
30#include <assert.h>
31#include <getopt.h>
32#include <unistd.h>
33
34#include "htslib/bgzf.h"
35#include "htslib/sam.h"
36#include "htslib/hfile.h"
37#include "htslib/cram.h"
38#include "samtools.h"
39
40#define BUF_SIZE 0x10000
41
42/*
43 * Reads a file and outputs a new BAM file to fd with 'h' replaced as
44 * the header.    No checks are made to the validity.
45 */
46int bam_reheader(BGZF *in, sam_hdr_t *h, int fd,
47                 const char *arg_list, int no_pg, int skip_header)
48{
49    BGZF *fp = NULL;
50    ssize_t len;
51    uint8_t *buf = NULL;
52    sam_hdr_t *tmp;
53    if (!h)
54        return -1;
55
56    if (in->is_write) return -1;
57    buf = malloc(BUF_SIZE);
58    if (!buf) {
59        fprintf(stderr, "Out of memory\n");
60        return -1;
61    }
62
63    if (!skip_header) {
64        if ((tmp = bam_hdr_read(in)) == NULL) {
65            fprintf(stderr, "Couldn't read header\n");
66            goto fail;
67        }
68        sam_hdr_destroy(tmp);
69    }
70
71    fp = bgzf_fdopen(fd, "w");
72    if (!fp) {
73        print_error_errno("reheader", "Couldn't open output file");
74        goto fail;
75    }
76
77    if (!no_pg && sam_hdr_add_pg(h, "samtools",
78                           "VN", samtools_version(),
79                           arg_list ? "CL": NULL,
80                           arg_list ? arg_list : NULL,
81                           NULL) != 0)
82            goto fail;
83
84    if (bam_hdr_write(fp, h) < 0) {
85        print_error_errno("reheader", "Couldn't write header");
86        goto fail;
87    }
88    if (in->block_offset < in->block_length) {
89        if (bgzf_write(fp, (char *)in->uncompressed_block + in->block_offset, in->block_length - in->block_offset) < 0) goto write_fail;
90        if (bgzf_flush(fp) < 0) goto write_fail;
91    }
92    while ((len = bgzf_raw_read(in, buf, BUF_SIZE)) > 0) {
93        if (bgzf_raw_write(fp, buf, len) < 0) goto write_fail;
94    }
95    if (len < 0) {
96        fprintf(stderr, "[%s] Error reading input file\n", __func__);
97        goto fail;
98    }
99    free(buf);
100    fp->block_offset = in->block_offset = 0;
101    if (bgzf_close(fp) < 0) {
102        fprintf(stderr, "[%s] Error closing output file\n", __func__);
103        return -1;
104    }
105    return 0;
106
107 write_fail:
108    print_error_errno("reheader", "Error writing to output file");
109 fail:
110    bgzf_close(fp);
111    free(buf);
112    return -1;
113}
114
115/*
116 * Reads a file and outputs a new CRAM file to stdout with 'h'
117 * replaced as the header.  No checks are made to the validity.
118 *
119 * FIXME: error checking
120 */
121int cram_reheader(cram_fd *in, sam_hdr_t *h, const char *arg_list, int no_pg)
122{
123    htsFile *h_out = hts_open("-", "wc");
124    cram_fd *out = h_out->fp.cram;
125    cram_container *c = NULL;
126    int ret = -1;
127    if (!h)
128        return ret;
129
130    // Match output version number with input file.
131    char vers[99];
132    sprintf(vers, "%d.%d", cram_major_vers(in), cram_minor_vers(in));
133    cram_set_option(out, CRAM_OPT_VERSION, vers);
134
135    // Attempt to fill out a cram->refs[] array from @SQ headers
136    sam_hdr_t *cram_h = sam_hdr_dup(h);
137    if (!cram_h)
138        return -1;
139    cram_fd_set_header(out, cram_h);
140    if (!no_pg && sam_hdr_add_pg(cram_fd_get_header(out), "samtools",
141                           "VN", samtools_version(),
142                           arg_list ? "CL": NULL,
143                           arg_list ? arg_list : NULL,
144                           NULL))
145            goto err;
146
147    if (sam_hdr_write(h_out, cram_h) != 0)
148        goto err;
149    cram_set_option(out, CRAM_OPT_REFERENCE, NULL);
150
151    while ((c = cram_read_container(in))) {
152        int32_t i, num_blocks = cram_container_get_num_blocks(c);
153        if (cram_write_container(out, c) != 0)
154            goto err;
155
156        for (i = 0; i < num_blocks; i++) {
157            cram_block *blk = cram_read_block(in);
158            if (!blk || cram_write_block(out, blk) != 0) {
159                if (blk) cram_free_block(blk);
160                goto err;
161            }
162            cram_free_block(blk);
163        }
164        cram_free_container(c);
165    }
166
167    ret = 0;
168
169 err:
170    if (hts_close(h_out) != 0)
171        ret = -1;
172
173    return ret;
174}
175
176
177
178/*
179 * Reads a version 2 CRAM file and replaces the header in-place,
180 * provided the header is small enough to fit without growing the
181 * entire file.
182 *
183 * Version 2 format has an uncompressed SAM header with multiple nul
184 * termination bytes to permit inline header editing.
185 *
186 * Returns 0 on success;
187 *        -1 on general failure;
188 *        -2 on failure due to insufficient size
189 */
190int cram_reheader_inplace2(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
191                          int no_pg)
192{
193    cram_container *c = NULL;
194    cram_block *b = NULL;
195    sam_hdr_t *cram_h = NULL;
196    off_t start;
197    int ret = -1;
198    if (!h)
199        goto err;
200
201    if (cram_major_vers(fd) < 2 ||
202        cram_major_vers(fd) > 3) {
203        fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
204                cram_major_vers(fd));
205        goto err;
206    }
207
208    cram_h = sam_hdr_dup(h);
209    if (!cram_h)
210        goto err;
211
212    if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
213                                 arg_list ? "CL": NULL,
214                                 arg_list ? arg_list : NULL,
215                                 NULL))
216        goto err;
217
218    int header_len = sam_hdr_length(cram_h);
219    /* Fix M5 strings? Maybe out of scope for this tool */
220
221    // Load the existing header
222    if ((start = hseek(cram_fd_get_fp(fd), 26, SEEK_SET)) != 26)
223        goto err;
224
225    if (!(c = cram_read_container(fd)))
226        goto err;
227
228    // Version 2.1 has a single uncompressed block which is nul
229    // terminated with many nuls to permit growth.
230    //
231    // So load old block and keep all contents identical bar the
232    // header text itself
233    if (!(b = cram_read_block(fd)))
234        goto err;
235
236    if (cram_block_get_uncomp_size(b) < header_len+4) {
237        fprintf(stderr, "New header will not fit. Use non-inplace version (%d > %d)\n",
238                header_len+4, cram_block_get_uncomp_size(b));
239        ret = -2;
240        goto err;
241    }
242
243    cram_block_set_offset(b, 0);   // rewind block
244    int32_put_blk(b, header_len);
245    cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
246    // Zero the remaining block
247    memset((char *)cram_block_get_data(b)+cram_block_get_offset(b), 0,
248           cram_block_get_uncomp_size(b) - cram_block_get_offset(b));
249    // Make sure all sizes and byte-offsets are consistent after memset
250    cram_block_set_offset(b, cram_block_get_uncomp_size(b));
251    cram_block_set_comp_size(b, cram_block_get_uncomp_size(b));
252
253    if (hseek(cram_fd_get_fp(fd), start, SEEK_SET) != start)
254        goto err;
255
256    if (cram_write_container(fd, c) == -1)
257        goto err;
258
259    if (cram_write_block(fd, b) == -1)
260        goto err;
261
262    ret = 0;
263 err:
264    if (c) cram_free_container(c);
265    if (b) cram_free_block(b);
266    if (cram_h) sam_hdr_destroy(cram_h);
267
268    return ret;
269}
270
271
272/*
273 * Reads a version 3 CRAM file and replaces the header in-place,
274 * provided the header is small enough to fit without growing the
275 * entire file.
276 *
277 * Version 3 format has a SAM header held as an (optionally)
278 * compressed block within the header container.  Additional
279 * uncompressed blocks or simply unallocated space (the difference
280 * between total block sizes and the container size) are used to
281 * provide room for growth or contraction of the compressed header.
282 *
283 * Returns 0 on success;
284 *        -1 on general failure;
285 *        -2 on failure due to insufficient size
286 */
287int cram_reheader_inplace3(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
288                          int no_pg)
289{
290    cram_container *c = NULL;
291    cram_block *b = NULL;
292    sam_hdr_t *cram_h = NULL;
293    off_t start, sz, end;
294    int container_sz, max_container_sz;
295    char *buf = NULL;
296    int ret = -1;
297    if (!h)
298        goto err;
299
300    if (cram_major_vers(fd) < 2 ||
301        cram_major_vers(fd) > 3) {
302        fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
303                cram_major_vers(fd));
304        goto err;
305    }
306
307    cram_h = sam_hdr_dup(h);
308    if (!cram_h)
309        goto err;
310
311    if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
312                                 arg_list ? "CL": NULL,
313                                 arg_list ? arg_list : NULL,
314                                 NULL))
315        goto err;
316
317    int header_len = sam_hdr_length(cram_h);
318    /* Fix M5 strings? Maybe out of scope for this tool */
319
320    // Find current size of SAM header block
321    if ((start = hseek(cram_fd_get_fp(fd), 26, SEEK_SET)) != 26)
322        goto err;
323
324    if (!(c = cram_read_container(fd)))
325        goto err;
326
327    // +5 allows num_landmarks to increase from 0 to 1 (Cramtools)
328    max_container_sz = cram_container_size(c)+5;
329
330    sz = htell(cram_fd_get_fp(fd)) + cram_container_get_length(c) - start;
331    end = htell(cram_fd_get_fp(fd)) + cram_container_get_length(c);
332
333    // We force 1 block instead of (optionally) 2.  C CRAM
334    // implementations for v3 were writing 1 compressed block followed
335    // by 1 uncompressed block.  However this is tricky to deal with
336    // as changing block sizes can mean the block header also changes
337    // size due to itf8 and variable size integers.
338    //
339    // If we had 1 block, this doesn't change anything.
340    // If we had 2 blocks, the new container header will be smaller by
341    // 1+ bytes, requiring the cram_container_get_length(c) to be larger in value.
342    // However this is an int32 instead of itf8 so the container
343    // header structure stays the same size.  This means we can always
344    // reduce the number of blocks without running into size problems.
345    cram_container_set_num_blocks(c, 1);
346    int32_t *landmark;
347    int32_t num_landmarks;
348    landmark = cram_container_get_landmarks(c, &num_landmarks);
349    if (num_landmarks && landmark) {
350        num_landmarks = 1;
351        landmark[0] = 0;
352    } else {
353        num_landmarks = 0;
354    }
355    cram_container_set_landmarks(c, num_landmarks, landmark);
356
357    buf = malloc(max_container_sz);
358    container_sz = max_container_sz;
359    if (cram_store_container(fd, c, buf, &container_sz) != 0)
360        goto err;
361
362    if (!buf)
363        goto err;
364
365    // Proposed new length, but changing cram_container_get_length(c) may change the
366    // container_sz and thus the remainder (cram_container_get_length(c) itself).
367    cram_container_set_length(c, sz - container_sz);
368
369    int old_container_sz = container_sz;
370    container_sz = max_container_sz;
371    if (cram_store_container(fd, c, buf, &container_sz) != 0)
372        goto err;
373
374    if (old_container_sz != container_sz) {
375        fprintf(stderr, "Quirk of fate makes this troublesome! "
376                "Please use non-inplace version.\n");
377        goto err;
378    }
379
380
381
382    // Version 3.0 supports compressed header
383    b = cram_new_block(FILE_HEADER, 0);
384    int32_put_blk(b, header_len);
385    cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
386    cram_block_update_size(b);
387
388    cram_compress_block(fd, b, NULL, -1, 9);
389
390    if (hseek(cram_fd_get_fp(fd), 26, SEEK_SET) != 26)
391        goto err;
392
393    if (cram_block_size(b) > cram_container_get_length(c)) {
394        fprintf(stderr, "New header will not fit. Use non-inplace version"
395                " (%d > %d)\n",
396                (int)cram_block_size(b), cram_container_get_length(c));
397        ret = -2;
398        goto err;
399    }
400
401    if (cram_write_container(fd, c) == -1)
402        goto err;
403
404    if (cram_write_block(fd, b) == -1)
405        goto err;
406
407    // Blank out the remainder
408    int rsz = end - htell(cram_fd_get_fp(fd));
409    assert(rsz >= 0);
410    if (rsz) {
411        char *rem = calloc(1, rsz);
412        ret = hwrite(cram_fd_get_fp(fd), rem, rsz) == rsz ? 0 : -1;
413        free(rem);
414    }
415
416 err:
417    if (c) cram_free_container(c);
418    if (buf) free(buf);
419    if (b) cram_free_block(b);
420    if (cram_h) sam_hdr_destroy(cram_h);
421
422    return ret;
423}
424
425int cram_reheader_inplace(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
426                         int no_pg)
427{
428    switch (cram_major_vers(fd)) {
429    case 2: return cram_reheader_inplace2(fd, h, arg_list, no_pg);
430    case 3: return cram_reheader_inplace3(fd, h, arg_list, no_pg);
431    default:
432        fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
433                cram_major_vers(fd));
434        return -1;
435    }
436}
437
438static void usage(FILE *fp, int ret) {
439    fprintf(fp,
440           "Usage: samtools reheader [-P] in.header.sam in.bam > out.bam\n"
441           "   or  samtools reheader [-P] -i in.header.sam file.cram\n"
442           "   or  samtools reheader -c CMD in.bam\n"
443           "   or  samtools reheader -c CMD in.cram\n"
444           "\n"
445           "Options:\n"
446           "    -P, --no-PG         Do not generate a @PG header line.\n"
447           "    -i, --in-place      Modify the CRAM file directly, if possible.\n"
448           "                        (Defaults to outputting to stdout.)\n"
449           "    -c, --command CMD   Pass the header in SAM format to external program CMD.\n");
450    exit(ret);
451}
452
453static sam_hdr_t* external_reheader(samFile* in, const char* external) {
454    char *command = NULL;
455    sam_hdr_t* h = NULL;
456    sam_hdr_t* ih = sam_hdr_read(in);
457    if (ih == NULL) {
458        fprintf(stderr, "[%s] failed to read the header for '%s'.\n", __func__, in->fn);
459        return NULL;
460    }
461    char tmp_fn[] = "reheaderXXXXXX";
462    int tmp_fd = mkstemp(tmp_fn);
463    if (tmp_fd < 0) {
464        print_error_errno("reheader", "fail to open temp file '%s'", tmp_fn);
465        return NULL;
466    }
467    hFILE* tmp_hf = hdopen(tmp_fd, "w");
468    if (!tmp_hf) {
469        fprintf(stderr, "[%s] failed to convert to hFILE.\n", __func__);
470        goto cleanup;
471    }
472    samFile* tmp_sf = hts_hopen(tmp_hf, tmp_fn, "w");
473    if (!tmp_sf) {
474        fprintf(stderr, "[%s] failed to convert to samFile.\n", __func__);
475        goto cleanup;
476    }
477    if (-1 == sam_hdr_write(tmp_sf, ih)) {
478        fprintf(stderr, "[%s] failed to write the header to the temp file.\n", __func__);
479        goto cleanup;
480    }
481    sam_close(tmp_sf);
482    sam_hdr_destroy(ih);
483    int comm_len = strlen(external) + strlen(tmp_fn) + 8;
484    command = calloc(comm_len, 1);
485    if (!command || snprintf(command, comm_len, "( %s ) < %s", external, tmp_fn) != comm_len - 1) {
486        fprintf(stderr, "[%s] failed to create command string.\n", __func__);
487        goto cleanup;
488    }
489    FILE* nh = popen(command, "r");
490    if (!nh) {
491        print_error_errno("reheader", "[%s] failed to run external command '%s'.\n", __func__, command);
492        goto cleanup;
493    }
494
495    int nh_fd = dup(fileno(nh));
496    if (nh_fd < 0) {
497        fprintf(stderr, "[%s] failed to get the file descriptor.\n", __func__);
498        goto cleanup;
499    }
500    hFILE* nh_hf = hdopen(nh_fd, "r");
501    if (!nh_hf) {
502        fprintf(stderr, "[%s] failed to convert to hFILE.\n", __func__);
503        goto cleanup;
504    }
505    samFile* nh_sf = hts_hopen(nh_hf, tmp_fn, "r");
506    if (!nh_sf) {
507        fprintf(stderr, "[%s] failed to convert to samFile.\n", __func__);
508        goto cleanup;
509    }
510
511    h = sam_hdr_read(nh_sf);
512    sam_close(nh_sf);
513    if (h == NULL) {
514        fprintf(stderr, "[%s] failed to read the header from the temp file.\n", __func__);
515    }
516    int res = pclose(nh);
517    if (res != 0) {
518        if (res < 0) {
519            print_error_errno("reheader",
520                              "Error on closing pipe from command '%s'.\n",
521                              command);
522        } else {
523            print_error("reheader",
524                        "Non-zero exit code returned by command '%s'\n",
525                        command);
526        }
527        if (h) sam_hdr_destroy(h);
528        h = NULL;
529    }
530cleanup:
531    free(command);
532    if (unlink(tmp_fn) != 0) {
533        print_error_errno("reheader", "failed to remove the temp file '%s'", tmp_fn);
534    }
535
536    return h;
537}
538
539int main_reheader(int argc, char *argv[])
540{
541    int inplace = 0, r, no_pg = 0, c, skip_header = 0;
542    sam_hdr_t *h;
543    samFile *in;
544    char *arg_list = NULL, *external = NULL;
545
546    static const struct option lopts[] = {
547        {"help",     no_argument, NULL, 'h'},
548        {"in-place", no_argument, NULL, 'i'},
549        {"no-PG",    no_argument, NULL, 'P'},
550        {"command", required_argument, NULL, 'c'},
551        {NULL, 0, NULL, 0}
552    };
553
554    while ((c = getopt_long(argc, argv, "hiPc:", lopts, NULL)) >= 0) {
555        switch (c) {
556        case 'P': no_pg = 1; break;
557        case 'i': inplace = 1; break;
558        case 'c': external = optarg; break;
559        case 'h': usage(stdout, 0); break;
560        default:
561            fprintf(stderr, "Invalid option '%c'\n", c);
562            usage(stderr, 1);
563        }
564    }
565
566    if ((argc - optind != 2 || external) && (argc - optind != 1 || !external))
567        usage(stderr, 1);
568
569    if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
570         print_error("reheader", "failed to create arg_list");
571         return 1;
572     }
573
574    if (external) {
575        skip_header = 1;
576        in = sam_open(argv[optind], inplace?"r+":"r");
577        if (in == 0) {
578            print_error_errno("reheader", "fail to open file '%s'", argv[optind]);
579            return 1;
580        }
581
582        h = external_reheader(in, external);
583        if (h == NULL) {
584            fprintf(stderr, "[%s] failed to read the header from '%s'.\n", __func__, external);
585            sam_close(in);
586            return 1;
587        }
588    } else { // read the header from a separate file
589        samFile *fph = sam_open(argv[optind], "r");
590        if (fph == 0) {
591            print_error_errno("reheader", "fail to read the header from '%s'", argv[optind]);
592            return 1;
593        }
594        h = sam_hdr_read(fph);
595        sam_close(fph);
596        if (h == NULL) {
597            fprintf(stderr, "[%s] failed to read the header for '%s'.\n",
598                    __func__, argv[1]);
599            return 1;
600        }
601        in = sam_open(argv[optind+1], inplace?"r+":"r");
602        if (in == 0) {
603            print_error_errno("reheader", "fail to open file '%s'", argv[optind+1]);
604            return 1;
605        }
606    }
607
608    if (hts_get_format(in)->format == bam) {
609        if (inplace) {
610            print_error("reheader", "cannot reheader BAM '%s' in-place", argv[optind+1]);
611            r = -1;
612        } else {
613            r = bam_reheader(in->fp.bgzf, h, fileno(stdout), arg_list, no_pg, skip_header);
614        }
615    } else if (hts_get_format(in)->format == cram) {
616        if (inplace)
617            r = cram_reheader_inplace(in->fp.cram, h, arg_list, no_pg);
618        else
619            r = cram_reheader(in->fp.cram, h, arg_list, no_pg);
620    } else {
621        print_error("reheader", "input file '%s' must be BAM or CRAM", argv[optind+1]);
622        r = -1;
623    }
624
625    if (sam_close(in) != 0)
626        r = -1;
627
628    sam_hdr_destroy(h);
629
630    if (arg_list)
631        free(arg_list);
632
633    return -r;
634}