upstream-dependencies/samtools/bam_reheader.c
1/* bam_reheader.c -- reheader subcommand.
2
3 Copyright (C) 2010 Broad Institute.
4 Copyright (C) 2012-2019 Genome Research Ltd.
5
6 Author: Heng Li <lh3@sanger.ac.uk>
7
8Permission is hereby granted, free of charge, to any person obtaining a copy
9of this software and associated documentation files (the "Software"), to deal
10in the Software without restriction, including without limitation the rights
11to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12copies of the Software, and to permit persons to whom the Software is
13furnished to do so, subject to the following conditions:
14
15The above copyright notice and this permission notice shall be included in
16all copies or substantial portions of the Software.
17
18THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24DEALINGS IN THE SOFTWARE. */
25
26#include <config.h>
27
28#include <stdio.h>
29#include <stdlib.h>
30#include <assert.h>
31#include <getopt.h>
32#include <unistd.h>
33
34#include "htslib/bgzf.h"
35#include "htslib/sam.h"
36#include "htslib/hfile.h"
37#include "htslib/cram.h"
38#include "samtools.h"
39
40#define BUF_SIZE 0x10000
41
42/*
43 * Reads a file and outputs a new BAM file to fd with 'h' replaced as
44 * the header. No checks are made to the validity.
45 */
46int bam_reheader(BGZF *in, sam_hdr_t *h, int fd,
47 const char *arg_list, int no_pg, int skip_header)
48{
49 BGZF *fp = NULL;
50 ssize_t len;
51 uint8_t *buf = NULL;
52 sam_hdr_t *tmp;
53 if (!h)
54 return -1;
55
56 if (in->is_write) return -1;
57 buf = malloc(BUF_SIZE);
58 if (!buf) {
59 fprintf(stderr, "Out of memory\n");
60 return -1;
61 }
62
63 if (!skip_header) {
64 if ((tmp = bam_hdr_read(in)) == NULL) {
65 fprintf(stderr, "Couldn't read header\n");
66 goto fail;
67 }
68 sam_hdr_destroy(tmp);
69 }
70
71 fp = bgzf_fdopen(fd, "w");
72 if (!fp) {
73 print_error_errno("reheader", "Couldn't open output file");
74 goto fail;
75 }
76
77 if (!no_pg && sam_hdr_add_pg(h, "samtools",
78 "VN", samtools_version(),
79 arg_list ? "CL": NULL,
80 arg_list ? arg_list : NULL,
81 NULL) != 0)
82 goto fail;
83
84 if (bam_hdr_write(fp, h) < 0) {
85 print_error_errno("reheader", "Couldn't write header");
86 goto fail;
87 }
88 if (in->block_offset < in->block_length) {
89 if (bgzf_write(fp, (char *)in->uncompressed_block + in->block_offset, in->block_length - in->block_offset) < 0) goto write_fail;
90 if (bgzf_flush(fp) < 0) goto write_fail;
91 }
92 while ((len = bgzf_raw_read(in, buf, BUF_SIZE)) > 0) {
93 if (bgzf_raw_write(fp, buf, len) < 0) goto write_fail;
94 }
95 if (len < 0) {
96 fprintf(stderr, "[%s] Error reading input file\n", __func__);
97 goto fail;
98 }
99 free(buf);
100 fp->block_offset = in->block_offset = 0;
101 if (bgzf_close(fp) < 0) {
102 fprintf(stderr, "[%s] Error closing output file\n", __func__);
103 return -1;
104 }
105 return 0;
106
107 write_fail:
108 print_error_errno("reheader", "Error writing to output file");
109 fail:
110 bgzf_close(fp);
111 free(buf);
112 return -1;
113}
114
115/*
116 * Reads a file and outputs a new CRAM file to stdout with 'h'
117 * replaced as the header. No checks are made to the validity.
118 *
119 * FIXME: error checking
120 */
121int cram_reheader(cram_fd *in, sam_hdr_t *h, const char *arg_list, int no_pg)
122{
123 htsFile *h_out = hts_open("-", "wc");
124 cram_fd *out = h_out->fp.cram;
125 cram_container *c = NULL;
126 int ret = -1;
127 if (!h)
128 return ret;
129
130 // Match output version number with input file.
131 char vers[99];
132 sprintf(vers, "%d.%d", cram_major_vers(in), cram_minor_vers(in));
133 cram_set_option(out, CRAM_OPT_VERSION, vers);
134
135 // Attempt to fill out a cram->refs[] array from @SQ headers
136 sam_hdr_t *cram_h = sam_hdr_dup(h);
137 if (!cram_h)
138 return -1;
139 cram_fd_set_header(out, cram_h);
140 if (!no_pg && sam_hdr_add_pg(cram_fd_get_header(out), "samtools",
141 "VN", samtools_version(),
142 arg_list ? "CL": NULL,
143 arg_list ? arg_list : NULL,
144 NULL))
145 goto err;
146
147 if (sam_hdr_write(h_out, cram_h) != 0)
148 goto err;
149 cram_set_option(out, CRAM_OPT_REFERENCE, NULL);
150
151 while ((c = cram_read_container(in))) {
152 int32_t i, num_blocks = cram_container_get_num_blocks(c);
153 if (cram_write_container(out, c) != 0)
154 goto err;
155
156 for (i = 0; i < num_blocks; i++) {
157 cram_block *blk = cram_read_block(in);
158 if (!blk || cram_write_block(out, blk) != 0) {
159 if (blk) cram_free_block(blk);
160 goto err;
161 }
162 cram_free_block(blk);
163 }
164 cram_free_container(c);
165 }
166
167 ret = 0;
168
169 err:
170 if (hts_close(h_out) != 0)
171 ret = -1;
172
173 return ret;
174}
175
176
177
178/*
179 * Reads a version 2 CRAM file and replaces the header in-place,
180 * provided the header is small enough to fit without growing the
181 * entire file.
182 *
183 * Version 2 format has an uncompressed SAM header with multiple nul
184 * termination bytes to permit inline header editing.
185 *
186 * Returns 0 on success;
187 * -1 on general failure;
188 * -2 on failure due to insufficient size
189 */
190int cram_reheader_inplace2(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
191 int no_pg)
192{
193 cram_container *c = NULL;
194 cram_block *b = NULL;
195 sam_hdr_t *cram_h = NULL;
196 off_t start;
197 int ret = -1;
198 if (!h)
199 goto err;
200
201 if (cram_major_vers(fd) < 2 ||
202 cram_major_vers(fd) > 3) {
203 fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
204 cram_major_vers(fd));
205 goto err;
206 }
207
208 cram_h = sam_hdr_dup(h);
209 if (!cram_h)
210 goto err;
211
212 if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
213 arg_list ? "CL": NULL,
214 arg_list ? arg_list : NULL,
215 NULL))
216 goto err;
217
218 int header_len = sam_hdr_length(cram_h);
219 /* Fix M5 strings? Maybe out of scope for this tool */
220
221 // Load the existing header
222 if ((start = hseek(cram_fd_get_fp(fd), 26, SEEK_SET)) != 26)
223 goto err;
224
225 if (!(c = cram_read_container(fd)))
226 goto err;
227
228 // Version 2.1 has a single uncompressed block which is nul
229 // terminated with many nuls to permit growth.
230 //
231 // So load old block and keep all contents identical bar the
232 // header text itself
233 if (!(b = cram_read_block(fd)))
234 goto err;
235
236 if (cram_block_get_uncomp_size(b) < header_len+4) {
237 fprintf(stderr, "New header will not fit. Use non-inplace version (%d > %d)\n",
238 header_len+4, cram_block_get_uncomp_size(b));
239 ret = -2;
240 goto err;
241 }
242
243 cram_block_set_offset(b, 0); // rewind block
244 int32_put_blk(b, header_len);
245 cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
246 // Zero the remaining block
247 memset((char *)cram_block_get_data(b)+cram_block_get_offset(b), 0,
248 cram_block_get_uncomp_size(b) - cram_block_get_offset(b));
249 // Make sure all sizes and byte-offsets are consistent after memset
250 cram_block_set_offset(b, cram_block_get_uncomp_size(b));
251 cram_block_set_comp_size(b, cram_block_get_uncomp_size(b));
252
253 if (hseek(cram_fd_get_fp(fd), start, SEEK_SET) != start)
254 goto err;
255
256 if (cram_write_container(fd, c) == -1)
257 goto err;
258
259 if (cram_write_block(fd, b) == -1)
260 goto err;
261
262 ret = 0;
263 err:
264 if (c) cram_free_container(c);
265 if (b) cram_free_block(b);
266 if (cram_h) sam_hdr_destroy(cram_h);
267
268 return ret;
269}
270
271
272/*
273 * Reads a version 3 CRAM file and replaces the header in-place,
274 * provided the header is small enough to fit without growing the
275 * entire file.
276 *
277 * Version 3 format has a SAM header held as an (optionally)
278 * compressed block within the header container. Additional
279 * uncompressed blocks or simply unallocated space (the difference
280 * between total block sizes and the container size) are used to
281 * provide room for growth or contraction of the compressed header.
282 *
283 * Returns 0 on success;
284 * -1 on general failure;
285 * -2 on failure due to insufficient size
286 */
287int cram_reheader_inplace3(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
288 int no_pg)
289{
290 cram_container *c = NULL;
291 cram_block *b = NULL;
292 sam_hdr_t *cram_h = NULL;
293 off_t start, sz, end;
294 int container_sz, max_container_sz;
295 char *buf = NULL;
296 int ret = -1;
297 if (!h)
298 goto err;
299
300 if (cram_major_vers(fd) < 2 ||
301 cram_major_vers(fd) > 3) {
302 fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
303 cram_major_vers(fd));
304 goto err;
305 }
306
307 cram_h = sam_hdr_dup(h);
308 if (!cram_h)
309 goto err;
310
311 if (!no_pg && sam_hdr_add_pg(cram_h, "samtools", "VN", samtools_version(),
312 arg_list ? "CL": NULL,
313 arg_list ? arg_list : NULL,
314 NULL))
315 goto err;
316
317 int header_len = sam_hdr_length(cram_h);
318 /* Fix M5 strings? Maybe out of scope for this tool */
319
320 // Find current size of SAM header block
321 if ((start = hseek(cram_fd_get_fp(fd), 26, SEEK_SET)) != 26)
322 goto err;
323
324 if (!(c = cram_read_container(fd)))
325 goto err;
326
327 // +5 allows num_landmarks to increase from 0 to 1 (Cramtools)
328 max_container_sz = cram_container_size(c)+5;
329
330 sz = htell(cram_fd_get_fp(fd)) + cram_container_get_length(c) - start;
331 end = htell(cram_fd_get_fp(fd)) + cram_container_get_length(c);
332
333 // We force 1 block instead of (optionally) 2. C CRAM
334 // implementations for v3 were writing 1 compressed block followed
335 // by 1 uncompressed block. However this is tricky to deal with
336 // as changing block sizes can mean the block header also changes
337 // size due to itf8 and variable size integers.
338 //
339 // If we had 1 block, this doesn't change anything.
340 // If we had 2 blocks, the new container header will be smaller by
341 // 1+ bytes, requiring the cram_container_get_length(c) to be larger in value.
342 // However this is an int32 instead of itf8 so the container
343 // header structure stays the same size. This means we can always
344 // reduce the number of blocks without running into size problems.
345 cram_container_set_num_blocks(c, 1);
346 int32_t *landmark;
347 int32_t num_landmarks;
348 landmark = cram_container_get_landmarks(c, &num_landmarks);
349 if (num_landmarks && landmark) {
350 num_landmarks = 1;
351 landmark[0] = 0;
352 } else {
353 num_landmarks = 0;
354 }
355 cram_container_set_landmarks(c, num_landmarks, landmark);
356
357 buf = malloc(max_container_sz);
358 container_sz = max_container_sz;
359 if (cram_store_container(fd, c, buf, &container_sz) != 0)
360 goto err;
361
362 if (!buf)
363 goto err;
364
365 // Proposed new length, but changing cram_container_get_length(c) may change the
366 // container_sz and thus the remainder (cram_container_get_length(c) itself).
367 cram_container_set_length(c, sz - container_sz);
368
369 int old_container_sz = container_sz;
370 container_sz = max_container_sz;
371 if (cram_store_container(fd, c, buf, &container_sz) != 0)
372 goto err;
373
374 if (old_container_sz != container_sz) {
375 fprintf(stderr, "Quirk of fate makes this troublesome! "
376 "Please use non-inplace version.\n");
377 goto err;
378 }
379
380
381
382 // Version 3.0 supports compressed header
383 b = cram_new_block(FILE_HEADER, 0);
384 int32_put_blk(b, header_len);
385 cram_block_append(b, (void *)sam_hdr_str(cram_h), header_len);
386 cram_block_update_size(b);
387
388 cram_compress_block(fd, b, NULL, -1, 9);
389
390 if (hseek(cram_fd_get_fp(fd), 26, SEEK_SET) != 26)
391 goto err;
392
393 if (cram_block_size(b) > cram_container_get_length(c)) {
394 fprintf(stderr, "New header will not fit. Use non-inplace version"
395 " (%d > %d)\n",
396 (int)cram_block_size(b), cram_container_get_length(c));
397 ret = -2;
398 goto err;
399 }
400
401 if (cram_write_container(fd, c) == -1)
402 goto err;
403
404 if (cram_write_block(fd, b) == -1)
405 goto err;
406
407 // Blank out the remainder
408 int rsz = end - htell(cram_fd_get_fp(fd));
409 assert(rsz >= 0);
410 if (rsz) {
411 char *rem = calloc(1, rsz);
412 ret = hwrite(cram_fd_get_fp(fd), rem, rsz) == rsz ? 0 : -1;
413 free(rem);
414 }
415
416 err:
417 if (c) cram_free_container(c);
418 if (buf) free(buf);
419 if (b) cram_free_block(b);
420 if (cram_h) sam_hdr_destroy(cram_h);
421
422 return ret;
423}
424
425int cram_reheader_inplace(cram_fd *fd, sam_hdr_t *h, const char *arg_list,
426 int no_pg)
427{
428 switch (cram_major_vers(fd)) {
429 case 2: return cram_reheader_inplace2(fd, h, arg_list, no_pg);
430 case 3: return cram_reheader_inplace3(fd, h, arg_list, no_pg);
431 default:
432 fprintf(stderr, "[%s] unsupported CRAM version %d\n", __func__,
433 cram_major_vers(fd));
434 return -1;
435 }
436}
437
438static void usage(FILE *fp, int ret) {
439 fprintf(fp,
440 "Usage: samtools reheader [-P] in.header.sam in.bam > out.bam\n"
441 " or samtools reheader [-P] -i in.header.sam file.cram\n"
442 " or samtools reheader -c CMD in.bam\n"
443 " or samtools reheader -c CMD in.cram\n"
444 "\n"
445 "Options:\n"
446 " -P, --no-PG Do not generate a @PG header line.\n"
447 " -i, --in-place Modify the CRAM file directly, if possible.\n"
448 " (Defaults to outputting to stdout.)\n"
449 " -c, --command CMD Pass the header in SAM format to external program CMD.\n");
450 exit(ret);
451}
452
453static sam_hdr_t* external_reheader(samFile* in, const char* external) {
454 char *command = NULL;
455 sam_hdr_t* h = NULL;
456 sam_hdr_t* ih = sam_hdr_read(in);
457 if (ih == NULL) {
458 fprintf(stderr, "[%s] failed to read the header for '%s'.\n", __func__, in->fn);
459 return NULL;
460 }
461 char tmp_fn[] = "reheaderXXXXXX";
462 int tmp_fd = mkstemp(tmp_fn);
463 if (tmp_fd < 0) {
464 print_error_errno("reheader", "fail to open temp file '%s'", tmp_fn);
465 return NULL;
466 }
467 hFILE* tmp_hf = hdopen(tmp_fd, "w");
468 if (!tmp_hf) {
469 fprintf(stderr, "[%s] failed to convert to hFILE.\n", __func__);
470 goto cleanup;
471 }
472 samFile* tmp_sf = hts_hopen(tmp_hf, tmp_fn, "w");
473 if (!tmp_sf) {
474 fprintf(stderr, "[%s] failed to convert to samFile.\n", __func__);
475 goto cleanup;
476 }
477 if (-1 == sam_hdr_write(tmp_sf, ih)) {
478 fprintf(stderr, "[%s] failed to write the header to the temp file.\n", __func__);
479 goto cleanup;
480 }
481 sam_close(tmp_sf);
482 sam_hdr_destroy(ih);
483 int comm_len = strlen(external) + strlen(tmp_fn) + 8;
484 command = calloc(comm_len, 1);
485 if (!command || snprintf(command, comm_len, "( %s ) < %s", external, tmp_fn) != comm_len - 1) {
486 fprintf(stderr, "[%s] failed to create command string.\n", __func__);
487 goto cleanup;
488 }
489 FILE* nh = popen(command, "r");
490 if (!nh) {
491 print_error_errno("reheader", "[%s] failed to run external command '%s'.\n", __func__, command);
492 goto cleanup;
493 }
494
495 int nh_fd = dup(fileno(nh));
496 if (nh_fd < 0) {
497 fprintf(stderr, "[%s] failed to get the file descriptor.\n", __func__);
498 goto cleanup;
499 }
500 hFILE* nh_hf = hdopen(nh_fd, "r");
501 if (!nh_hf) {
502 fprintf(stderr, "[%s] failed to convert to hFILE.\n", __func__);
503 goto cleanup;
504 }
505 samFile* nh_sf = hts_hopen(nh_hf, tmp_fn, "r");
506 if (!nh_sf) {
507 fprintf(stderr, "[%s] failed to convert to samFile.\n", __func__);
508 goto cleanup;
509 }
510
511 h = sam_hdr_read(nh_sf);
512 sam_close(nh_sf);
513 if (h == NULL) {
514 fprintf(stderr, "[%s] failed to read the header from the temp file.\n", __func__);
515 }
516 int res = pclose(nh);
517 if (res != 0) {
518 if (res < 0) {
519 print_error_errno("reheader",
520 "Error on closing pipe from command '%s'.\n",
521 command);
522 } else {
523 print_error("reheader",
524 "Non-zero exit code returned by command '%s'\n",
525 command);
526 }
527 if (h) sam_hdr_destroy(h);
528 h = NULL;
529 }
530cleanup:
531 free(command);
532 if (unlink(tmp_fn) != 0) {
533 print_error_errno("reheader", "failed to remove the temp file '%s'", tmp_fn);
534 }
535
536 return h;
537}
538
539int main_reheader(int argc, char *argv[])
540{
541 int inplace = 0, r, no_pg = 0, c, skip_header = 0;
542 sam_hdr_t *h;
543 samFile *in;
544 char *arg_list = NULL, *external = NULL;
545
546 static const struct option lopts[] = {
547 {"help", no_argument, NULL, 'h'},
548 {"in-place", no_argument, NULL, 'i'},
549 {"no-PG", no_argument, NULL, 'P'},
550 {"command", required_argument, NULL, 'c'},
551 {NULL, 0, NULL, 0}
552 };
553
554 while ((c = getopt_long(argc, argv, "hiPc:", lopts, NULL)) >= 0) {
555 switch (c) {
556 case 'P': no_pg = 1; break;
557 case 'i': inplace = 1; break;
558 case 'c': external = optarg; break;
559 case 'h': usage(stdout, 0); break;
560 default:
561 fprintf(stderr, "Invalid option '%c'\n", c);
562 usage(stderr, 1);
563 }
564 }
565
566 if ((argc - optind != 2 || external) && (argc - optind != 1 || !external))
567 usage(stderr, 1);
568
569 if (!no_pg && !(arg_list = stringify_argv(argc+1, argv-1))) {
570 print_error("reheader", "failed to create arg_list");
571 return 1;
572 }
573
574 if (external) {
575 skip_header = 1;
576 in = sam_open(argv[optind], inplace?"r+":"r");
577 if (in == 0) {
578 print_error_errno("reheader", "fail to open file '%s'", argv[optind]);
579 return 1;
580 }
581
582 h = external_reheader(in, external);
583 if (h == NULL) {
584 fprintf(stderr, "[%s] failed to read the header from '%s'.\n", __func__, external);
585 sam_close(in);
586 return 1;
587 }
588 } else { // read the header from a separate file
589 samFile *fph = sam_open(argv[optind], "r");
590 if (fph == 0) {
591 print_error_errno("reheader", "fail to read the header from '%s'", argv[optind]);
592 return 1;
593 }
594 h = sam_hdr_read(fph);
595 sam_close(fph);
596 if (h == NULL) {
597 fprintf(stderr, "[%s] failed to read the header for '%s'.\n",
598 __func__, argv[1]);
599 return 1;
600 }
601 in = sam_open(argv[optind+1], inplace?"r+":"r");
602 if (in == 0) {
603 print_error_errno("reheader", "fail to open file '%s'", argv[optind+1]);
604 return 1;
605 }
606 }
607
608 if (hts_get_format(in)->format == bam) {
609 if (inplace) {
610 print_error("reheader", "cannot reheader BAM '%s' in-place", argv[optind+1]);
611 r = -1;
612 } else {
613 r = bam_reheader(in->fp.bgzf, h, fileno(stdout), arg_list, no_pg, skip_header);
614 }
615 } else if (hts_get_format(in)->format == cram) {
616 if (inplace)
617 r = cram_reheader_inplace(in->fp.cram, h, arg_list, no_pg);
618 else
619 r = cram_reheader(in->fp.cram, h, arg_list, no_pg);
620 } else {
621 print_error("reheader", "input file '%s' must be BAM or CRAM", argv[optind+1]);
622 r = -1;
623 }
624
625 if (sam_close(in) != 0)
626 r = -1;
627
628 sam_hdr_destroy(h);
629
630 if (arg_list)
631 free(arg_list);
632
633 return -r;
634}