upstream-dependencies/galaxy-sequence-utils/galaxy_utils/sequence/fastq.py
1# Dan Blankenberg
2
3import bz2
4import gzip
5import math
6import string
7
8from . import transform
9from .fasta import fastaSequence
10from .sequence import SequencingRead
11
12LETTERS = string.ascii_letters
13
14
15class fastqSequencingRead(SequencingRead):
16 format = 'sanger' # sanger is default
17 ascii_min = 33
18 ascii_max = 126
19 quality_min = 0
20 quality_max = 93
21 score_system = 'phred' # phred or solexa
22 sequence_space = 'base' # base or color
23
24 @classmethod
25 def get_class_by_format(cls, format):
26 assert format in FASTQ_FORMATS, f'Unknown format type specified: {format}'
27 return FASTQ_FORMATS[format]
28
29 @classmethod
30 def convert_score_phred_to_solexa(cls, decimal_score_list):
31 def phred_to_solexa(score):
32 if score <= 0: # can't take log10( 1 - 1 ); make <= 0 into -5
33 return -5
34 return int(round(10.0 * math.log10(math.pow(10.0, (float(score) / 10.0)) - 1.0)))
35 return [phred_to_solexa(_) for _ in decimal_score_list]
36
37 @classmethod
38 def convert_score_solexa_to_phred(cls, decimal_score_list):
39 def solexa_to_phred(score):
40 return int(round(10.0 * math.log10(math.pow(10.0, (float(score) / 10.0)) + 1.0)))
41 return [solexa_to_phred(_) for _ in decimal_score_list]
42
43 @classmethod
44 def restrict_scores_to_valid_range(cls, decimal_score_list):
45 def restrict_score(score):
46 return max(min(score, cls.quality_max), cls.quality_min)
47 return [restrict_score(_) for _ in decimal_score_list]
48
49 @classmethod
50 def transform_scores_to_valid_range(cls, decimal_score_list):
51 cls_quality_max = cls.quality_max
52 cls_quality_min = cls.quality_min
53 for i in range(len(decimal_score_list)):
54 score = decimal_score_list[i]
55 if score > cls_quality_max:
56 transformed_score = cls_quality_max
57 elif score < cls_quality_min:
58 transformed_score = cls_quality_min
59 else:
60 transformed_score = score
61 decimal_score_list[i] = str(transformed_score)
62
63 @classmethod
64 def transform_scores_to_valid_range_ascii(cls, decimal_score_list):
65 cls_quality_max = cls.quality_max
66 cls_quality_min = cls.quality_min
67 to_quality = cls.ascii_min - cls.quality_min
68 for i in range(len(decimal_score_list)):
69 score = decimal_score_list[i]
70 if score > cls_quality_max:
71 transformed_score = cls_quality_max
72 elif score < cls_quality_min:
73 transformed_score = cls_quality_min
74 else:
75 transformed_score = score
76 transformed_score = chr(transformed_score + to_quality)
77 decimal_score_list[i] = transformed_score
78
79 @classmethod
80 def convert_base_to_color_space(cls, sequence):
81 return cls.color_space_converter.to_color_space(sequence)
82
83 @classmethod
84 def convert_color_to_base_space(cls, sequence):
85 return cls.color_space_converter.to_base_space(sequence)
86
87 def is_ascii_encoded(self):
88 # as per fastq definition only decimal quality strings can have spaces (and TABs for our purposes) in them (and must have a trailing space)
89 if ' ' in self.quality:
90 return False
91 if '\t' in self.quality:
92 return False
93 return True
94
95 def get_ascii_quality_scores(self):
96 if self.is_ascii_encoded():
97 return list(self.quality)
98 else:
99 quality = self.quality.rstrip() # decimal scores should have a trailing space
100 if quality:
101 try:
102 to_quality = self.ascii_min - self.quality_min
103 return [chr(int(val) + to_quality) for val in quality.split()]
104 except ValueError as e:
105 raise ValueError(f'Error Parsing quality String. ASCII quality strings cannot contain spaces ({self.quality}): {e}')
106 else:
107 return []
108
109 def get_ascii_quality_scores_len(self):
110 """
111 Compute ascii quality score length, without generating relatively
112 expensive qualty score array.
113 """
114 if self.is_ascii_encoded():
115 return len(self.quality)
116 else:
117 quality = self.quality.rstrip()
118 if quality:
119 try:
120 return len(quality.split())
121 except ValueError as e:
122 raise ValueError(f'Error Parsing quality String. ASCII quality strings cannot contain spaces ({self.quality}): {e}')
123 else:
124 return 0
125
126 def get_decimal_quality_scores(self):
127 return self.__get_decimal_quality_scores(self.is_ascii_encoded())
128
129 def __get_decimal_quality_scores(self, ascii):
130 if ascii:
131 to_quality = self.quality_min - self.ascii_min
132 return [ord(val) + to_quality for val in self.quality]
133 else:
134 quality = self.quality.rstrip() # decimal scores should have a trailing space
135 if quality:
136 return [int(val) for val in quality.split() if val.strip()]
137 else:
138 return []
139
140 def convert_read_to_format(self, format, force_quality_encoding=None):
141 assert format in FASTQ_FORMATS, f'Unknown format type specified: {format}'
142 assert force_quality_encoding in [None, 'ascii', 'decimal'], f'Invalid force_quality_encoding: {force_quality_encoding}'
143 new_class = FASTQ_FORMATS[format]
144 new_read = new_class()
145 new_read.identifier = self.identifier
146 if self.sequence_space == new_class.sequence_space:
147 new_read.sequence = self.sequence
148 else:
149 if self.sequence_space == 'base':
150 new_read.sequence = self.convert_base_to_color_space(self.sequence)
151 else:
152 new_read.sequence = self.convert_color_to_base_space(self.sequence)
153 new_read.description = self.description
154 is_ascii = self.is_ascii_encoded()
155 if self.score_system != new_read.score_system:
156 if self.score_system == 'phred':
157 score_list = self.convert_score_phred_to_solexa(self.__get_decimal_quality_scores(is_ascii))
158 else:
159 score_list = self.convert_score_solexa_to_phred(self.__get_decimal_quality_scores(is_ascii))
160 else:
161 score_list = self.__get_decimal_quality_scores(is_ascii)
162 if force_quality_encoding is None:
163 if is_ascii:
164 new_encoding = 'ascii'
165 else:
166 new_encoding = 'decimal'
167 else:
168 new_encoding = force_quality_encoding
169 if new_encoding == 'ascii':
170 new_class.transform_scores_to_valid_range_ascii(score_list)
171 new_read.quality = "".join(score_list)
172 else: # decimal
173 new_class.transform_scores_to_valid_range(score_list)
174 new_read.quality = " ".join(score_list) + " " # need trailing space to be valid decimal fastq
175 return new_read
176
177 def get_sequence(self):
178 return self.sequence
179
180 def slice(self, left_column_offset, right_column_offset):
181 new_read = fastqSequencingRead.get_class_by_format(self.format)()
182 new_read.identifier = self.identifier
183 new_read.sequence = self.get_sequence()[left_column_offset:right_column_offset]
184 new_read.description = self.description
185 if self.is_ascii_encoded():
186 new_read.quality = self.quality[left_column_offset:right_column_offset]
187 else:
188 quality = [str(_) for _ in self.get_decimal_quality_scores()[left_column_offset:right_column_offset]]
189 if quality:
190 new_read.quality = " ".join(quality) + " "
191 else:
192 new_read.quality = ''
193 return new_read
194
195 def is_valid_format(self):
196 if self.is_ascii_encoded():
197 for val in self.get_ascii_quality_scores():
198 val = ord(val)
199 if val < self.ascii_min or val > self.ascii_max:
200 return False
201 else:
202 for val in self.get_decimal_quality_scores():
203 if val < self.quality_min or val > self.quality_max:
204 return False
205 if not self.is_valid_sequence():
206 return False
207 return True
208
209 def is_valid_sequence(self):
210 for base in self.get_sequence():
211 if base not in self.valid_sequence_list:
212 return False
213 return True
214
215 def insufficient_quality_length(self):
216 return self.get_ascii_quality_scores_len() < len(self.sequence)
217
218 def assert_sequence_quality_lengths(self):
219 qual_len = self.get_ascii_quality_scores_len()
220 seq_len = len(self.sequence)
221 assert qual_len == seq_len, f"Invalid FASTQ file: quality score length ({qual_len:d}) does not match sequence length ({seq_len:d})"
222
223 def reverse(self, clone=True):
224 # need to override how decimal quality scores are reversed
225 if clone:
226 rval = self.clone()
227 else:
228 rval = self
229 rval.sequence = transform.reverse(self.sequence)
230 if rval.is_ascii_encoded():
231 rval.quality = rval.quality[::-1]
232 else:
233 rval.quality = reversed(rval.get_decimal_quality_scores())
234 rval.quality = " ".join(map(str, rval.quality)) + " "
235 return rval
236
237 def apply_galaxy_conventions(self):
238 pass
239
240
241class fastqSangerRead(fastqSequencingRead):
242 format = 'sanger'
243 ascii_min = 33
244 ascii_max = 126
245 quality_min = 0
246 quality_max = 93
247 score_system = 'phred'
248 sequence_space = 'base'
249
250
251class fastqIlluminaRead(fastqSequencingRead):
252 format = 'illumina'
253 ascii_min = 64
254 ascii_max = 126
255 quality_min = 0
256 quality_max = 62
257 score_system = 'phred'
258 sequence_space = 'base'
259
260
261class fastqSolexaRead(fastqSequencingRead):
262 format = 'solexa'
263 ascii_min = 59
264 ascii_max = 126
265 quality_min = -5
266 quality_max = 62
267 score_system = 'solexa'
268 sequence_space = 'base'
269
270
271class fastqCSSangerRead(fastqSequencingRead):
272 format = 'cssanger' # color space
273 ascii_min = 33
274 ascii_max = 126
275 quality_min = 0
276 quality_max = 93
277 score_system = 'phred'
278 sequence_space = 'color'
279 valid_sequence_list = [str(_) for _ in range(7)] + ['.']
280
281 def __len__(self):
282 if self.has_adapter_base(): # Adapter base is not counted in length of read
283 return len(self.sequence) - 1
284 return fastqSequencingRead.__len__(self)
285
286 def has_adapter_base(self):
287 if self.sequence and self.sequence[0] in LETTERS: # adapter base must be a letter
288 return True
289 return False
290
291 def insufficient_quality_length(self):
292 if self.has_adapter_base():
293 return self.get_ascii_quality_scores_len() + 1 < len(self.sequence)
294 return fastqSequencingRead.insufficient_quality_length(self)
295
296 def assert_sequence_quality_lengths(self):
297 if self.has_adapter_base():
298 qual_len = self.get_ascii_quality_scores_len()
299 seq_len = len(self.sequence)
300 assert qual_len + 1 == seq_len or qual_len == seq_len, f"Invalid FASTQ file: quality score length ({qual_len:d}) does not match sequence length ({seq_len:d} with adapter base)" # SRA adds FAKE/DUMMY quality scores to the adapter base, we'll allow the reading of the Improper score here, but remove it in the Reader when "apply_galaxy_conventions" is set to True
301 else:
302 return fastqSequencingRead.assert_sequence_quality_lengths(self)
303
304 def get_sequence(self):
305 if self.has_adapter_base():
306 return self.sequence[1:]
307 return self.sequence
308
309 def reverse(self, clone=True):
310 # need to override how color space is reversed
311 if clone:
312 rval = self.clone()
313 else:
314 rval = self
315 if rval.has_adapter_base():
316 adapter = rval.sequence[0]
317 # sequence = rval.sequence[1:]
318 rval.sequence = self.color_space_converter.to_color_space(transform.reverse(self.color_space_converter.to_base_space(rval.sequence)), adapter_base=adapter)
319 else:
320 rval.sequence = transform.reverse(rval.sequence)
321
322 if rval.is_ascii_encoded():
323 rval.quality = rval.quality[::-1]
324 else:
325 rval.quality = reversed(rval.get_decimal_quality_scores())
326 rval.quality = " ".join(map(str, rval.quality)) + " "
327 return rval
328
329 def complement(self, clone=True):
330 # need to override how color space is complemented
331 if clone:
332 rval = self.clone()
333 else:
334 rval = self
335 if rval.has_adapter_base(): # No adapter, color space stays the same
336 adapter = rval.sequence[0]
337 sequence = rval.sequence[1:]
338 if adapter.lower() != 'u':
339 adapter = transform.DNA_complement(adapter)
340 else:
341 adapter = transform.RNA_complement(adapter)
342 rval.sequence = f"{adapter}{sequence}"
343 return rval
344
345 def change_adapter(self, new_adapter, clone=True):
346 # if new_adapter is empty, remove adapter, otherwise replace with new_adapter
347 if clone:
348 rval = self.clone()
349 else:
350 rval = self
351 if rval.has_adapter_base():
352 if new_adapter:
353 if new_adapter != rval.sequence[0]:
354 rval.sequence = rval.color_space_converter.to_color_space(rval.color_space_converter.to_base_space(rval.sequence), adapter_base=new_adapter)
355 else:
356 rval.sequence = rval.sequence[1:]
357 elif new_adapter:
358 rval.sequence = f"{new_adapter}{rval.sequence}"
359 return rval
360
361 def apply_galaxy_conventions(self):
362 if self.has_adapter_base() and len(self.sequence) == len(self.get_ascii_quality_scores()): # SRA adds FAKE/DUMMY quality scores to the adapter base, we remove them here
363 if self.is_ascii_encoded():
364 self.quality = self.quality[1:]
365 else:
366 self.quality = " ".join(map(str, self.get_decimal_quality_scores()[1:]))
367
368
369FASTQ_FORMATS = {}
370for format in [fastqIlluminaRead, fastqSolexaRead, fastqSangerRead, fastqCSSangerRead]:
371 FASTQ_FORMATS[format.format] = format
372 FASTQ_FORMATS[format.format + ".gz"] = format
373 FASTQ_FORMATS[format.format + ".bz2"] = format
374
375
376class fastqAggregator:
377 VALID_FORMATS = list(FASTQ_FORMATS.keys())
378
379 def __init__(self):
380 self.ascii_values_used = [] # quick lookup of all ascii chars used
381 self.seq_lens = {} # counts of seqs by read len
382 self.nuc_index_quality = [] # counts of scores by read column
383 self.nuc_index_base = [] # counts of bases by read column
384
385 def consume_read(self, fastq_read):
386 # ascii values used
387 for val in fastq_read.get_ascii_quality_scores():
388 if val not in self.ascii_values_used:
389 self.ascii_values_used.append(val)
390 # lengths
391 seq_len = len(fastq_read)
392 self.seq_lens[seq_len] = self.seq_lens.get(seq_len, 0) + 1
393 # decimal qualities by column
394 for i, val in enumerate(fastq_read.get_decimal_quality_scores()):
395 if i == len(self.nuc_index_quality):
396 self.nuc_index_quality.append({})
397 self.nuc_index_quality[i][val] = self.nuc_index_quality[i].get(val, 0) + 1
398 # bases by column
399 for i, nuc in enumerate(fastq_read.get_sequence()):
400 if i == len(self.nuc_index_base):
401 self.nuc_index_base.append({})
402 nuc = nuc.upper()
403 self.nuc_index_base[i][nuc] = self.nuc_index_base[i].get(nuc, 0) + 1
404
405 def get_valid_formats(self, check_list=None):
406 if not check_list:
407 check_list = self.VALID_FORMATS
408 rval = []
409 sequence = []
410 for nuc_dict in self.nuc_index_base:
411 for nuc in nuc_dict.keys():
412 if nuc not in sequence:
413 sequence.append(nuc)
414 sequence = "".join(sequence)
415 quality = "".join(self.ascii_values_used)
416 for fastq_format in check_list:
417 fastq_read = fastqSequencingRead.get_class_by_format(fastq_format)()
418 fastq_read.quality = quality
419 fastq_read.sequence = sequence
420 if fastq_read.is_valid_format():
421 rval.append(fastq_format)
422 return rval
423
424 def get_ascii_range(self):
425 if not self.ascii_values_used:
426 return None
427 return (min(self.ascii_values_used), max(self.ascii_values_used))
428
429 def get_decimal_range(self):
430 if not self.nuc_index_quality:
431 return None
432 decimal_values_used = []
433 for scores in self.nuc_index_quality:
434 decimal_values_used.extend(scores.keys())
435 return (min(decimal_values_used), max(decimal_values_used))
436
437 def get_length_counts(self):
438 return self.seq_lens
439
440 def get_max_read_length(self):
441 return len(self.nuc_index_quality)
442
443 def get_read_count_for_column(self, column):
444 if column >= len(self.nuc_index_quality):
445 return 0
446 return sum(self.nuc_index_quality[column].values())
447
448 def get_read_count(self):
449 return self.get_read_count_for_column(0)
450
451 def get_base_counts_for_column(self, column):
452 return self.nuc_index_base[column]
453
454 def get_score_list_for_column(self, column):
455 return list(self.nuc_index_quality[column].keys())
456
457 def get_score_min_for_column(self, column):
458 return min(self.nuc_index_quality[column].keys())
459
460 def get_score_max_for_column(self, column):
461 return max(self.nuc_index_quality[column].keys())
462
463 def get_score_sum_for_column(self, column):
464 return sum(score * count for score, count in self.nuc_index_quality[column].items())
465
466 def get_score_at_position_for_column(self, column, position):
467 score_value_dict = self.nuc_index_quality[column]
468 scores = sorted(score_value_dict.keys())
469 for score in scores:
470 if score_value_dict[score] <= position:
471 position -= score_value_dict[score]
472 else:
473 return score
474
475 def get_summary_statistics_for_column(self, i):
476 def _get_med_pos(size):
477 halfed = int(size / 2)
478 if size % 2 == 1:
479 return [halfed]
480 return [halfed - 1, halfed]
481 read_count = self.get_read_count_for_column(i)
482
483 min_score = self.get_score_min_for_column(i)
484 max_score = self.get_score_max_for_column(i)
485 sum_score = self.get_score_sum_for_column(i)
486 mean_score = float(sum_score) / float(read_count)
487 # get positions
488 med_pos = _get_med_pos(read_count)
489 if 0 in med_pos:
490 q1_pos = [0]
491 q3_pos = [read_count - 1]
492 else:
493 q1_pos = _get_med_pos(min(med_pos))
494 q3_pos = []
495 for pos in q1_pos:
496 q3_pos.append(max(med_pos) + 1 + pos)
497 # get scores at position
498 med_score = float(sum(self.get_score_at_position_for_column(i, pos) for pos in med_pos)) / float(len(med_pos))
499 q1 = float(sum(self.get_score_at_position_for_column(i, pos) for pos in q1_pos)) / float(len(q1_pos))
500 q3 = float(sum(self.get_score_at_position_for_column(i, pos) for pos in q3_pos)) / float(len(q3_pos))
501 # determine iqr and step
502 iqr = q3 - q1
503 step = 1.5 * iqr
504
505 # Determine whiskers and outliers
506 outliers = []
507 score_list = sorted(self.get_score_list_for_column(i))
508 left_whisker = q1 - step
509 for score in score_list:
510 if left_whisker <= score:
511 left_whisker = score
512 break
513 else:
514 outliers.append(score)
515
516 right_whisker = q3 + step
517 score_list.reverse()
518 for score in score_list:
519 if right_whisker >= score:
520 right_whisker = score
521 break
522 else:
523 outliers.append(score)
524
525 column_stats = {'read_count': read_count,
526 'min_score': min_score,
527 'max_score': max_score,
528 'sum_score': sum_score,
529 'mean_score': mean_score,
530 'q1': q1,
531 'med_score': med_score,
532 'q3': q3,
533 'iqr': iqr,
534 'left_whisker': left_whisker,
535 'right_whisker': right_whisker,
536 'outliers': outliers}
537 return column_stats
538
539
540def _fastq_open_stream(fh=None, format="sanger", path=None, mode="r"):
541 if fh is None:
542 assert path is not None
543 if format and format.endswith(".gz"):
544 fh = gzip.open(path, mode=f"{mode}t")
545 elif format and format.endswith(".bz2"):
546 fh = bz2.open(path, mode=f"{mode}t")
547 else:
548 fh = open(path, f"{mode}t")
549 else:
550 if format and format.endswith(".gz"):
551 fh = gzip.GzipFile(fileobj=fh, mode=mode)
552 elif format and format.endswith(".bz2"):
553 raise Exception("bz2 formats do not support file handle inputs")
554 return fh
555
556
557class fileHandler:
558
559 def __init__(self, fh, format, path):
560 self.fh = fh
561 self.format = format
562 self.path = path
563 self._file = None
564
565 @property
566 def file(self):
567 if self._file is None:
568 self._file = _fastq_open_stream(fh=self.fh, format=self.format, path=self.path, mode=self.mode)
569 return self._file
570
571 @file.setter
572 def file(self, fh):
573 self._file = fh
574
575 def __enter__(self):
576 fh = _fastq_open_stream(fh=self.fh, format=self.format, path=self.path, mode=self.mode)
577 self._set_file_handle(fh)
578 return self
579
580 def __exit__(self, exc_type, exc_value, traceback):
581 self.close()
582
583 def _set_file_handle(self, fh):
584 # Extension point for subclasses to wrap file handler
585 self.file = fh
586
587 def close(self):
588 return self.file.close()
589
590 def _close_on_error(self):
591 # Extension point for subclasses (fastqVerboseErrorReader needs file access after error)
592 return self.close()
593
594
595class fastqReader(fileHandler):
596
597 mode = 'r'
598
599 def __init__(
600 self, fh=None, format='sanger', apply_galaxy_conventions=False, path=None, fix_id=False):
601 super().__init__(fh=fh, format=format, path=path)
602 self.apply_galaxy_conventions = apply_galaxy_conventions
603 self.fix_id = fix_id # fix inconsistent identifiers (source: SRA data dumps)
604
605 def __next__(self):
606 rval = fastqSequencingRead.get_class_by_format(self.format)()
607
608 id_line = self._read_fastq_header() # We need the raw line1 to compare w/line3
609 rval.identifier = id_line
610
611 self._read_fastq_sequence(rval, id_line)
612 self._read_fastq_qualityscores(rval)
613
614 rval.assert_sequence_quality_lengths()
615 if self.apply_galaxy_conventions:
616 rval.apply_galaxy_conventions()
617
618 return rval
619
620 def _read_fastq_header(self):
621 while True:
622 line = self.file.readline()
623 if not line:
624 raise StopIteration
625 line = line.rstrip('\n\r')
626 if line:
627 break
628
629 if not line.startswith('@'):
630 self.close()
631 raise fastqFormatError(f'Invalid FASTQ header: {line}')
632
633 return line
634
635 def _read_fastq_sequence(self, rval, id_line):
636 while True:
637 line = self.file.readline()
638 if not line:
639 self.close()
640 raise fastqFormatError(
641 'Invalid FASTQ file: could not find quality score of '
642 f'sequence identifier {rval.identifier}.')
643 line = line.rstrip('\n\r')
644
645 if not line.startswith('+'):
646 rval.append_sequence(line)
647 else: # Sequence is over, read the quality score identifier
648 if len(line) == 1 or line[1:] == id_line[1:]: # Line is valid
649 rval.description = line
650 break
651 elif self.fix_id: # Line is not valid, but we must fix it
652 rval.description = '+'
653 break
654 else: # Line is not valid, sound the alarm!
655 self._close_on_error()
656 raise fastqFormatError(
657 f'Invalid FASTQ file: quality score identifier ({line})'
658 f'does not match sequence identifier ({rval.identifier}).'
659 )
660
661 def _read_fastq_qualityscores(self, rval):
662 while rval.insufficient_quality_length():
663 line = self.file.readline()
664 if not line:
665 break
666
667 line = line.rstrip('\n\r')
668 rval.append_quality(line)
669
670 def __iter__(self):
671 while True:
672 try:
673 yield next(self)
674 except StopIteration:
675 self.close()
676 # Catch exception and return normally
677 return
678
679
680class ReadlineCountFile:
681 def __init__(self, f):
682 self.__file = f
683 self.readline_count = 0
684
685 def readline(self, *args, **kwds):
686 self.readline_count += 1
687 return self.__file.readline(*args, **kwds)
688
689 def __getattr__(self, name):
690 return getattr(self.__file, name)
691
692
693class fastqVerboseErrorReader(fastqReader):
694 MAX_PRINT_ERROR_BYTES = 1024
695
696 def __init__(self, fh=None, **kwds):
697 super().__init__(fh=fh, **kwds)
698 self.last_good_identifier = None
699
700 def _close_on_error(self):
701 pass # Do not close it yet: need file access to provide error details (see __next__())
702
703 def _set_file_handle(self, fh):
704 # Extension point for subclasses to wrap file handler
705 self.file = ReadlineCountFile(fh)
706
707 def __next__(self):
708 if not hasattr(self.file, "readline_count"):
709 return super().__next__()
710
711 last_good_end_offset = self.file.tell()
712 last_readline_count = self.file.readline_count
713 try:
714 block = super().__next__()
715 self.last_good_identifier = block.identifier
716 return block
717 except StopIteration as e:
718 raise e
719 except Exception as e:
720 print(f"There was an error reading your input file. Your input file is likely malformed.\nIt is suggested that you double-check your original input file for errors -- helpful information for this purpose has been provided below.\nHowever, if you think that you have encountered an actual error with this tool, please do tell us by using the bug reporting mechanism.\n\nThe reported error is: '{e}'.")
721 if self.last_good_identifier is not None:
722 print(f"The last valid FASTQ read had an identifier of '{self.last_good_identifier}'.")
723 else:
724 print("The error occurred at the start of your file and no valid FASTQ reads were found.")
725 error_offset = self.file.tell()
726 error_byte_count = error_offset - last_good_end_offset
727 print_error_bytes = min(self.MAX_PRINT_ERROR_BYTES, error_byte_count)
728 print(f"The error in your file occurs between lines '{last_readline_count + 1:d}' and '{self.file.readline_count:d}', which corresponds to byte-offsets '{last_good_end_offset:d}' and '{error_offset:d}', and contains the text ({print_error_bytes:d} of {error_byte_count:d} bytes shown):\n")
729 self.file.seek(last_good_end_offset)
730 print(self.file.read(print_error_bytes))
731 self.file.close()
732 raise e
733
734
735class fastqNamedReader(fastqReader):
736 def __init__(self, fh=None, format='sanger', apply_galaxy_conventions=False, path=None):
737 super().__init__(fh=fh, format=format, apply_galaxy_conventions=apply_galaxy_conventions, path=path)
738 self.offset_dict = {}
739 self.eof = False
740
741 def get(self, sequence_identifier):
742 # Input is either a sequence ID or a sequence object
743 if not isinstance(sequence_identifier, str):
744 # Input was a sequence object (not a sequence ID). Get the sequence ID
745 sequence_identifier = sequence_identifier.identifier
746 # Get only the ID part of the sequence header
747 sequence_id, sequence_sep, sequence_desc = sequence_identifier.partition(' ')
748 rval = None
749 if sequence_id in self.offset_dict:
750 initial_offset = self.file.tell()
751 seq_offset = self.offset_dict[sequence_id].pop(0)
752 if not self.offset_dict[sequence_id]:
753 del self.offset_dict[sequence_id]
754 self.file.seek(seq_offset)
755 rval = next(self)
756 # assert rval.id == sequence_id, 'seq id mismatch' #should be able to remove this
757 self.file.seek(initial_offset)
758 else:
759 while True:
760 offset = self.file.tell()
761 try:
762 fastq_read = next(self)
763 except StopIteration:
764 self.eof = True
765 break # eof, id not found, will return None
766 fastq_read_id, fastq_read_sep, fastq_read_desc = fastq_read.identifier.partition(' ')
767 if fastq_read_id == sequence_id:
768 rval = fastq_read
769 break
770 else:
771 if fastq_read_id not in self.offset_dict:
772 self.offset_dict[fastq_read_id] = []
773 self.offset_dict[fastq_read_id].append(offset)
774 if rval is not None and self.apply_galaxy_conventions:
775 rval.apply_galaxy_conventions()
776 return rval
777
778 def has_data(self):
779 # returns a string representation of remaining data, or empty string (False) if no data remaining
780 eof = self.eof
781 count = 0
782 rval = ''
783 if self.offset_dict:
784 count = sum(map(len, self.offset_dict.values()))
785 if not eof:
786 offset = self.file.tell()
787 try:
788 next(self)
789 except StopIteration:
790 eof = True
791 self.file.seek(offset)
792 if count:
793 rval = f"There were {count:d} known sequence reads not utilized. "
794 if not eof:
795 rval += "An additional unknown number of reads exist in the input that were not utilized."
796 return rval
797
798
799class fastqWriter(fileHandler):
800
801 mode = 'w'
802
803 def __init__(self, fh=None, format=None, force_quality_encoding=None, path=None):
804 super().__init__(fh=fh, format=format, path=path)
805 self.force_quality_encoding = force_quality_encoding
806
807 def write(self, fastq_read):
808 if self.format:
809 fastq_read = fastq_read.convert_read_to_format(self.format, force_quality_encoding=self.force_quality_encoding)
810 self.file.write(str(fastq_read))
811
812
813class fastqJoiner:
814
815 def __init__(self, format, force_quality_encoding=None, paste=""):
816 self.format = format
817 self.force_quality_encoding = force_quality_encoding
818 outformat = fastqSequencingRead.get_class_by_format(format)
819 self.paste_sequence = paste
820 self.paste_ascii_quality = chr(outformat.ascii_max) * len(paste)
821 self.paste_decimal_quality = " ".join(str(outformat.quality_max) for x in range(len(paste)))
822
823 def join(self, read1, read2):
824 read1_id, read1_sep, read1_desc = read1.identifier.partition(' ')
825 read2_id, read2_sep, read2_desc = read2.identifier.partition(' ')
826 if read1_id.endswith('/2') and read2_id.endswith('/1'):
827 # swap 1 and 2
828 tmp = read1
829 read1 = read2
830 read2 = tmp
831 del tmp
832 if read1_id.endswith('/1') and read2_id.endswith('/2'):
833 read1_id = read1_id[:-2]
834
835 identifier = read1_id
836 if read1_desc:
837 identifier = identifier + ' ' + read1_desc
838
839 # use force quality encoding, if not present force to encoding of first read
840 force_quality_encoding = self.force_quality_encoding
841 if not force_quality_encoding:
842 if read1.is_ascii_encoded():
843 force_quality_encoding = 'ascii'
844 else:
845 force_quality_encoding = 'decimal'
846
847 new_read1 = read1.convert_read_to_format(self.format, force_quality_encoding=force_quality_encoding)
848 new_read2 = read2.convert_read_to_format(self.format, force_quality_encoding=force_quality_encoding)
849 rval = FASTQ_FORMATS[self.format]()
850 rval.identifier = identifier
851 if len(read1.description) > 1:
852 rval.description = "+" + identifier[1:]
853 else:
854 rval.description = '+'
855 if rval.sequence_space == 'color':
856 # need to handle color space joining differently
857 # convert to nuc space, join, then convert back
858 rval.sequence = rval.convert_base_to_color_space(new_read1.convert_color_to_base_space(new_read1.sequence) + self.paste_sequence + new_read2.convert_color_to_base_space(new_read2.sequence))
859 else:
860 rval.sequence = new_read1.sequence + self.paste_sequence + new_read2.sequence
861 if force_quality_encoding == 'ascii':
862 rval.quality = new_read1.quality + self.paste_ascii_quality + new_read2.quality
863 else:
864 rval.quality = f"{new_read1.quality.strip()} {self.paste_decimal_quality}"
865 rval.quality = (f"{rval.quality.strip()} {new_read2.quality.strip()}").strip()
866 return rval
867
868 def get_paired_identifier(self, fastq_read):
869 read_id, read_sep, read_desc = fastq_read.identifier.partition(' ')
870 if read_id[-2] == '/':
871 if read_id[-1] == "1":
872 read_id = f"{read_id[:-1]}2"
873 elif read_id[-1] == "2":
874 read_id = f"{read_id[:-1]}1"
875 return read_id
876
877 def is_first_mate(self, sequence_id):
878 is_first = None
879 if not isinstance(sequence_id, str):
880 sequence_id = sequence_id.identifier
881 sequence_id, sequence_sep, sequence_desc = sequence_id.partition(' ')
882 if sequence_id[-2] == '/':
883 if sequence_id[-1] == "1":
884 is_first = True
885 else:
886 is_first = False
887 return is_first
888
889
890class fastqSplitter:
891 def split(self, fastq_read):
892 length = len(fastq_read)
893 # Only reads of even lengths can be split
894 if length % 2 != 0:
895 return None, None
896 half = int(length / 2)
897 read1 = fastq_read.slice(0, half)
898 read1.identifier += "/1"
899 if len(read1.description) > 1:
900 read1.description += "/1"
901 read2 = fastq_read.slice(half, None)
902 read2.identifier += "/2"
903 if len(read2.description) > 1:
904 read2.description += "/2"
905 return read1, read2
906
907
908class fastqCombiner:
909 def __init__(self, format):
910 self.format = format
911
912 def combine(self, fasta_seq, quality_seq):
913 fastq_read = fastqSequencingRead.get_class_by_format(self.format)()
914 fastq_read.identifier = f"@{fasta_seq.identifier[1:]}"
915 fastq_read.description = '+'
916 fastq_read.sequence = fasta_seq.sequence
917 fastq_read.quality = quality_seq.sequence
918 return fastq_read
919
920
921class fastqFakeFastaScoreReader:
922 def __init__(self, format='sanger', quality_encoding=None):
923 self.fastq_read = fastqSequencingRead.get_class_by_format(format)()
924 if quality_encoding != 'decimal':
925 quality_encoding = 'ascii'
926 self.quality_encoding = quality_encoding
927
928 def close(self):
929 return # nothing to close
930
931 def get(self, sequence):
932 assert isinstance(sequence, fastaSequence), 'fastqFakeFastaScoreReader requires a fastaSequence object as the parameter'
933 # add sequence to fastq_read, then get_sequence(), color space adapters do not have quality score values
934 self.fastq_read.sequence = sequence.sequence
935 new_sequence = fastaSequence()
936 new_sequence.identifier = sequence.identifier
937 if self.quality_encoding == 'ascii':
938 new_sequence.sequence = chr(self.fastq_read.ascii_max) * len(self.fastq_read.get_sequence())
939 else:
940 new_sequence.sequence = (f"{self.fastq_read.quality_max:d} " * len(self.fastq_read.get_sequence()))
941 return new_sequence
942
943 def has_data(self):
944 return '' # No actual data exist, none can be remaining
945
946
947class fastqFormatError(Exception):
948 pass