Source code

Download source archive ↓

upstream-dependencies/galaxy-sequence-utils/galaxy_utils/sequence/fastq.py

1# Dan Blankenberg
2
3import bz2
4import gzip
5import math
6import string
7
8from . import transform
9from .fasta import fastaSequence
10from .sequence import SequencingRead
11
12LETTERS = string.ascii_letters
13
14
15class fastqSequencingRead(SequencingRead):
16    format = 'sanger'  # sanger is default
17    ascii_min = 33
18    ascii_max = 126
19    quality_min = 0
20    quality_max = 93
21    score_system = 'phred'  # phred or solexa
22    sequence_space = 'base'  # base or color
23
24    @classmethod
25    def get_class_by_format(cls, format):
26        assert format in FASTQ_FORMATS, f'Unknown format type specified: {format}'
27        return FASTQ_FORMATS[format]
28
29    @classmethod
30    def convert_score_phred_to_solexa(cls, decimal_score_list):
31        def phred_to_solexa(score):
32            if score <= 0:  # can't take log10( 1 - 1 ); make <= 0 into -5
33                return -5
34            return int(round(10.0 * math.log10(math.pow(10.0, (float(score) / 10.0)) - 1.0)))
35        return [phred_to_solexa(_) for _ in decimal_score_list]
36
37    @classmethod
38    def convert_score_solexa_to_phred(cls, decimal_score_list):
39        def solexa_to_phred(score):
40            return int(round(10.0 * math.log10(math.pow(10.0, (float(score) / 10.0)) + 1.0)))
41        return [solexa_to_phred(_) for _ in decimal_score_list]
42
43    @classmethod
44    def restrict_scores_to_valid_range(cls, decimal_score_list):
45        def restrict_score(score):
46            return max(min(score, cls.quality_max), cls.quality_min)
47        return [restrict_score(_) for _ in decimal_score_list]
48
49    @classmethod
50    def transform_scores_to_valid_range(cls, decimal_score_list):
51        cls_quality_max = cls.quality_max
52        cls_quality_min = cls.quality_min
53        for i in range(len(decimal_score_list)):
54            score = decimal_score_list[i]
55            if score > cls_quality_max:
56                transformed_score = cls_quality_max
57            elif score < cls_quality_min:
58                transformed_score = cls_quality_min
59            else:
60                transformed_score = score
61            decimal_score_list[i] = str(transformed_score)
62
63    @classmethod
64    def transform_scores_to_valid_range_ascii(cls, decimal_score_list):
65        cls_quality_max = cls.quality_max
66        cls_quality_min = cls.quality_min
67        to_quality = cls.ascii_min - cls.quality_min
68        for i in range(len(decimal_score_list)):
69            score = decimal_score_list[i]
70            if score > cls_quality_max:
71                transformed_score = cls_quality_max
72            elif score < cls_quality_min:
73                transformed_score = cls_quality_min
74            else:
75                transformed_score = score
76            transformed_score = chr(transformed_score + to_quality)
77            decimal_score_list[i] = transformed_score
78
79    @classmethod
80    def convert_base_to_color_space(cls, sequence):
81        return cls.color_space_converter.to_color_space(sequence)
82
83    @classmethod
84    def convert_color_to_base_space(cls, sequence):
85        return cls.color_space_converter.to_base_space(sequence)
86
87    def is_ascii_encoded(self):
88        # as per fastq definition only decimal quality strings can have spaces (and TABs for our purposes) in them (and must have a trailing space)
89        if ' ' in self.quality:
90            return False
91        if '\t' in self.quality:
92            return False
93        return True
94
95    def get_ascii_quality_scores(self):
96        if self.is_ascii_encoded():
97            return list(self.quality)
98        else:
99            quality = self.quality.rstrip()  # decimal scores should have a trailing space
100            if quality:
101                try:
102                    to_quality = self.ascii_min - self.quality_min
103                    return [chr(int(val) + to_quality) for val in quality.split()]
104                except ValueError as e:
105                    raise ValueError(f'Error Parsing quality String. ASCII quality strings cannot contain spaces ({self.quality}): {e}')
106            else:
107                return []
108
109    def get_ascii_quality_scores_len(self):
110        """
111        Compute ascii quality score length, without generating relatively
112        expensive qualty score array.
113        """
114        if self.is_ascii_encoded():
115            return len(self.quality)
116        else:
117            quality = self.quality.rstrip()
118            if quality:
119                try:
120                    return len(quality.split())
121                except ValueError as e:
122                    raise ValueError(f'Error Parsing quality String. ASCII quality strings cannot contain spaces ({self.quality}): {e}')
123            else:
124                return 0
125
126    def get_decimal_quality_scores(self):
127        return self.__get_decimal_quality_scores(self.is_ascii_encoded())
128
129    def __get_decimal_quality_scores(self, ascii):
130        if ascii:
131            to_quality = self.quality_min - self.ascii_min
132            return [ord(val) + to_quality for val in self.quality]
133        else:
134            quality = self.quality.rstrip()  # decimal scores should have a trailing space
135            if quality:
136                return [int(val) for val in quality.split() if val.strip()]
137            else:
138                return []
139
140    def convert_read_to_format(self, format, force_quality_encoding=None):
141        assert format in FASTQ_FORMATS, f'Unknown format type specified: {format}'
142        assert force_quality_encoding in [None, 'ascii', 'decimal'], f'Invalid force_quality_encoding: {force_quality_encoding}'
143        new_class = FASTQ_FORMATS[format]
144        new_read = new_class()
145        new_read.identifier = self.identifier
146        if self.sequence_space == new_class.sequence_space:
147            new_read.sequence = self.sequence
148        else:
149            if self.sequence_space == 'base':
150                new_read.sequence = self.convert_base_to_color_space(self.sequence)
151            else:
152                new_read.sequence = self.convert_color_to_base_space(self.sequence)
153        new_read.description = self.description
154        is_ascii = self.is_ascii_encoded()
155        if self.score_system != new_read.score_system:
156            if self.score_system == 'phred':
157                score_list = self.convert_score_phred_to_solexa(self.__get_decimal_quality_scores(is_ascii))
158            else:
159                score_list = self.convert_score_solexa_to_phred(self.__get_decimal_quality_scores(is_ascii))
160        else:
161            score_list = self.__get_decimal_quality_scores(is_ascii)
162        if force_quality_encoding is None:
163            if is_ascii:
164                new_encoding = 'ascii'
165            else:
166                new_encoding = 'decimal'
167        else:
168            new_encoding = force_quality_encoding
169        if new_encoding == 'ascii':
170            new_class.transform_scores_to_valid_range_ascii(score_list)
171            new_read.quality = "".join(score_list)
172        else:  # decimal
173            new_class.transform_scores_to_valid_range(score_list)
174            new_read.quality = " ".join(score_list) + " "  # need trailing space to be valid decimal fastq
175        return new_read
176
177    def get_sequence(self):
178        return self.sequence
179
180    def slice(self, left_column_offset, right_column_offset):
181        new_read = fastqSequencingRead.get_class_by_format(self.format)()
182        new_read.identifier = self.identifier
183        new_read.sequence = self.get_sequence()[left_column_offset:right_column_offset]
184        new_read.description = self.description
185        if self.is_ascii_encoded():
186            new_read.quality = self.quality[left_column_offset:right_column_offset]
187        else:
188            quality = [str(_) for _ in self.get_decimal_quality_scores()[left_column_offset:right_column_offset]]
189            if quality:
190                new_read.quality = " ".join(quality) + " "
191            else:
192                new_read.quality = ''
193        return new_read
194
195    def is_valid_format(self):
196        if self.is_ascii_encoded():
197            for val in self.get_ascii_quality_scores():
198                val = ord(val)
199                if val < self.ascii_min or val > self.ascii_max:
200                    return False
201        else:
202            for val in self.get_decimal_quality_scores():
203                if val < self.quality_min or val > self.quality_max:
204                    return False
205        if not self.is_valid_sequence():
206            return False
207        return True
208
209    def is_valid_sequence(self):
210        for base in self.get_sequence():
211            if base not in self.valid_sequence_list:
212                return False
213        return True
214
215    def insufficient_quality_length(self):
216        return self.get_ascii_quality_scores_len() < len(self.sequence)
217
218    def assert_sequence_quality_lengths(self):
219        qual_len = self.get_ascii_quality_scores_len()
220        seq_len = len(self.sequence)
221        assert qual_len == seq_len, f"Invalid FASTQ file: quality score length ({qual_len:d}) does not match sequence length ({seq_len:d})"
222
223    def reverse(self, clone=True):
224        # need to override how decimal quality scores are reversed
225        if clone:
226            rval = self.clone()
227        else:
228            rval = self
229        rval.sequence = transform.reverse(self.sequence)
230        if rval.is_ascii_encoded():
231            rval.quality = rval.quality[::-1]
232        else:
233            rval.quality = reversed(rval.get_decimal_quality_scores())
234            rval.quality = " ".join(map(str, rval.quality)) + " "
235        return rval
236
237    def apply_galaxy_conventions(self):
238        pass
239
240
241class fastqSangerRead(fastqSequencingRead):
242    format = 'sanger'
243    ascii_min = 33
244    ascii_max = 126
245    quality_min = 0
246    quality_max = 93
247    score_system = 'phred'
248    sequence_space = 'base'
249
250
251class fastqIlluminaRead(fastqSequencingRead):
252    format = 'illumina'
253    ascii_min = 64
254    ascii_max = 126
255    quality_min = 0
256    quality_max = 62
257    score_system = 'phred'
258    sequence_space = 'base'
259
260
261class fastqSolexaRead(fastqSequencingRead):
262    format = 'solexa'
263    ascii_min = 59
264    ascii_max = 126
265    quality_min = -5
266    quality_max = 62
267    score_system = 'solexa'
268    sequence_space = 'base'
269
270
271class fastqCSSangerRead(fastqSequencingRead):
272    format = 'cssanger'  # color space
273    ascii_min = 33
274    ascii_max = 126
275    quality_min = 0
276    quality_max = 93
277    score_system = 'phred'
278    sequence_space = 'color'
279    valid_sequence_list = [str(_) for _ in range(7)] + ['.']
280
281    def __len__(self):
282        if self.has_adapter_base():  # Adapter base is not counted in length of read
283            return len(self.sequence) - 1
284        return fastqSequencingRead.__len__(self)
285
286    def has_adapter_base(self):
287        if self.sequence and self.sequence[0] in LETTERS:  # adapter base must be a letter
288            return True
289        return False
290
291    def insufficient_quality_length(self):
292        if self.has_adapter_base():
293            return self.get_ascii_quality_scores_len() + 1 < len(self.sequence)
294        return fastqSequencingRead.insufficient_quality_length(self)
295
296    def assert_sequence_quality_lengths(self):
297        if self.has_adapter_base():
298            qual_len = self.get_ascii_quality_scores_len()
299            seq_len = len(self.sequence)
300            assert qual_len + 1 == seq_len or qual_len == seq_len, f"Invalid FASTQ file: quality score length ({qual_len:d}) does not match sequence length ({seq_len:d} with adapter base)"  # SRA adds FAKE/DUMMY quality scores to the adapter base, we'll allow the reading of the Improper score here, but remove it in the Reader when "apply_galaxy_conventions" is set to True
301        else:
302            return fastqSequencingRead.assert_sequence_quality_lengths(self)
303
304    def get_sequence(self):
305        if self.has_adapter_base():
306            return self.sequence[1:]
307        return self.sequence
308
309    def reverse(self, clone=True):
310        # need to override how color space is reversed
311        if clone:
312            rval = self.clone()
313        else:
314            rval = self
315        if rval.has_adapter_base():
316            adapter = rval.sequence[0]
317            # sequence = rval.sequence[1:]
318            rval.sequence = self.color_space_converter.to_color_space(transform.reverse(self.color_space_converter.to_base_space(rval.sequence)), adapter_base=adapter)
319        else:
320            rval.sequence = transform.reverse(rval.sequence)
321
322        if rval.is_ascii_encoded():
323            rval.quality = rval.quality[::-1]
324        else:
325            rval.quality = reversed(rval.get_decimal_quality_scores())
326            rval.quality = " ".join(map(str, rval.quality)) + " "
327        return rval
328
329    def complement(self, clone=True):
330        # need to override how color space is complemented
331        if clone:
332            rval = self.clone()
333        else:
334            rval = self
335        if rval.has_adapter_base():  # No adapter, color space stays the same
336            adapter = rval.sequence[0]
337            sequence = rval.sequence[1:]
338            if adapter.lower() != 'u':
339                adapter = transform.DNA_complement(adapter)
340            else:
341                adapter = transform.RNA_complement(adapter)
342            rval.sequence = f"{adapter}{sequence}"
343        return rval
344
345    def change_adapter(self, new_adapter, clone=True):
346        # if new_adapter is empty, remove adapter, otherwise replace with new_adapter
347        if clone:
348            rval = self.clone()
349        else:
350            rval = self
351        if rval.has_adapter_base():
352            if new_adapter:
353                if new_adapter != rval.sequence[0]:
354                    rval.sequence = rval.color_space_converter.to_color_space(rval.color_space_converter.to_base_space(rval.sequence), adapter_base=new_adapter)
355            else:
356                rval.sequence = rval.sequence[1:]
357        elif new_adapter:
358            rval.sequence = f"{new_adapter}{rval.sequence}"
359        return rval
360
361    def apply_galaxy_conventions(self):
362        if self.has_adapter_base() and len(self.sequence) == len(self.get_ascii_quality_scores()):  # SRA adds FAKE/DUMMY quality scores to the adapter base, we remove them here
363            if self.is_ascii_encoded():
364                self.quality = self.quality[1:]
365            else:
366                self.quality = " ".join(map(str, self.get_decimal_quality_scores()[1:]))
367
368
369FASTQ_FORMATS = {}
370for format in [fastqIlluminaRead, fastqSolexaRead, fastqSangerRead, fastqCSSangerRead]:
371    FASTQ_FORMATS[format.format] = format
372    FASTQ_FORMATS[format.format + ".gz"] = format
373    FASTQ_FORMATS[format.format + ".bz2"] = format
374
375
376class fastqAggregator:
377    VALID_FORMATS = list(FASTQ_FORMATS.keys())
378
379    def __init__(self):
380        self.ascii_values_used = []  # quick lookup of all ascii chars used
381        self.seq_lens = {}  # counts of seqs by read len
382        self.nuc_index_quality = []  # counts of scores by read column
383        self.nuc_index_base = []  # counts of bases by read column
384
385    def consume_read(self, fastq_read):
386        # ascii values used
387        for val in fastq_read.get_ascii_quality_scores():
388            if val not in self.ascii_values_used:
389                self.ascii_values_used.append(val)
390        # lengths
391        seq_len = len(fastq_read)
392        self.seq_lens[seq_len] = self.seq_lens.get(seq_len, 0) + 1
393        # decimal qualities by column
394        for i, val in enumerate(fastq_read.get_decimal_quality_scores()):
395            if i == len(self.nuc_index_quality):
396                self.nuc_index_quality.append({})
397            self.nuc_index_quality[i][val] = self.nuc_index_quality[i].get(val, 0) + 1
398        # bases by column
399        for i, nuc in enumerate(fastq_read.get_sequence()):
400            if i == len(self.nuc_index_base):
401                self.nuc_index_base.append({})
402            nuc = nuc.upper()
403            self.nuc_index_base[i][nuc] = self.nuc_index_base[i].get(nuc, 0) + 1
404
405    def get_valid_formats(self, check_list=None):
406        if not check_list:
407            check_list = self.VALID_FORMATS
408        rval = []
409        sequence = []
410        for nuc_dict in self.nuc_index_base:
411            for nuc in nuc_dict.keys():
412                if nuc not in sequence:
413                    sequence.append(nuc)
414        sequence = "".join(sequence)
415        quality = "".join(self.ascii_values_used)
416        for fastq_format in check_list:
417            fastq_read = fastqSequencingRead.get_class_by_format(fastq_format)()
418            fastq_read.quality = quality
419            fastq_read.sequence = sequence
420            if fastq_read.is_valid_format():
421                rval.append(fastq_format)
422        return rval
423
424    def get_ascii_range(self):
425        if not self.ascii_values_used:
426            return None
427        return (min(self.ascii_values_used), max(self.ascii_values_used))
428
429    def get_decimal_range(self):
430        if not self.nuc_index_quality:
431            return None
432        decimal_values_used = []
433        for scores in self.nuc_index_quality:
434            decimal_values_used.extend(scores.keys())
435        return (min(decimal_values_used), max(decimal_values_used))
436
437    def get_length_counts(self):
438        return self.seq_lens
439
440    def get_max_read_length(self):
441        return len(self.nuc_index_quality)
442
443    def get_read_count_for_column(self, column):
444        if column >= len(self.nuc_index_quality):
445            return 0
446        return sum(self.nuc_index_quality[column].values())
447
448    def get_read_count(self):
449        return self.get_read_count_for_column(0)
450
451    def get_base_counts_for_column(self, column):
452        return self.nuc_index_base[column]
453
454    def get_score_list_for_column(self, column):
455        return list(self.nuc_index_quality[column].keys())
456
457    def get_score_min_for_column(self, column):
458        return min(self.nuc_index_quality[column].keys())
459
460    def get_score_max_for_column(self, column):
461        return max(self.nuc_index_quality[column].keys())
462
463    def get_score_sum_for_column(self, column):
464        return sum(score * count for score, count in self.nuc_index_quality[column].items())
465
466    def get_score_at_position_for_column(self, column, position):
467        score_value_dict = self.nuc_index_quality[column]
468        scores = sorted(score_value_dict.keys())
469        for score in scores:
470            if score_value_dict[score] <= position:
471                position -= score_value_dict[score]
472            else:
473                return score
474
475    def get_summary_statistics_for_column(self, i):
476        def _get_med_pos(size):
477            halfed = int(size / 2)
478            if size % 2 == 1:
479                return [halfed]
480            return [halfed - 1, halfed]
481        read_count = self.get_read_count_for_column(i)
482
483        min_score = self.get_score_min_for_column(i)
484        max_score = self.get_score_max_for_column(i)
485        sum_score = self.get_score_sum_for_column(i)
486        mean_score = float(sum_score) / float(read_count)
487        # get positions
488        med_pos = _get_med_pos(read_count)
489        if 0 in med_pos:
490            q1_pos = [0]
491            q3_pos = [read_count - 1]
492        else:
493            q1_pos = _get_med_pos(min(med_pos))
494            q3_pos = []
495            for pos in q1_pos:
496                q3_pos.append(max(med_pos) + 1 + pos)
497        # get scores at position
498        med_score = float(sum(self.get_score_at_position_for_column(i, pos) for pos in med_pos)) / float(len(med_pos))
499        q1 = float(sum(self.get_score_at_position_for_column(i, pos) for pos in q1_pos)) / float(len(q1_pos))
500        q3 = float(sum(self.get_score_at_position_for_column(i, pos) for pos in q3_pos)) / float(len(q3_pos))
501        # determine iqr and step
502        iqr = q3 - q1
503        step = 1.5 * iqr
504
505        # Determine whiskers and outliers
506        outliers = []
507        score_list = sorted(self.get_score_list_for_column(i))
508        left_whisker = q1 - step
509        for score in score_list:
510            if left_whisker <= score:
511                left_whisker = score
512                break
513            else:
514                outliers.append(score)
515
516        right_whisker = q3 + step
517        score_list.reverse()
518        for score in score_list:
519            if right_whisker >= score:
520                right_whisker = score
521                break
522            else:
523                outliers.append(score)
524
525        column_stats = {'read_count': read_count,
526                        'min_score': min_score,
527                        'max_score': max_score,
528                        'sum_score': sum_score,
529                        'mean_score': mean_score,
530                        'q1': q1,
531                        'med_score': med_score,
532                        'q3': q3,
533                        'iqr': iqr,
534                        'left_whisker': left_whisker,
535                        'right_whisker': right_whisker,
536                        'outliers': outliers}
537        return column_stats
538
539
540def _fastq_open_stream(fh=None, format="sanger", path=None, mode="r"):
541    if fh is None:
542        assert path is not None
543        if format and format.endswith(".gz"):
544            fh = gzip.open(path, mode=f"{mode}t")
545        elif format and format.endswith(".bz2"):
546            fh = bz2.open(path, mode=f"{mode}t")
547        else:
548            fh = open(path, f"{mode}t")
549    else:
550        if format and format.endswith(".gz"):
551            fh = gzip.GzipFile(fileobj=fh, mode=mode)
552        elif format and format.endswith(".bz2"):
553            raise Exception("bz2 formats do not support file handle inputs")
554    return fh
555
556
557class fileHandler:
558
559    def __init__(self, fh, format, path):
560        self.fh = fh
561        self.format = format
562        self.path = path
563        self._file = None
564
565    @property
566    def file(self):
567        if self._file is None:
568            self._file = _fastq_open_stream(fh=self.fh, format=self.format, path=self.path, mode=self.mode)
569        return self._file
570
571    @file.setter
572    def file(self, fh):
573        self._file = fh
574
575    def __enter__(self):
576        fh = _fastq_open_stream(fh=self.fh, format=self.format, path=self.path, mode=self.mode)
577        self._set_file_handle(fh)
578        return self
579
580    def __exit__(self, exc_type, exc_value, traceback):
581        self.close()
582
583    def _set_file_handle(self, fh):
584        # Extension point for subclasses to wrap file handler
585        self.file = fh
586
587    def close(self):
588        return self.file.close()
589
590    def _close_on_error(self):
591        # Extension point for subclasses (fastqVerboseErrorReader needs file access after error)
592        return self.close()
593
594
595class fastqReader(fileHandler):
596
597    mode = 'r'
598
599    def __init__(
600            self, fh=None, format='sanger', apply_galaxy_conventions=False, path=None, fix_id=False):
601        super().__init__(fh=fh, format=format, path=path)
602        self.apply_galaxy_conventions = apply_galaxy_conventions
603        self.fix_id = fix_id  # fix inconsistent identifiers (source: SRA data dumps)
604
605    def __next__(self):
606        rval = fastqSequencingRead.get_class_by_format(self.format)()
607
608        id_line = self._read_fastq_header()  # We need the raw line1 to compare w/line3
609        rval.identifier = id_line
610
611        self._read_fastq_sequence(rval, id_line)
612        self._read_fastq_qualityscores(rval)
613
614        rval.assert_sequence_quality_lengths()
615        if self.apply_galaxy_conventions:
616            rval.apply_galaxy_conventions()
617
618        return rval
619
620    def _read_fastq_header(self):
621        while True:
622            line = self.file.readline()
623            if not line:
624                raise StopIteration
625            line = line.rstrip('\n\r')
626            if line:
627                break
628
629        if not line.startswith('@'):
630            self.close()
631            raise fastqFormatError(f'Invalid FASTQ header: {line}')
632
633        return line
634
635    def _read_fastq_sequence(self, rval, id_line):
636        while True:
637            line = self.file.readline()
638            if not line:
639                self.close()
640                raise fastqFormatError(
641                    'Invalid FASTQ file: could not find quality score of '
642                    f'sequence identifier {rval.identifier}.')
643            line = line.rstrip('\n\r')
644
645            if not line.startswith('+'):
646                rval.append_sequence(line)
647            else:  # Sequence is over, read the quality score identifier
648                if len(line) == 1 or line[1:] == id_line[1:]:  # Line is valid
649                    rval.description = line
650                    break
651                elif self.fix_id:  # Line is not valid, but we must fix it
652                    rval.description = '+'
653                    break
654                else:  # Line is not valid, sound the alarm!
655                    self._close_on_error()
656                    raise fastqFormatError(
657                        f'Invalid FASTQ file: quality score identifier ({line})'
658                        f'does not match sequence identifier ({rval.identifier}).'
659                    )
660
661    def _read_fastq_qualityscores(self, rval):
662        while rval.insufficient_quality_length():
663            line = self.file.readline()
664            if not line:
665                break
666
667            line = line.rstrip('\n\r')
668            rval.append_quality(line)
669
670    def __iter__(self):
671        while True:
672            try:
673                yield next(self)
674            except StopIteration:
675                self.close()
676                # Catch exception and return normally
677                return
678
679
680class ReadlineCountFile:
681    def __init__(self, f):
682        self.__file = f
683        self.readline_count = 0
684
685    def readline(self, *args, **kwds):
686        self.readline_count += 1
687        return self.__file.readline(*args, **kwds)
688
689    def __getattr__(self, name):
690        return getattr(self.__file, name)
691
692
693class fastqVerboseErrorReader(fastqReader):
694    MAX_PRINT_ERROR_BYTES = 1024
695
696    def __init__(self, fh=None, **kwds):
697        super().__init__(fh=fh, **kwds)
698        self.last_good_identifier = None
699
700    def _close_on_error(self):
701        pass  # Do not close it yet: need file access to provide error details (see __next__())
702
703    def _set_file_handle(self, fh):
704        # Extension point for subclasses to wrap file handler
705        self.file = ReadlineCountFile(fh)
706
707    def __next__(self):
708        if not hasattr(self.file, "readline_count"):
709            return super().__next__()
710
711        last_good_end_offset = self.file.tell()
712        last_readline_count = self.file.readline_count
713        try:
714            block = super().__next__()
715            self.last_good_identifier = block.identifier
716            return block
717        except StopIteration as e:
718            raise e
719        except Exception as e:
720            print(f"There was an error reading your input file. Your input file is likely malformed.\nIt is suggested that you double-check your original input file for errors -- helpful information for this purpose has been provided below.\nHowever, if you think that you have encountered an actual error with this tool, please do tell us by using the bug reporting mechanism.\n\nThe reported error is: '{e}'.")
721            if self.last_good_identifier is not None:
722                print(f"The last valid FASTQ read had an identifier of '{self.last_good_identifier}'.")
723            else:
724                print("The error occurred at the start of your file and no valid FASTQ reads were found.")
725            error_offset = self.file.tell()
726            error_byte_count = error_offset - last_good_end_offset
727            print_error_bytes = min(self.MAX_PRINT_ERROR_BYTES, error_byte_count)
728            print(f"The error in your file occurs between lines '{last_readline_count + 1:d}' and '{self.file.readline_count:d}', which corresponds to byte-offsets '{last_good_end_offset:d}' and '{error_offset:d}', and contains the text ({print_error_bytes:d} of {error_byte_count:d} bytes shown):\n")
729            self.file.seek(last_good_end_offset)
730            print(self.file.read(print_error_bytes))
731            self.file.close()
732            raise e
733
734
735class fastqNamedReader(fastqReader):
736    def __init__(self, fh=None, format='sanger', apply_galaxy_conventions=False, path=None):
737        super().__init__(fh=fh, format=format, apply_galaxy_conventions=apply_galaxy_conventions, path=path)
738        self.offset_dict = {}
739        self.eof = False
740
741    def get(self, sequence_identifier):
742        # Input is either a sequence ID or a sequence object
743        if not isinstance(sequence_identifier, str):
744            # Input was a sequence object (not a sequence ID). Get the sequence ID
745            sequence_identifier = sequence_identifier.identifier
746        # Get only the ID part of the sequence header
747        sequence_id, sequence_sep, sequence_desc = sequence_identifier.partition(' ')
748        rval = None
749        if sequence_id in self.offset_dict:
750            initial_offset = self.file.tell()
751            seq_offset = self.offset_dict[sequence_id].pop(0)
752            if not self.offset_dict[sequence_id]:
753                del self.offset_dict[sequence_id]
754            self.file.seek(seq_offset)
755            rval = next(self)
756            # assert rval.id == sequence_id, 'seq id mismatch' #should be able to remove this
757            self.file.seek(initial_offset)
758        else:
759            while True:
760                offset = self.file.tell()
761                try:
762                    fastq_read = next(self)
763                except StopIteration:
764                    self.eof = True
765                    break  # eof, id not found, will return None
766                fastq_read_id, fastq_read_sep, fastq_read_desc = fastq_read.identifier.partition(' ')
767                if fastq_read_id == sequence_id:
768                    rval = fastq_read
769                    break
770                else:
771                    if fastq_read_id not in self.offset_dict:
772                        self.offset_dict[fastq_read_id] = []
773                    self.offset_dict[fastq_read_id].append(offset)
774        if rval is not None and self.apply_galaxy_conventions:
775            rval.apply_galaxy_conventions()
776        return rval
777
778    def has_data(self):
779        # returns a string representation of remaining data, or empty string (False) if no data remaining
780        eof = self.eof
781        count = 0
782        rval = ''
783        if self.offset_dict:
784            count = sum(map(len, self.offset_dict.values()))
785        if not eof:
786            offset = self.file.tell()
787            try:
788                next(self)
789            except StopIteration:
790                eof = True
791            self.file.seek(offset)
792        if count:
793            rval = f"There were {count:d} known sequence reads not utilized. "
794        if not eof:
795            rval += "An additional unknown number of reads exist in the input that were not utilized."
796        return rval
797
798
799class fastqWriter(fileHandler):
800
801    mode = 'w'
802
803    def __init__(self, fh=None, format=None, force_quality_encoding=None, path=None):
804        super().__init__(fh=fh, format=format, path=path)
805        self.force_quality_encoding = force_quality_encoding
806
807    def write(self, fastq_read):
808        if self.format:
809            fastq_read = fastq_read.convert_read_to_format(self.format, force_quality_encoding=self.force_quality_encoding)
810        self.file.write(str(fastq_read))
811
812
813class fastqJoiner:
814
815    def __init__(self, format, force_quality_encoding=None, paste=""):
816        self.format = format
817        self.force_quality_encoding = force_quality_encoding
818        outformat = fastqSequencingRead.get_class_by_format(format)
819        self.paste_sequence = paste
820        self.paste_ascii_quality = chr(outformat.ascii_max) * len(paste)
821        self.paste_decimal_quality = " ".join(str(outformat.quality_max) for x in range(len(paste)))
822
823    def join(self, read1, read2):
824        read1_id, read1_sep, read1_desc = read1.identifier.partition(' ')
825        read2_id, read2_sep, read2_desc = read2.identifier.partition(' ')
826        if read1_id.endswith('/2') and read2_id.endswith('/1'):
827            # swap 1 and 2
828            tmp = read1
829            read1 = read2
830            read2 = tmp
831            del tmp
832        if read1_id.endswith('/1') and read2_id.endswith('/2'):
833            read1_id = read1_id[:-2]
834
835        identifier = read1_id
836        if read1_desc:
837            identifier = identifier + ' ' + read1_desc
838
839        # use force quality encoding, if not present force to encoding of first read
840        force_quality_encoding = self.force_quality_encoding
841        if not force_quality_encoding:
842            if read1.is_ascii_encoded():
843                force_quality_encoding = 'ascii'
844            else:
845                force_quality_encoding = 'decimal'
846
847        new_read1 = read1.convert_read_to_format(self.format, force_quality_encoding=force_quality_encoding)
848        new_read2 = read2.convert_read_to_format(self.format, force_quality_encoding=force_quality_encoding)
849        rval = FASTQ_FORMATS[self.format]()
850        rval.identifier = identifier
851        if len(read1.description) > 1:
852            rval.description = "+" + identifier[1:]
853        else:
854            rval.description = '+'
855        if rval.sequence_space == 'color':
856            # need to handle color space joining differently
857            # convert to nuc space, join, then convert back
858            rval.sequence = rval.convert_base_to_color_space(new_read1.convert_color_to_base_space(new_read1.sequence) + self.paste_sequence + new_read2.convert_color_to_base_space(new_read2.sequence))
859        else:
860            rval.sequence = new_read1.sequence + self.paste_sequence + new_read2.sequence
861        if force_quality_encoding == 'ascii':
862            rval.quality = new_read1.quality + self.paste_ascii_quality + new_read2.quality
863        else:
864            rval.quality = f"{new_read1.quality.strip()} {self.paste_decimal_quality}"
865            rval.quality = (f"{rval.quality.strip()} {new_read2.quality.strip()}").strip()
866        return rval
867
868    def get_paired_identifier(self, fastq_read):
869        read_id, read_sep, read_desc = fastq_read.identifier.partition(' ')
870        if read_id[-2] == '/':
871            if read_id[-1] == "1":
872                read_id = f"{read_id[:-1]}2"
873            elif read_id[-1] == "2":
874                read_id = f"{read_id[:-1]}1"
875        return read_id
876
877    def is_first_mate(self, sequence_id):
878        is_first = None
879        if not isinstance(sequence_id, str):
880            sequence_id = sequence_id.identifier
881        sequence_id, sequence_sep, sequence_desc = sequence_id.partition(' ')
882        if sequence_id[-2] == '/':
883            if sequence_id[-1] == "1":
884                is_first = True
885            else:
886                is_first = False
887        return is_first
888
889
890class fastqSplitter:
891    def split(self, fastq_read):
892        length = len(fastq_read)
893        # Only reads of even lengths can be split
894        if length % 2 != 0:
895            return None, None
896        half = int(length / 2)
897        read1 = fastq_read.slice(0, half)
898        read1.identifier += "/1"
899        if len(read1.description) > 1:
900            read1.description += "/1"
901        read2 = fastq_read.slice(half, None)
902        read2.identifier += "/2"
903        if len(read2.description) > 1:
904            read2.description += "/2"
905        return read1, read2
906
907
908class fastqCombiner:
909    def __init__(self, format):
910        self.format = format
911
912    def combine(self, fasta_seq, quality_seq):
913        fastq_read = fastqSequencingRead.get_class_by_format(self.format)()
914        fastq_read.identifier = f"@{fasta_seq.identifier[1:]}"
915        fastq_read.description = '+'
916        fastq_read.sequence = fasta_seq.sequence
917        fastq_read.quality = quality_seq.sequence
918        return fastq_read
919
920
921class fastqFakeFastaScoreReader:
922    def __init__(self, format='sanger', quality_encoding=None):
923        self.fastq_read = fastqSequencingRead.get_class_by_format(format)()
924        if quality_encoding != 'decimal':
925            quality_encoding = 'ascii'
926        self.quality_encoding = quality_encoding
927
928    def close(self):
929        return  # nothing to close
930
931    def get(self, sequence):
932        assert isinstance(sequence, fastaSequence), 'fastqFakeFastaScoreReader requires a fastaSequence object as the parameter'
933        # add sequence to fastq_read, then get_sequence(), color space adapters do not have quality score values
934        self.fastq_read.sequence = sequence.sequence
935        new_sequence = fastaSequence()
936        new_sequence.identifier = sequence.identifier
937        if self.quality_encoding == 'ascii':
938            new_sequence.sequence = chr(self.fastq_read.ascii_max) * len(self.fastq_read.get_sequence())
939        else:
940            new_sequence.sequence = (f"{self.fastq_read.quality_max:d} " * len(self.fastq_read.get_sequence()))
941        return new_sequence
942
943    def has_data(self):
944        return ''  # No actual data exist, none can be remaining
945
946
947class fastqFormatError(Exception):
948    pass