Source code

Download source archive ↓

src/pypdf/generic/_image_inline.py

1# Copyright (c) 2024, pypdf contributors
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8# * Redistributions of source code must retain the above copyright notice,
9# this list of conditions and the following disclaimer.
10# * Redistributions in binary form must reproduce the above copyright notice,
11# this list of conditions and the following disclaimer in the documentation
12# and/or other materials provided with the distribution.
13# * The name of the author may not be used to endorse or promote products
14# derived from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26# POSSIBILITY OF SUCH DAMAGE.
27
28import logging
29from io import BytesIO
30from math import ceil
31from typing import IO, Any
32
33from .._configuration import get_configuration
34from .._utils import (
35    WHITESPACES,
36    WHITESPACES_AS_BYTES,
37    BinaryStreamType,
38    StreamType,
39    logger_warning,
40    read_non_whitespace,
41)
42from ..errors import LimitReachedError, PdfReadError, PdfStreamError
43from ._data_structures import ContentStream, DictionaryObject, read_object
44from ._utils import StreamParsingBudget
45
46logger = logging.getLogger(__name__)
47
48# An inline image should be used only for small images (4096 bytes or less),
49# but allow twice this for cases where this has been exceeded.
50BUFFER_SIZE = 8192
51
52
53def _check_end_image_marker(stream: StreamType) -> bool:
54    ei_tok = read_non_whitespace(stream)
55    ei_tok += stream.read(2)
56    stream.seek(-len(ei_tok), 1)
57    return ei_tok[:2] == b"EI" and (ei_tok[2:3] == b"" or ei_tok[2:3] in WHITESPACES)
58
59
60def extract_inline__ascii_hex_decode(stream: StreamType) -> bytes:
61    """
62    Extract HexEncoded stream from inline image.
63    The stream will be moved onto the EI.
64    """
65    data_out = bytearray()
66    # Read data until delimiter > and EI as backup.
67    while True:
68        data_buffered = read_non_whitespace(stream) + (read_bytes := stream.read(BUFFER_SIZE))
69        if not data_buffered or not read_bytes:
70            raise PdfReadError("Unexpected end of stream.")
71        pos_tok = data_buffered.find(b">")
72        if pos_tok >= 0:  # found >
73            data_out += data_buffered[: pos_tok + 1]
74            stream.seek(-len(data_buffered) + pos_tok + 1, 1)
75            break
76        pos_ei = data_buffered.find(b"EI")
77        if pos_ei >= 0:  # found EI
78            stream.seek(-len(data_buffered) + pos_ei - 1, 1)
79            c = stream.read(1)
80            while c in WHITESPACES:
81                stream.seek(-2, 1)
82                c = stream.read(1)
83                pos_ei -= 1
84            data_out += data_buffered[:pos_ei]
85            break
86        if len(data_buffered) == 2:
87            data_out += data_buffered
88            raise PdfReadError("Unexpected end of stream.")
89        # Neither > nor EI found
90        data_out += data_buffered[:-2]
91        stream.seek(-2, 1)
92
93    if not _check_end_image_marker(stream):
94        raise PdfReadError("EI stream not found.")
95    return bytes(data_out)
96
97
98def extract_inline__ascii85_decode(stream: StreamType) -> bytes:
99    """
100    Extract A85 stream from inline image.
101    The stream will be moved onto the EI.
102    """
103    data_out = bytearray()
104    # Read data until delimiter ~>
105    while True:
106        data_buffered = read_non_whitespace(stream) + (read_bytes := stream.read(BUFFER_SIZE))
107        if not data_buffered or not read_bytes:
108            raise PdfReadError("Unexpected end of stream.")
109        pos_tok = data_buffered.find(b"~>")
110        if pos_tok >= 0:  # found!
111            data_out += data_buffered[: pos_tok + 2]
112            stream.seek(-len(data_buffered) + pos_tok + 2, 1)
113            break
114        if len(data_buffered) == 2:  # end of buffer
115            data_out += data_buffered
116            raise PdfReadError("Unexpected end of stream.")
117        data_out += data_buffered[
118            :-2
119        ]  # back by one char in case of in the middle of ~>
120        stream.seek(-2, 1)
121
122    if not _check_end_image_marker(stream):
123        raise PdfReadError("EI stream not found.")
124    return bytes(data_out)
125
126
127def extract_inline__run_length_decode(stream: StreamType) -> bytes:
128    """
129    Extract RL (RunLengthDecode) stream from inline image.
130    The stream will be moved onto the EI.
131    """
132    data_out = bytearray()
133    # Read data until delimiter 128
134    while True:
135        data_buffered = stream.read(BUFFER_SIZE)
136        if not data_buffered:
137            raise PdfReadError("Unexpected end of stream.")
138        pos_tok = data_buffered.find(b"\x80")
139        if pos_tok >= 0:  # found
140            # Ideally, we could just use plain run-length decoding here, where 80_16 = 128_10
141            # marks the EOD. But there apparently are cases like in issue #3517, where we have
142            # an inline image with up to 51 EOD markers. In these cases, be resilient here and
143            # use the default `EI` marker detection instead. Please note that this fallback
144            # still omits special `EI` handling within the stream, but for now assume that having
145            # both of these cases occur at the same time is very unlikely (and the image stream
146            # is broken anyway).
147            # For now, do not skip over more than one whitespace character.
148            after_token = data_buffered[pos_tok + 1 : pos_tok + 4]
149            if after_token.startswith(b"EI") or after_token.endswith(b"EI"):
150                data_out += data_buffered[: pos_tok + 1]
151                stream.seek(-len(data_buffered) + pos_tok + 1, 1)
152            else:
153                logger_warning("Early EOD in RunLengthDecode of inline image, using fallback.", source=__name__)
154                ei_marker = data_buffered.find(b"EI")
155                if ei_marker > 0:
156                    data_out += data_buffered[: ei_marker]
157                    stream.seek(-len(data_buffered) + ei_marker - 1, 1)
158            break
159        data_out += data_buffered
160
161    if not _check_end_image_marker(stream):
162        raise PdfReadError("EI stream not found.")
163    return bytes(data_out)
164
165
166def extract_inline__dct_decode(stream: BinaryStreamType) -> bytes:
167    """
168    Extract DCT (JPEG) stream from inline image.
169    The stream will be moved onto the EI.
170    """
171    def read(length: int) -> bytes:
172        # If 0 bytes are returned, and *size* was not 0, this indicates end of file.
173        # If the object is in non-blocking mode and no bytes are available, `None` is returned.
174        _result = stream.read(length)
175        if _result is None or len(_result) != length:
176            raise PdfReadError("Unexpected end of stream.")
177        return _result
178
179    data_out = bytearray()
180    # Read Blocks of data (ID/Size/data) up to ID=FF/D9
181    # https://www.digicamsoft.com/itu/itu-t81-36.html
182    not_first = False
183    while True:
184        c = read(1)
185        if not_first or (c == b"\xff"):
186            data_out += c
187        if c != b"\xff":
188            continue
189        not_first = True
190        c = read(1)
191        data_out += c
192        if c == b"\xff":
193            stream.seek(-1, 1)  # pragma: no cover
194        elif c == b"\x00":  # stuffing
195            pass
196        elif c == b"\xd9":  # end
197            break
198        elif c in (
199            b"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc9\xca\xcb\xcc\xcd\xce\xcf"
200            b"\xda\xdb\xdc\xdd\xde\xdf"
201            b"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xfe"
202        ):
203            c = read(2)
204            data_out += c
205            sz = c[0] * 256 + c[1]
206            data_out += read(sz - 2)
207
208    if not _check_end_image_marker(stream):
209        raise PdfReadError("EI stream not found.")
210    return bytes(data_out)
211
212
213def _is_preceded_by_whitespace(stream_out: BytesIO, position: int) -> bool:
214    """
215    Check whether the byte in front of `position` is a whitespace.
216
217    Returns:
218        Whether the preceding byte is whitespace. If no preceding byte exists,
219        returns ``False`` because the ``EI`` marker is at the beginning of the
220        image data and is therefore not delimited by anything.
221    """
222    if position <= 0:
223        return False
224    saved_position = stream_out.tell()
225    # Avoid exporting the BytesIO buffer: PyPy may keep the view alive,
226    # preventing later writes from resizing the output stream.
227    stream_out.seek(position - 1)
228    is_whitespace = stream_out.read(1) in WHITESPACES
229    stream_out.seek(saved_position)
230    return is_whitespace
231
232
233def _is_followed_by_closing_operator(stream: StreamType, first_byte: bytes) -> bool:
234    """
235    Check whether `first_byte` starts an exact `Q` or `EMC` operator.
236
237    `first_byte` has already been consumed from the stream, which is restored to its
238    current position afterwards. Looking at this byte alone is not sufficient, as
239    binary image data can easily be mistaken for one of these operators.
240    """
241    position = stream.tell()
242    if first_byte == b"Q":
243        suffix = stream.read(1)
244        is_closing_operator = suffix == b"" or suffix in WHITESPACES
245    elif first_byte == b"E":
246        suffix = stream.read(3)
247        is_closing_operator = suffix[:2] == b"MC" and (suffix[2:3] == b"" or suffix[2:3] in WHITESPACES)
248    else:
249        is_closing_operator = False
250    stream.seek(position, 0)
251    return is_closing_operator
252
253
254def extract_inline_default(stream: StreamType) -> bytes:
255    """Legacy method, used by default"""
256    stream_out = BytesIO()
257    # Read the inline image, while checking for EI (End Image) operator.
258    while True:
259        data_buffered = stream.read(BUFFER_SIZE)
260        if not data_buffered:
261            raise PdfReadError("Unexpected end of stream.")
262        pos_ei = data_buffered.find(
263            b"E"
264        )  # We can not look straight for "EI" because it may not have been loaded in the buffer
265
266        if pos_ei == -1:
267            stream_out.write(data_buffered)
268        else:
269            # Write out everything including E (the one from EI to be removed)
270            stream_out.write(data_buffered[0 : pos_ei + 1])
271            sav_pos_ei = stream_out.tell() - 1
272            # Seek back in the stream to read the E next
273            stream.seek(pos_ei + 1 - len(data_buffered), 1)
274            saved_pos = stream.tell()
275            # Check for End Image
276            tok2 = stream.read(1)  # I of "EI"
277            if tok2 != b"I":
278                stream.seek(saved_pos, 0)
279                continue
280            tok3 = stream.read(1)  # possible space after "EI"
281            if tok3 == b"":
282                # The `EI` marker is at the very end of the stream. There can be
283                # no trailing binary data, so this is unambiguously the end of the
284                # inline image (#3468).
285                stream.seek(saved_pos - 1, 0)
286                stream_out.truncate(sav_pos_ei)
287                break
288            if tok3 not in WHITESPACES:
289                stream.seek(saved_pos, 0)
290                continue
291            while tok3 in WHITESPACES:
292                tok3 = stream.read(1)
293            # The `EI` marker should be delimited by whitespace on both sides. Preserve the
294            # legacy tolerance for a missing leading delimiter only when it is followed by an
295            # exact closing operator, as binary image data would match otherwise.
296            if not _is_preceded_by_whitespace(stream_out, sav_pos_ei) and not _is_followed_by_closing_operator(
297                stream, tok3
298            ):
299                stream.seek(saved_pos, 0)
300                continue
301            if is_followed_by_binary_data(stream):
302                # Inline image contains `EI ` sequence usually marking the end of it, but
303                # is followed by binary data which does not make sense for the actual end.
304                stream.seek(saved_pos, 0)
305                continue
306            # Data contains [\s]EI[\s](Q|EMC): 4 chars are sufficient
307            # remove E(I) wrongly inserted earlier
308            stream.seek(saved_pos - 1, 0)
309            stream_out.truncate(sav_pos_ei)
310            break
311
312    return stream_out.getvalue()
313
314
315def is_followed_by_binary_data(stream: IO[bytes], length: int = 10) -> bool:
316    """
317    Check if the next bytes of the stream look like binary image data or regular page content.
318
319    This is just some heuristics due to the PDF specification being too imprecise about
320    inline images containing the `EI` marker which would end an image. Starting with PDF 2.0,
321    we finally get a mandatory length field, but with (proper) PDF 2.0 support being very limited
322    everywhere, we should not expect to be able to remove such hacks in the near future - especially
323    considering legacy documents as well.
324
325    The actual implementation draws some inspiration from
326    https://github.com/itext/itext-java/blob/9.1.0/kernel/src/main/java/com/itextpdf/kernel/pdf/canvas/parser/util/InlineImageParsingUtils.java
327    """
328    position = stream.tell()
329    data = stream.read(length)
330    stream.seek(position)
331    if not data:
332        return False
333    operator_start = None
334    operator_end = None
335
336    for index, byte in enumerate(data):
337        if byte < 32 and byte not in WHITESPACES_AS_BYTES:
338            # This covers all characters not being displayable directly, although omitting whitespace
339            # to allow for operator detection.
340            return True
341        is_whitespace = byte in WHITESPACES_AS_BYTES
342        if operator_start is None and not is_whitespace:
343            # Interpret all other non-whitespace characters as the start of an operation.
344            operator_start = index
345        if operator_start is not None and is_whitespace:
346            # A whitespace stops an operation.
347            # Assume that having an inline image with tons of whitespace is rather unlikely.
348            operator_end = index
349            break
350
351    if operator_start is None:
352        # Inline images should not have tons of whitespaces, which would lead to no operator start.
353        return False
354    if operator_end is None:
355        # We probably are inside an operation.
356        operator_end = length
357    operator_length = operator_end - operator_start
358    operator = data[operator_start:operator_end]
359    if operator.startswith(b"/") and operator_length > 1:
360        # Name object.
361        return False
362    if operator.replace(b".", b"").isdigit():
363        # Graphics operator, for example a move. A number (integer or float).
364        return False
365    if operator_length > 3:  # noqa: SIM103
366        # Usually, the operators inside a content stream should not have more than three characters,
367        # especially after an inline image.
368        return True
369    return False
370
371
372def _read_settings_dictionary(
373        *,
374        content_stream: ContentStream,
375        stream: StreamType,
376        parsing_budget: StreamParsingBudget,
377        remaining_count: int | None
378) -> DictionaryObject:
379    settings = DictionaryObject()
380
381    while True:
382        tok = read_non_whitespace(stream)
383        if not tok:
384            raise PdfReadError("Unexpected end of stream.")
385        stream.seek(-1, 1)
386        if tok == b"I":
387            # "ID" - begin of image data
388            break
389        remaining_count = ContentStream._check_limits(
390            remaining_elements=remaining_count, parsing_budget=parsing_budget, token_decrement=2
391        )
392
393        key = read_object(stream, content_stream.pdf, parsing_budget=parsing_budget)
394        _ = read_non_whitespace(stream)
395        stream.seek(-1, 1)
396        value = read_object(stream, content_stream.pdf, parsing_budget=parsing_budget)
397        settings[key] = value
398
399    return settings
400
401
402def extract_inline__not_set(
403        *,
404        content_stream: ContentStream,
405        saved_position: int,
406        settings: DictionaryObject,
407        stream: StreamType
408) -> bytes:
409    color_space = settings.get("/CS", "")
410    if isinstance(color_space, list):
411        color_space = color_space[0] if color_space else ""
412    if not isinstance(color_space, str):
413        color_space = ""
414
415    if "RGB" in color_space:
416        bytes_per_pixel: float = 3
417    elif "CMYK" in color_space:
418        bytes_per_pixel = 4
419    else:
420        bits_per_pixel = settings.get(
421            "/BPC",
422            8 if color_space in {"/I", "/G", "/Indexed", "/DeviceGray"} else -1,
423        )
424        if isinstance(bits_per_pixel, (int, float)) and bits_per_pixel > 0:
425            bytes_per_pixel = bits_per_pixel / 8.0
426        else:
427            data = extract_inline_default(stream)
428
429            # Move to the `EI` if possible.
430            _ei = read_non_whitespace(stream)
431            stream.seek(-1, 1)
432
433            return data
434
435    width = settings.get("/W")
436    height = settings.get("/H")
437    if not isinstance(width, int) or not isinstance(height, int) or not bytes_per_pixel:
438        sample_length = -1
439    else:
440        sample_length = ceil(width * bytes_per_pixel) * height
441    remaining_bytes = len(content_stream._data) - saved_position
442    if sample_length > get_configuration().image_maximum_buffer_size:
443        # TODO: Replace by general inline image limit and employ everywhere.
444        raise LimitReachedError("Requested image size exceeds total limit.")
445
446    if 0 < sample_length < remaining_bytes:
447        data = stream.read(sample_length)
448    else:
449        # Without usable dimensions the raw sample length is unknown;
450        # fall back to scanning for the `EI` marker.
451        data = extract_inline_default(stream)
452
453    # Move to the `EI` if possible.
454    _ei = read_non_whitespace(stream)
455    stream.seek(-1, 1)
456
457    return data
458
459
460def _handle_filters(
461        *,
462        filters: Any,
463        stream: StreamType,
464        settings: DictionaryObject,
465        content_stream: ContentStream,
466        saved_position: int
467) -> bytes:
468    filter_to_use = filters
469    if isinstance(filters, list):
470        filter_to_use = filter_to_use[0] if filter_to_use else "not set"
471    if not isinstance(filter_to_use, str):
472        # A valid filter is a name or an array of names; anything else
473        # cannot match the abbreviations below, so treat it as unfiltered.
474        filter_to_use = "not set"
475
476    if "AHx" in filter_to_use or "ASCIIHexDecode" in filter_to_use:
477        return extract_inline__ascii_hex_decode(stream)
478    if "A85" in filter_to_use or "ASCII85Decode" in filter_to_use:
479        return extract_inline__ascii85_decode(stream)
480    if "RL" in filter_to_use or "RunLengthDecode" in filter_to_use:
481        return extract_inline__run_length_decode(stream)
482    if "DCT" in filter_to_use or "DCTDecode" in filter_to_use:
483        return extract_inline__dct_decode(stream)
484    if filter_to_use == "not set":
485        return extract_inline__not_set(
486            settings=settings, stream=stream, content_stream=content_stream, saved_position=saved_position
487        )
488    return extract_inline_default(stream)
489
490
491def read_inline_image(
492        *,
493        content_stream: ContentStream,
494        stream: StreamType,
495        parsing_budget: StreamParsingBudget
496) -> dict[str, DictionaryObject | bytes]:
497    """Begin reading the inline image just after the "BI" - begin image."""
498    remaining_count: int | None = get_configuration().dictionary_maximum_element_count
499
500    # First read the dictionary of settings.
501    settings = _read_settings_dictionary(
502        content_stream=content_stream, stream=stream, parsing_budget=parsing_budget, remaining_count=remaining_count
503    )
504
505    # We are at the beginning of ID now.
506    read = stream.read(3)
507    assert read[:2] == b"ID"
508
509    filters = settings.get("/F", settings.get("/Filter", "not set"))
510    saved_position = stream.tell()
511
512    data = _handle_filters(
513        filters=filters, stream=stream, settings=settings, content_stream=content_stream, saved_position=saved_position
514    )
515
516    ei = stream.read(3)
517    # An `EI` at the very end of the stream yields only two bytes; rewinding
518    # unconditionally would step back into the marker (#3468).
519    if len(ei) == 3:
520        stream.seek(-1, 1)
521    ei_trailing = ei[2:3]
522    if ei[:2] != b"EI" or (ei_trailing != b"" and ei_trailing not in WHITESPACES):
523        # Deal with wrong/missing `EI` tags. Example: Wrong dimensions.
524        stream.seek(saved_position, 0)
525        data = extract_inline_default(stream)
526        ei = stream.read(3)
527        if len(ei) == 3:
528            stream.seek(-1, 1)
529        ei_trailing = ei[2:3]
530        if ei[:2] != b"EI" or (
531            ei_trailing != b"" and ei_trailing not in WHITESPACES
532        ):  # pragma: no cover
533            # Check the same condition again. This should never fail as
534            # edge cases are covered by `extract_inline_default` above,
535            # but check this to make sure that we are behind the `EI` afterwards.
536            raise PdfStreamError(
537                f"Could not extract inline image, even using fallback. Expected 'EI', got {ei!r}"
538            )
539
540    return {"settings": settings, "data": data}