src/pypdf/generic/_image_inline.py
1# Copyright (c) 2024, pypdf contributors
2# All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8# * Redistributions of source code must retain the above copyright notice,
9# this list of conditions and the following disclaimer.
10# * Redistributions in binary form must reproduce the above copyright notice,
11# this list of conditions and the following disclaimer in the documentation
12# and/or other materials provided with the distribution.
13# * The name of the author may not be used to endorse or promote products
14# derived from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26# POSSIBILITY OF SUCH DAMAGE.
27
28import logging
29from io import BytesIO
30from math import ceil
31from typing import IO, Any
32
33from .._configuration import get_configuration
34from .._utils import (
35 WHITESPACES,
36 WHITESPACES_AS_BYTES,
37 BinaryStreamType,
38 StreamType,
39 logger_warning,
40 read_non_whitespace,
41)
42from ..errors import LimitReachedError, PdfReadError, PdfStreamError
43from ._data_structures import ContentStream, DictionaryObject, read_object
44from ._utils import StreamParsingBudget
45
46logger = logging.getLogger(__name__)
47
48# An inline image should be used only for small images (4096 bytes or less),
49# but allow twice this for cases where this has been exceeded.
50BUFFER_SIZE = 8192
51
52
53def _check_end_image_marker(stream: StreamType) -> bool:
54 ei_tok = read_non_whitespace(stream)
55 ei_tok += stream.read(2)
56 stream.seek(-len(ei_tok), 1)
57 return ei_tok[:2] == b"EI" and (ei_tok[2:3] == b"" or ei_tok[2:3] in WHITESPACES)
58
59
60def extract_inline__ascii_hex_decode(stream: StreamType) -> bytes:
61 """
62 Extract HexEncoded stream from inline image.
63 The stream will be moved onto the EI.
64 """
65 data_out = bytearray()
66 # Read data until delimiter > and EI as backup.
67 while True:
68 data_buffered = read_non_whitespace(stream) + (read_bytes := stream.read(BUFFER_SIZE))
69 if not data_buffered or not read_bytes:
70 raise PdfReadError("Unexpected end of stream.")
71 pos_tok = data_buffered.find(b">")
72 if pos_tok >= 0: # found >
73 data_out += data_buffered[: pos_tok + 1]
74 stream.seek(-len(data_buffered) + pos_tok + 1, 1)
75 break
76 pos_ei = data_buffered.find(b"EI")
77 if pos_ei >= 0: # found EI
78 stream.seek(-len(data_buffered) + pos_ei - 1, 1)
79 c = stream.read(1)
80 while c in WHITESPACES:
81 stream.seek(-2, 1)
82 c = stream.read(1)
83 pos_ei -= 1
84 data_out += data_buffered[:pos_ei]
85 break
86 if len(data_buffered) == 2:
87 data_out += data_buffered
88 raise PdfReadError("Unexpected end of stream.")
89 # Neither > nor EI found
90 data_out += data_buffered[:-2]
91 stream.seek(-2, 1)
92
93 if not _check_end_image_marker(stream):
94 raise PdfReadError("EI stream not found.")
95 return bytes(data_out)
96
97
98def extract_inline__ascii85_decode(stream: StreamType) -> bytes:
99 """
100 Extract A85 stream from inline image.
101 The stream will be moved onto the EI.
102 """
103 data_out = bytearray()
104 # Read data until delimiter ~>
105 while True:
106 data_buffered = read_non_whitespace(stream) + (read_bytes := stream.read(BUFFER_SIZE))
107 if not data_buffered or not read_bytes:
108 raise PdfReadError("Unexpected end of stream.")
109 pos_tok = data_buffered.find(b"~>")
110 if pos_tok >= 0: # found!
111 data_out += data_buffered[: pos_tok + 2]
112 stream.seek(-len(data_buffered) + pos_tok + 2, 1)
113 break
114 if len(data_buffered) == 2: # end of buffer
115 data_out += data_buffered
116 raise PdfReadError("Unexpected end of stream.")
117 data_out += data_buffered[
118 :-2
119 ] # back by one char in case of in the middle of ~>
120 stream.seek(-2, 1)
121
122 if not _check_end_image_marker(stream):
123 raise PdfReadError("EI stream not found.")
124 return bytes(data_out)
125
126
127def extract_inline__run_length_decode(stream: StreamType) -> bytes:
128 """
129 Extract RL (RunLengthDecode) stream from inline image.
130 The stream will be moved onto the EI.
131 """
132 data_out = bytearray()
133 # Read data until delimiter 128
134 while True:
135 data_buffered = stream.read(BUFFER_SIZE)
136 if not data_buffered:
137 raise PdfReadError("Unexpected end of stream.")
138 pos_tok = data_buffered.find(b"\x80")
139 if pos_tok >= 0: # found
140 # Ideally, we could just use plain run-length decoding here, where 80_16 = 128_10
141 # marks the EOD. But there apparently are cases like in issue #3517, where we have
142 # an inline image with up to 51 EOD markers. In these cases, be resilient here and
143 # use the default `EI` marker detection instead. Please note that this fallback
144 # still omits special `EI` handling within the stream, but for now assume that having
145 # both of these cases occur at the same time is very unlikely (and the image stream
146 # is broken anyway).
147 # For now, do not skip over more than one whitespace character.
148 after_token = data_buffered[pos_tok + 1 : pos_tok + 4]
149 if after_token.startswith(b"EI") or after_token.endswith(b"EI"):
150 data_out += data_buffered[: pos_tok + 1]
151 stream.seek(-len(data_buffered) + pos_tok + 1, 1)
152 else:
153 logger_warning("Early EOD in RunLengthDecode of inline image, using fallback.", source=__name__)
154 ei_marker = data_buffered.find(b"EI")
155 if ei_marker > 0:
156 data_out += data_buffered[: ei_marker]
157 stream.seek(-len(data_buffered) + ei_marker - 1, 1)
158 break
159 data_out += data_buffered
160
161 if not _check_end_image_marker(stream):
162 raise PdfReadError("EI stream not found.")
163 return bytes(data_out)
164
165
166def extract_inline__dct_decode(stream: BinaryStreamType) -> bytes:
167 """
168 Extract DCT (JPEG) stream from inline image.
169 The stream will be moved onto the EI.
170 """
171 def read(length: int) -> bytes:
172 # If 0 bytes are returned, and *size* was not 0, this indicates end of file.
173 # If the object is in non-blocking mode and no bytes are available, `None` is returned.
174 _result = stream.read(length)
175 if _result is None or len(_result) != length:
176 raise PdfReadError("Unexpected end of stream.")
177 return _result
178
179 data_out = bytearray()
180 # Read Blocks of data (ID/Size/data) up to ID=FF/D9
181 # https://www.digicamsoft.com/itu/itu-t81-36.html
182 not_first = False
183 while True:
184 c = read(1)
185 if not_first or (c == b"\xff"):
186 data_out += c
187 if c != b"\xff":
188 continue
189 not_first = True
190 c = read(1)
191 data_out += c
192 if c == b"\xff":
193 stream.seek(-1, 1) # pragma: no cover
194 elif c == b"\x00": # stuffing
195 pass
196 elif c == b"\xd9": # end
197 break
198 elif c in (
199 b"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc9\xca\xcb\xcc\xcd\xce\xcf"
200 b"\xda\xdb\xdc\xdd\xde\xdf"
201 b"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xfe"
202 ):
203 c = read(2)
204 data_out += c
205 sz = c[0] * 256 + c[1]
206 data_out += read(sz - 2)
207
208 if not _check_end_image_marker(stream):
209 raise PdfReadError("EI stream not found.")
210 return bytes(data_out)
211
212
213def _is_preceded_by_whitespace(stream_out: BytesIO, position: int) -> bool:
214 """
215 Check whether the byte in front of `position` is a whitespace.
216
217 Returns:
218 Whether the preceding byte is whitespace. If no preceding byte exists,
219 returns ``False`` because the ``EI`` marker is at the beginning of the
220 image data and is therefore not delimited by anything.
221 """
222 if position <= 0:
223 return False
224 saved_position = stream_out.tell()
225 # Avoid exporting the BytesIO buffer: PyPy may keep the view alive,
226 # preventing later writes from resizing the output stream.
227 stream_out.seek(position - 1)
228 is_whitespace = stream_out.read(1) in WHITESPACES
229 stream_out.seek(saved_position)
230 return is_whitespace
231
232
233def _is_followed_by_closing_operator(stream: StreamType, first_byte: bytes) -> bool:
234 """
235 Check whether `first_byte` starts an exact `Q` or `EMC` operator.
236
237 `first_byte` has already been consumed from the stream, which is restored to its
238 current position afterwards. Looking at this byte alone is not sufficient, as
239 binary image data can easily be mistaken for one of these operators.
240 """
241 position = stream.tell()
242 if first_byte == b"Q":
243 suffix = stream.read(1)
244 is_closing_operator = suffix == b"" or suffix in WHITESPACES
245 elif first_byte == b"E":
246 suffix = stream.read(3)
247 is_closing_operator = suffix[:2] == b"MC" and (suffix[2:3] == b"" or suffix[2:3] in WHITESPACES)
248 else:
249 is_closing_operator = False
250 stream.seek(position, 0)
251 return is_closing_operator
252
253
254def extract_inline_default(stream: StreamType) -> bytes:
255 """Legacy method, used by default"""
256 stream_out = BytesIO()
257 # Read the inline image, while checking for EI (End Image) operator.
258 while True:
259 data_buffered = stream.read(BUFFER_SIZE)
260 if not data_buffered:
261 raise PdfReadError("Unexpected end of stream.")
262 pos_ei = data_buffered.find(
263 b"E"
264 ) # We can not look straight for "EI" because it may not have been loaded in the buffer
265
266 if pos_ei == -1:
267 stream_out.write(data_buffered)
268 else:
269 # Write out everything including E (the one from EI to be removed)
270 stream_out.write(data_buffered[0 : pos_ei + 1])
271 sav_pos_ei = stream_out.tell() - 1
272 # Seek back in the stream to read the E next
273 stream.seek(pos_ei + 1 - len(data_buffered), 1)
274 saved_pos = stream.tell()
275 # Check for End Image
276 tok2 = stream.read(1) # I of "EI"
277 if tok2 != b"I":
278 stream.seek(saved_pos, 0)
279 continue
280 tok3 = stream.read(1) # possible space after "EI"
281 if tok3 == b"":
282 # The `EI` marker is at the very end of the stream. There can be
283 # no trailing binary data, so this is unambiguously the end of the
284 # inline image (#3468).
285 stream.seek(saved_pos - 1, 0)
286 stream_out.truncate(sav_pos_ei)
287 break
288 if tok3 not in WHITESPACES:
289 stream.seek(saved_pos, 0)
290 continue
291 while tok3 in WHITESPACES:
292 tok3 = stream.read(1)
293 # The `EI` marker should be delimited by whitespace on both sides. Preserve the
294 # legacy tolerance for a missing leading delimiter only when it is followed by an
295 # exact closing operator, as binary image data would match otherwise.
296 if not _is_preceded_by_whitespace(stream_out, sav_pos_ei) and not _is_followed_by_closing_operator(
297 stream, tok3
298 ):
299 stream.seek(saved_pos, 0)
300 continue
301 if is_followed_by_binary_data(stream):
302 # Inline image contains `EI ` sequence usually marking the end of it, but
303 # is followed by binary data which does not make sense for the actual end.
304 stream.seek(saved_pos, 0)
305 continue
306 # Data contains [\s]EI[\s](Q|EMC): 4 chars are sufficient
307 # remove E(I) wrongly inserted earlier
308 stream.seek(saved_pos - 1, 0)
309 stream_out.truncate(sav_pos_ei)
310 break
311
312 return stream_out.getvalue()
313
314
315def is_followed_by_binary_data(stream: IO[bytes], length: int = 10) -> bool:
316 """
317 Check if the next bytes of the stream look like binary image data or regular page content.
318
319 This is just some heuristics due to the PDF specification being too imprecise about
320 inline images containing the `EI` marker which would end an image. Starting with PDF 2.0,
321 we finally get a mandatory length field, but with (proper) PDF 2.0 support being very limited
322 everywhere, we should not expect to be able to remove such hacks in the near future - especially
323 considering legacy documents as well.
324
325 The actual implementation draws some inspiration from
326 https://github.com/itext/itext-java/blob/9.1.0/kernel/src/main/java/com/itextpdf/kernel/pdf/canvas/parser/util/InlineImageParsingUtils.java
327 """
328 position = stream.tell()
329 data = stream.read(length)
330 stream.seek(position)
331 if not data:
332 return False
333 operator_start = None
334 operator_end = None
335
336 for index, byte in enumerate(data):
337 if byte < 32 and byte not in WHITESPACES_AS_BYTES:
338 # This covers all characters not being displayable directly, although omitting whitespace
339 # to allow for operator detection.
340 return True
341 is_whitespace = byte in WHITESPACES_AS_BYTES
342 if operator_start is None and not is_whitespace:
343 # Interpret all other non-whitespace characters as the start of an operation.
344 operator_start = index
345 if operator_start is not None and is_whitespace:
346 # A whitespace stops an operation.
347 # Assume that having an inline image with tons of whitespace is rather unlikely.
348 operator_end = index
349 break
350
351 if operator_start is None:
352 # Inline images should not have tons of whitespaces, which would lead to no operator start.
353 return False
354 if operator_end is None:
355 # We probably are inside an operation.
356 operator_end = length
357 operator_length = operator_end - operator_start
358 operator = data[operator_start:operator_end]
359 if operator.startswith(b"/") and operator_length > 1:
360 # Name object.
361 return False
362 if operator.replace(b".", b"").isdigit():
363 # Graphics operator, for example a move. A number (integer or float).
364 return False
365 if operator_length > 3: # noqa: SIM103
366 # Usually, the operators inside a content stream should not have more than three characters,
367 # especially after an inline image.
368 return True
369 return False
370
371
372def _read_settings_dictionary(
373 *,
374 content_stream: ContentStream,
375 stream: StreamType,
376 parsing_budget: StreamParsingBudget,
377 remaining_count: int | None
378) -> DictionaryObject:
379 settings = DictionaryObject()
380
381 while True:
382 tok = read_non_whitespace(stream)
383 if not tok:
384 raise PdfReadError("Unexpected end of stream.")
385 stream.seek(-1, 1)
386 if tok == b"I":
387 # "ID" - begin of image data
388 break
389 remaining_count = ContentStream._check_limits(
390 remaining_elements=remaining_count, parsing_budget=parsing_budget, token_decrement=2
391 )
392
393 key = read_object(stream, content_stream.pdf, parsing_budget=parsing_budget)
394 _ = read_non_whitespace(stream)
395 stream.seek(-1, 1)
396 value = read_object(stream, content_stream.pdf, parsing_budget=parsing_budget)
397 settings[key] = value
398
399 return settings
400
401
402def extract_inline__not_set(
403 *,
404 content_stream: ContentStream,
405 saved_position: int,
406 settings: DictionaryObject,
407 stream: StreamType
408) -> bytes:
409 color_space = settings.get("/CS", "")
410 if isinstance(color_space, list):
411 color_space = color_space[0] if color_space else ""
412 if not isinstance(color_space, str):
413 color_space = ""
414
415 if "RGB" in color_space:
416 bytes_per_pixel: float = 3
417 elif "CMYK" in color_space:
418 bytes_per_pixel = 4
419 else:
420 bits_per_pixel = settings.get(
421 "/BPC",
422 8 if color_space in {"/I", "/G", "/Indexed", "/DeviceGray"} else -1,
423 )
424 if isinstance(bits_per_pixel, (int, float)) and bits_per_pixel > 0:
425 bytes_per_pixel = bits_per_pixel / 8.0
426 else:
427 data = extract_inline_default(stream)
428
429 # Move to the `EI` if possible.
430 _ei = read_non_whitespace(stream)
431 stream.seek(-1, 1)
432
433 return data
434
435 width = settings.get("/W")
436 height = settings.get("/H")
437 if not isinstance(width, int) or not isinstance(height, int) or not bytes_per_pixel:
438 sample_length = -1
439 else:
440 sample_length = ceil(width * bytes_per_pixel) * height
441 remaining_bytes = len(content_stream._data) - saved_position
442 if sample_length > get_configuration().image_maximum_buffer_size:
443 # TODO: Replace by general inline image limit and employ everywhere.
444 raise LimitReachedError("Requested image size exceeds total limit.")
445
446 if 0 < sample_length < remaining_bytes:
447 data = stream.read(sample_length)
448 else:
449 # Without usable dimensions the raw sample length is unknown;
450 # fall back to scanning for the `EI` marker.
451 data = extract_inline_default(stream)
452
453 # Move to the `EI` if possible.
454 _ei = read_non_whitespace(stream)
455 stream.seek(-1, 1)
456
457 return data
458
459
460def _handle_filters(
461 *,
462 filters: Any,
463 stream: StreamType,
464 settings: DictionaryObject,
465 content_stream: ContentStream,
466 saved_position: int
467) -> bytes:
468 filter_to_use = filters
469 if isinstance(filters, list):
470 filter_to_use = filter_to_use[0] if filter_to_use else "not set"
471 if not isinstance(filter_to_use, str):
472 # A valid filter is a name or an array of names; anything else
473 # cannot match the abbreviations below, so treat it as unfiltered.
474 filter_to_use = "not set"
475
476 if "AHx" in filter_to_use or "ASCIIHexDecode" in filter_to_use:
477 return extract_inline__ascii_hex_decode(stream)
478 if "A85" in filter_to_use or "ASCII85Decode" in filter_to_use:
479 return extract_inline__ascii85_decode(stream)
480 if "RL" in filter_to_use or "RunLengthDecode" in filter_to_use:
481 return extract_inline__run_length_decode(stream)
482 if "DCT" in filter_to_use or "DCTDecode" in filter_to_use:
483 return extract_inline__dct_decode(stream)
484 if filter_to_use == "not set":
485 return extract_inline__not_set(
486 settings=settings, stream=stream, content_stream=content_stream, saved_position=saved_position
487 )
488 return extract_inline_default(stream)
489
490
491def read_inline_image(
492 *,
493 content_stream: ContentStream,
494 stream: StreamType,
495 parsing_budget: StreamParsingBudget
496) -> dict[str, DictionaryObject | bytes]:
497 """Begin reading the inline image just after the "BI" - begin image."""
498 remaining_count: int | None = get_configuration().dictionary_maximum_element_count
499
500 # First read the dictionary of settings.
501 settings = _read_settings_dictionary(
502 content_stream=content_stream, stream=stream, parsing_budget=parsing_budget, remaining_count=remaining_count
503 )
504
505 # We are at the beginning of ID now.
506 read = stream.read(3)
507 assert read[:2] == b"ID"
508
509 filters = settings.get("/F", settings.get("/Filter", "not set"))
510 saved_position = stream.tell()
511
512 data = _handle_filters(
513 filters=filters, stream=stream, settings=settings, content_stream=content_stream, saved_position=saved_position
514 )
515
516 ei = stream.read(3)
517 # An `EI` at the very end of the stream yields only two bytes; rewinding
518 # unconditionally would step back into the marker (#3468).
519 if len(ei) == 3:
520 stream.seek(-1, 1)
521 ei_trailing = ei[2:3]
522 if ei[:2] != b"EI" or (ei_trailing != b"" and ei_trailing not in WHITESPACES):
523 # Deal with wrong/missing `EI` tags. Example: Wrong dimensions.
524 stream.seek(saved_position, 0)
525 data = extract_inline_default(stream)
526 ei = stream.read(3)
527 if len(ei) == 3:
528 stream.seek(-1, 1)
529 ei_trailing = ei[2:3]
530 if ei[:2] != b"EI" or (
531 ei_trailing != b"" and ei_trailing not in WHITESPACES
532 ): # pragma: no cover
533 # Check the same condition again. This should never fail as
534 # edge cases are covered by `extract_inline_default` above,
535 # but check this to make sure that we are behind the `EI` afterwards.
536 raise PdfStreamError(
537 f"Could not extract inline image, even using fallback. Expected 'EI', got {ei!r}"
538 )
539
540 return {"settings": settings, "data": data}