src/pypdf/_cmap.py
1from binascii import Error as BinasciiError
2from binascii import unhexlify
3from collections.abc import Callable
4from functools import partial
5from hashlib import sha256
6from io import BytesIO
7from typing import Any, cast
8
9from ._codecs import adobe_glyphs, charset_encoding
10from ._utils import logger_error, logger_warning
11from .errors import LimitReachedError
12from .generic import (
13 ArrayObject,
14 DecodedStreamObject,
15 DictionaryObject,
16 NullObject,
17 StreamObject,
18)
19from .generic._data_structures import _FontFileCharacterMap
20
21_predefined_cmap: dict[str, str] = {
22 "/Identity-H": "utf-16-be",
23 "/Identity-V": "utf-16-be",
24 "/GB-EUC-H": "gbk",
25 "/GB-EUC-V": "gbk",
26 "/GBpc-EUC-H": "gb2312",
27 "/GBpc-EUC-V": "gb2312",
28 "/GBK-EUC-H": "gbk",
29 "/GBK-EUC-V": "gbk",
30 "/GBK2K-H": "gb18030",
31 "/GBK2K-V": "gb18030",
32 "/ETen-B5-H": "cp950",
33 "/ETen-B5-V": "cp950",
34 "/ETenms-B5-H": "cp950",
35 "/ETenms-B5-V": "cp950",
36 # Unicode-based CMaps: character codes are UTF-16BE, see
37 # ISO 32000-2:2020, 9.7.5.2, Note 1 and Table 116.
38 # (UniGB covers the GB18030-2000 character set but is not GB18030-encoded.)
39 "/UniCNS-UTF16-H": "utf-16-be",
40 "/UniCNS-UTF16-V": "utf-16-be",
41 "/UniGB-UTF16-H": "utf-16-be",
42 "/UniGB-UTF16-V": "utf-16-be",
43 # Japanese CMaps (PDF Reference 1.7, Appendix H)
44 "/90ms-RKSJ-H": "cp932", # Shift-JIS (JIS X 0208-1990), horizontal
45 "/90ms-RKSJ-V": "cp932", # Shift-JIS (JIS X 0208-1990), vertical
46 "/UniJIS-UTF16-H": "utf-16-be", # Unicode UTF-16BE -> JIS, horizontal
47 "/UniJIS-UTF16-V": "utf-16-be", # Unicode UTF-16BE -> JIS, vertical
48 # UCS2 in code
49}
50
51
52def get_encoding(
53 ft: DictionaryObject
54) -> tuple[str | dict[int, str], dict[Any, Any]]:
55 encoding = _parse_encoding(ft)
56 map_dict, int_entry = _parse_to_unicode(ft)
57
58 # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
59 # if cmap not empty encoding should be discarded
60 # (here transformed into identity for those characters)
61 # If encoding is a string, it is expected to be an identity translation.
62 if isinstance(encoding, dict):
63 for x in int_entry:
64 if x <= 255:
65 encoding[x] = chr(x)
66
67 return encoding, map_dict
68
69
70def _parse_encoding(
71 ft: DictionaryObject
72) -> str | dict[int, str]:
73 encoding: str | list[str] | dict[int, str] = []
74 # If ft["/Encoding"] exists, then use that for encoding. Otherwise, use StandardEncoding as a basis,
75 # and add what the embedded font file says, if present. See Table 114, PDF Reference 1.7 / 2.0
76 if "/Encoding" not in ft:
77 if "/BaseFont" in ft and cast(str, ft["/BaseFont"]) in charset_encoding:
78 # This will match Symbol and ZapfDingBats
79 return dict(
80 zip(range(256), charset_encoding[cast(str, ft["/BaseFont"])])
81 )
82
83 # Return StandardEncoding as fallback option. Note that a font's internal encoding can be used
84 # to overwrite this, which we do for Type1 fonts in _derive_character_map_from_(cff_)type1_font_file.
85 return dict(
86 zip(range(256), charset_encoding["/StandardEncoding"])
87 )
88
89 enc: str | DictionaryObject | NullObject = cast(
90 str | DictionaryObject | NullObject, ft["/Encoding"].get_object()
91 )
92 if isinstance(enc, str):
93 try:
94 # already done : enc = NameObject.unnumber(enc.encode()).decode()
95 # for #xx decoding
96 if enc in charset_encoding:
97 encoding = charset_encoding[enc].copy()
98 elif enc in _predefined_cmap:
99 encoding = _predefined_cmap[enc]
100 elif "-UCS2-" in enc:
101 encoding = "utf-16-be"
102 else:
103 raise KeyError("not found")
104 except KeyError:
105 encoding = (
106 "utf-16-be" if ft.get("/Subtype", "") == "/Type0" else charset_encoding["/StandardEncoding"].copy()
107 )
108 logger_error(
109 "Advanced encoding %(encoding)s not implemented yet, using %(new_encoding)s instead.",
110 source=__name__,
111 encoding=enc,
112 new_encoding=encoding
113 )
114 elif isinstance(enc, DictionaryObject) and "/BaseEncoding" in enc:
115 try:
116 encoding = charset_encoding[cast(str, enc["/BaseEncoding"])].copy()
117 except KeyError:
118 logger_error(
119 "Advanced encoding %(encoding)s not implemented yet",
120 source=__name__, encoding=encoding
121 )
122 encoding = charset_encoding["/StandardEncoding"].copy()
123 else:
124 encoding = charset_encoding["/StandardEncoding"].copy()
125 if isinstance(enc, DictionaryObject) and "/Differences" in enc:
126 x: int = 0
127 o: int | str
128 differences = enc["/Differences"].get_object()
129 if not isinstance(differences, ArrayObject):
130 logger_warning(
131 "Font encoding differences are not an array: %(differences)s",
132 source=__name__,
133 differences=differences,
134 )
135 differences = ArrayObject()
136 for o in differences:
137 if isinstance(o, int):
138 x = o
139 else: # isinstance(o, str):
140 try:
141 if x < len(encoding):
142 encoding[x] = _glyph_name_to_unicode(o) or o # type: ignore[index]
143 except MemoryError: # pragma: no cover # TODO: BLE001
144 raise
145 except Exception:
146 encoding[x] = o # type: ignore[index]
147 x += 1
148 if isinstance(encoding, list):
149 encoding = dict(zip(range(256), encoding))
150 return encoding
151
152
153def _parse_to_unicode(
154 ft: DictionaryObject
155) -> tuple[dict[Any, Any], list[int]]:
156 from .generic._font import HAS_FONTTOOLS # noqa: PLC0415
157
158 # We store all character mappings in map_dict. In map_dict[-1] we store the byte length
159 # of the character codes (or CIDs) encoded inside the ToUnicode stream.
160 map_dict: dict[Any, Any] = {}
161
162 # We provide the list of cmap keys in int_entry to correct encoding later on in get_encoding().
163 int_entry: list[int] = []
164
165 if "/ToUnicode" not in ft:
166 if ft.get("/Subtype", "") == "/Type1":
167 font_descriptor = ft.get("/FontDescriptor")
168 if not font_descriptor:
169 return map_dict, int_entry
170
171 # We try to read encoding from an embedded font file, if we can. See Table 126 about embedded font
172 # file organization in the PDF specification 1.7 for details.
173 font_file_handlers = (
174 # A normal Type1 font file, can be part of a Type1 or MMType1 font dictionary.
175 (
176 "/FontFile",
177 lambda _: True,
178 _derive_character_map_from_type1_font_file
179 ),
180 # A CFF Type1 font file, as part of a Type1 or MMType1 font dictionary, when subtype is Type1C.
181 (
182 "/FontFile3",
183 lambda stream: stream.get("/Subtype") == "/Type1C",
184 _derive_character_map_from_cff_type1_font_file,
185 )
186 )
187 for font_file, condition, font_file_processor in font_file_handlers:
188 if (
189 font_file in font_descriptor and
190 isinstance(font_file_dict := font_descriptor[font_file], StreamObject) and
191 condition(font_file_dict)
192 ):
193 if font_file == "/FontFile3" and not HAS_FONTTOOLS:
194 logger_warning(
195 (
196 "fontTools is required to fully parse the encoding of a CFF Type1 font in font "
197 "dictionary %(ft)s, but is not installed. Consider installing fontTools if you "
198 "encounter encoding problems."
199 ),
200 source=__name__,
201 ft=ft,
202 )
203 return map_dict, int_entry
204
205 font_file_data = font_file_dict.get_data()
206 if not font_file_data:
207 return map_dict, int_entry
208
209 return _derive_character_map_from_font_file(
210 font_file_dict, font_file_data, font_file_processor, map_dict, int_entry
211 )
212
213 return map_dict, int_entry
214
215 return {}, []
216
217 process_rg: bool = False
218 process_char: bool = False
219 multiline_rg: (
220 tuple[int, int] | None
221 ) = None # tuple = (current_char, remaining size) ; cf #1285 for example of file
222 cm = prepare_cm(ft)
223 for line in cm.split(b"\n"):
224 process_rg, process_char, multiline_rg = process_cm_line(
225 line.strip(b" \t"),
226 process_rg,
227 process_char,
228 multiline_rg,
229 map_dict,
230 int_entry,
231 )
232
233 map_dict.pop(-1, None) # Don't pass the -1 key, we only used it to temporarily store encoding length
234
235 return map_dict, int_entry
236
237
238def prepare_cm(ft: DictionaryObject) -> bytes:
239 tu = ft["/ToUnicode"]
240 cm: bytes
241 if isinstance(tu, StreamObject):
242 cm = cast(DecodedStreamObject, ft["/ToUnicode"]).get_data()
243 else: # if (tu is None) or cast(str, tu).startswith("/Identity"):
244 # the full range 0000-FFFF will be processed
245 cm = b"beginbfrange\n<0000> <0001> <0000>\nendbfrange"
246 if isinstance(cm, str):
247 cm = cm.encode()
248 # we need to prepare cm before due to missing return line in pdf printed
249 # to pdf from word
250 cm = (
251 cm.strip()
252 .replace(b"beginbfchar", b"\nbeginbfchar\n")
253 .replace(b"endbfchar", b"\nendbfchar\n")
254 .replace(b"beginbfrange", b"\nbeginbfrange\n")
255 .replace(b"endbfrange", b"\nendbfrange\n")
256 .replace(b"<<", b"\n{\n") # text between << and >> not used but
257 .replace(b">>", b"\n}\n") # some solution to find it back
258 )
259 ll = cm.split(b"<")
260 for i in range(len(ll)):
261 j = ll[i].find(b">")
262 if j >= 0:
263 if j == 0:
264 # string is empty: stash a placeholder here (see below)
265 # see https://github.com/py-pdf/pypdf/issues/1111
266 content = b"."
267 else:
268 content = ll[i][:j].replace(b" ", b"")
269 ll[i] = content + b" " + ll[i][j + 1 :]
270 cm = (
271 (b" ".join(ll))
272 .replace(b"[", b" [ ")
273 .replace(b"]", b" ]\n ")
274 .replace(b"\r", b"\n")
275 )
276 return cm
277
278
279def process_cm_line(
280 line: bytes,
281 process_rg: bool,
282 process_char: bool,
283 multiline_rg: tuple[int, int] | None,
284 map_dict: dict[Any, Any],
285 int_entry: list[int],
286) -> tuple[bool, bool, tuple[int, int] | None]:
287 if line == b"" or line[0] == 37: # 37 = %
288 return process_rg, process_char, multiline_rg
289 line = line.replace(b"\t", b" ")
290 if b"beginbfrange" in line:
291 process_rg = True
292 elif b"endbfrange" in line:
293 process_rg = False
294 elif b"beginbfchar" in line:
295 process_char = True
296 elif b"endbfchar" in line:
297 process_char = False
298 elif process_rg:
299 try:
300 multiline_rg = parse_bfrange(line, map_dict, int_entry, multiline_rg)
301 except (ValueError, IndexError) as error:
302 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
303 elif process_char:
304 try:
305 parse_bfchar(line, map_dict, int_entry)
306 except ValueError as error:
307 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
308 return process_rg, process_char, multiline_rg
309
310
311# Usual values should be up to 65_536.
312MAPPING_DICTIONARY_SIZE_LIMIT = 100_000
313
314# Typical /ToUnicode CMaps use 1-4 byte source codes.
315# This is intentionally generous.
316# The actual limit is doubled, as each byte is represented by two hex characters.
317MAX_CMAP_CODE_BYTES = 8
318MAX_CMAP_STRING_BYTES = 512
319MAX_CMAP_CODE_BYTES_LIMIT = MAX_CMAP_CODE_BYTES * 2
320MAX_CMAP_STRING_BYTES_LIMIT = MAX_CMAP_STRING_BYTES * 2
321
322
323def _check_mapping_size(size: int) -> None:
324 if size > MAPPING_DICTIONARY_SIZE_LIMIT:
325 raise LimitReachedError(f"Maximum /ToUnicode size limit reached: {size} > {MAPPING_DICTIONARY_SIZE_LIMIT}.")
326
327
328def _check_token_length(token: bytes, limit: int) -> None:
329 token_length = len(token)
330 if token_length > limit:
331 description = {
332 MAX_CMAP_CODE_BYTES_LIMIT: "code",
333 MAX_CMAP_STRING_BYTES_LIMIT: "string",
334 }.get(limit, "token")
335
336 raise LimitReachedError(
337 f"Maximum /ToUnicode {description} length exceeded: {token_length} > {limit}."
338 )
339
340
341def __parse_bfrange__decode(map_dict: dict[Any, Any], code: int) -> str:
342 # `map_dict[-1]` is the number of bytes each source code occupies. Building
343 # the bytes directly with `int.to_bytes` avoids the hex round-trip of
344 # `unhexlify(b"%%0%dX" % (map_dict[-1] * 2) % code)` (format to hex, parse
345 # the hex back to bytes), which is measurably cheaper for large maps.
346 return code.to_bytes(map_dict[-1], "big").decode(
347 "charmap" if map_dict[-1] == 1 else "utf-16-be",
348 "surrogatepass",
349 )
350
351
352def parse_bfrange(
353 line: bytes,
354 map_dict: dict[Any, Any],
355 int_entry: list[int],
356 multiline_rg: tuple[int, int] | None,
357) -> tuple[int, int] | None:
358 lst = line.split()
359 closure_found = False
360 entry_count = len(int_entry)
361 _check_mapping_size(entry_count)
362 decode_utf16 = partial(bytes.decode, encoding="utf-16-be", errors="surrogatepass")
363 if multiline_rg is not None:
364 a = multiline_rg[0] # a, b not in the current line
365 b = multiline_rg[1]
366 for sq in lst:
367 if sq == b"]":
368 closure_found = True
369 break
370 _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
371 entry_count += 1
372 _check_mapping_size(entry_count)
373 map_dict[
374 __parse_bfrange__decode(map_dict=map_dict, code=a)
375 ] = decode_utf16(unhexlify(sq))
376 int_entry.append(a)
377 a += 1
378 else:
379 _check_token_length(lst[0], limit=MAX_CMAP_CODE_BYTES_LIMIT)
380 _check_token_length(lst[1], limit=MAX_CMAP_CODE_BYTES_LIMIT)
381 a = int(lst[0], 16)
382 b = int(lst[1], 16)
383 if a < 0 or b < 0:
384 # int(token, 16) accepts a leading "-"; negative range bounds
385 # would leak an OverflowError from the to_bytes encoding below.
386 raise ValueError(f"Negative bfrange source range: {lst[0]!r} {lst[1]!r}")
387 nbi = max(len(lst[0]), len(lst[1]))
388 map_dict[-1] = (nbi + 1) // 2
389 if lst[2] == b"[":
390 for sq in lst[3:]:
391 if sq == b"]":
392 closure_found = True
393 break
394 _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
395 entry_count += 1
396 _check_mapping_size(entry_count)
397 map_dict[
398 __parse_bfrange__decode(map_dict=map_dict, code=a)
399 ] = decode_utf16(unhexlify(sq))
400 int_entry.append(a)
401 a += 1
402 else: # case without list
403 _check_token_length(lst[2], limit=MAX_CMAP_STRING_BYTES_LIMIT)
404 c = int(lst[2], 16)
405 fmt2 = b"%%0%dX" % max(4, len(lst[2]))
406 closure_found = True
407 range_size = max(0, b - a + 1)
408 _check_mapping_size(entry_count + range_size) # This can be checked beforehand.
409 while a <= b:
410 destination = unhexlify(fmt2 % c)
411 _check_token_length(destination, limit=MAX_CMAP_CODE_BYTES_LIMIT)
412 map_dict[
413 __parse_bfrange__decode(map_dict=map_dict, code=a)
414 ] = decode_utf16(destination)
415 int_entry.append(a)
416 a += 1
417 c += 1
418 return None if closure_found else (a, b)
419
420
421def parse_bfchar(line: bytes, map_dict: dict[Any, Any], int_entry: list[int]) -> None:
422 lst = [x for x in line.split(b" ") if x]
423 lst_length = len(lst)
424 if lst_length == 0:
425 logger_warning("Skipping broken line %(line)r: Line is empty.", source=__name__, line=line)
426 return
427 if lst_length % 2:
428 logger_warning("Ignoring final token of odd-length line %(line)r.", source=__name__, line=line)
429
430 new_count = lst_length // 2
431 _check_mapping_size(len(int_entry) + new_count) # This can be checked beforehand.
432 map_dict[-1] = len(lst[0]) // 2
433
434 for index in range(0, lst_length - 1, 2):
435 source = lst[index]
436 destination = lst[index + 1]
437
438 _check_token_length(source, limit=MAX_CMAP_CODE_BYTES_LIMIT)
439
440 map_to = ""
441 # placeholder (see above) means empty string
442 if destination != b".":
443 _check_token_length(destination, limit=MAX_CMAP_STRING_BYTES_LIMIT)
444 try:
445 map_to = unhexlify(destination).decode(
446 "charmap" if len(destination) < 4 else "utf-16-be", "surrogatepass"
447 ) # join is here as some cases where the code was split
448 except BinasciiError as exception:
449 logger_warning(
450 "Got invalid hex string: %(exception)s (%(destination)r)",
451 source=__name__,
452 exception=exception,
453 destination=destination,
454 )
455 map_dict[
456 unhexlify(source).decode(
457 "charmap" if map_dict[-1] == 1 else "utf-16-be", "surrogatepass"
458 )
459 ] = map_to
460 int_entry.append(int(source, 16))
461
462
463def _glyph_name_to_unicode(glyph_name: str) -> str | None:
464 try:
465 return adobe_glyphs[glyph_name]
466 except KeyError:
467 if glyph_name.startswith("/uni"):
468 code_point = glyph_name[4:]
469 elif glyph_name.startswith("/u"):
470 code_point = glyph_name[2:]
471 else:
472 return None
473 try:
474 return chr(int(code_point, 16))
475 except ValueError: # pragma: no cover
476 return None
477
478
479def _derive_character_map_from_font_file(
480 font_file: StreamObject,
481 font_data: bytes,
482 font_file_processor: Callable[[bytes, dict[Any, Any], list[int]], tuple[dict[Any, Any], list[int]]],
483 map_dict: dict[Any, Any],
484 int_entry: list[int],
485) -> tuple[dict[Any, Any], list[int]]:
486 """
487 Derive the character map from an embedded font program, caching the result on its stream.
488
489 The character map depends only on the decoded font data, and the same font program is reached from every
490 font dictionary, resource name and page referencing its stream. Parsing a CFF font program with fontTools is
491 expensive, so each stream is parsed once and reused until its data changes, which a digest of the data
492 detects (a writer may replace the font program). See #4156.
493 """
494 digest = sha256(font_data).digest()
495 cached = font_file._font_file_character_map
496 if cached is None or cached.digest != digest:
497 cached_map_dict, cached_int_entry = font_file_processor(font_data, {}, [])
498 cached = _FontFileCharacterMap(digest, cached_map_dict, cached_int_entry)
499 font_file._font_file_character_map = cached
500 map_dict.update(cached.map_dict)
501 int_entry.extend(cached.int_entry)
502 return map_dict, int_entry
503
504
505def _derive_character_map_from_cff_type1_font_file(
506 font_data: bytes,
507 map_dict: dict[Any, Any],
508 int_entry: list[int],
509) -> tuple[dict[Any, Any], list[int]]:
510 try:
511 from fontTools.cffLib import CFFFontSet # noqa: PLC0415
512 cff_set = CFFFontSet()
513 cff_set.decompile(BytesIO(font_data), None)
514 cff_font = cff_set.topDictIndex[0]
515 cff_encoding = cff_font.Encoding
516 # Encoding can fall back to literal strings "StandardEncoding" or "ExpertEncoding", which we do not parse.
517 if isinstance(cff_encoding, str):
518 return map_dict, int_entry
519 for i in range(min(len(cff_encoding), 256)):
520 glyph_name = cff_encoding[i]
521 if not glyph_name or glyph_name == ".notdef":
522 continue
523 if unipoint := _glyph_name_to_unicode(f"/{glyph_name}"):
524 map_dict[chr(i)] = unipoint
525 int_entry.append(i)
526 except MemoryError: # pragma: no cover # TODO: BLE001
527 raise
528 except Exception:
529 return map_dict, int_entry
530 else:
531 return map_dict, int_entry
532
533
534def _derive_character_map_from_type1_font_file(
535 font_data: bytes,
536 map_dict: dict[Any, Any],
537 int_entry: list[int],
538) -> tuple[dict[Any, Any], list[int]]:
539 txt = font_data.split(b"eexec\n")[0] # Only the clear part
540 encoding_part = txt.split(b"/Encoding")
541 if len(encoding_part) < 2:
542 return map_dict, int_entry
543 txt = encoding_part[1] # To get the encoding part
544 lines = txt.replace(b"\r", b"\n").split(b"\n")
545 for li in lines:
546 if li.startswith(b"dup"):
547 words = [_w for _w in li.split(b" ") if _w != b""]
548 if len(words) < 3 or (len(words) > 3 and words[3] != b"put"):
549 continue
550 try:
551 i = int(words[1])
552 except ValueError:
553 continue
554
555 unipoint = _glyph_name_to_unicode(words[2].decode())
556 if unipoint:
557 map_dict[chr(i)] = unipoint
558 int_entry.append(i)
559
560 return map_dict, int_entry