Source code

Download source archive ↓

src/pypdf/_cmap.py

1from binascii import Error as BinasciiError
2from binascii import unhexlify
3from collections.abc import Callable
4from functools import partial
5from hashlib import sha256
6from io import BytesIO
7from typing import Any, cast
8
9from ._codecs import adobe_glyphs, charset_encoding
10from ._utils import logger_error, logger_warning
11from .errors import LimitReachedError
12from .generic import (
13    ArrayObject,
14    DecodedStreamObject,
15    DictionaryObject,
16    NullObject,
17    StreamObject,
18)
19from .generic._data_structures import _FontFileCharacterMap
20
21_predefined_cmap: dict[str, str] = {
22    "/Identity-H": "utf-16-be",
23    "/Identity-V": "utf-16-be",
24    "/GB-EUC-H": "gbk",
25    "/GB-EUC-V": "gbk",
26    "/GBpc-EUC-H": "gb2312",
27    "/GBpc-EUC-V": "gb2312",
28    "/GBK-EUC-H": "gbk",
29    "/GBK-EUC-V": "gbk",
30    "/GBK2K-H": "gb18030",
31    "/GBK2K-V": "gb18030",
32    "/ETen-B5-H": "cp950",
33    "/ETen-B5-V": "cp950",
34    "/ETenms-B5-H": "cp950",
35    "/ETenms-B5-V": "cp950",
36    # Unicode-based CMaps: character codes are UTF-16BE, see
37    # ISO 32000-2:2020, 9.7.5.2, Note 1 and Table 116.
38    # (UniGB covers the GB18030-2000 character set but is not GB18030-encoded.)
39    "/UniCNS-UTF16-H": "utf-16-be",
40    "/UniCNS-UTF16-V": "utf-16-be",
41    "/UniGB-UTF16-H": "utf-16-be",
42    "/UniGB-UTF16-V": "utf-16-be",
43    # Japanese CMaps (PDF Reference 1.7, Appendix H)
44    "/90ms-RKSJ-H": "cp932",  # Shift-JIS (JIS X 0208-1990), horizontal
45    "/90ms-RKSJ-V": "cp932",  # Shift-JIS (JIS X 0208-1990), vertical
46    "/UniJIS-UTF16-H": "utf-16-be",  # Unicode UTF-16BE -> JIS, horizontal
47    "/UniJIS-UTF16-V": "utf-16-be",  # Unicode UTF-16BE -> JIS, vertical
48    # UCS2 in code
49}
50
51
52def get_encoding(
53    ft: DictionaryObject
54) -> tuple[str | dict[int, str], dict[Any, Any]]:
55    encoding = _parse_encoding(ft)
56    map_dict, int_entry = _parse_to_unicode(ft)
57
58    # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
59    #   if cmap not empty encoding should be discarded
60    #   (here transformed into identity for those characters)
61    # If encoding is a string, it is expected to be an identity translation.
62    if isinstance(encoding, dict):
63        for x in int_entry:
64            if x <= 255:
65                encoding[x] = chr(x)
66
67    return encoding, map_dict
68
69
70def _parse_encoding(
71    ft: DictionaryObject
72) -> str | dict[int, str]:
73    encoding: str | list[str] | dict[int, str] = []
74    # If ft["/Encoding"] exists, then use that for encoding. Otherwise, use StandardEncoding as a basis,
75    # and add what the embedded font file says, if present. See Table 114, PDF Reference 1.7 / 2.0
76    if "/Encoding" not in ft:
77        if "/BaseFont" in ft and cast(str, ft["/BaseFont"]) in charset_encoding:
78            # This will match Symbol and ZapfDingBats
79            return dict(
80                zip(range(256), charset_encoding[cast(str, ft["/BaseFont"])])
81            )
82
83        # Return StandardEncoding as fallback option. Note that a font's internal encoding can be used
84        # to overwrite this, which we do for Type1 fonts in _derive_character_map_from_(cff_)type1_font_file.
85        return dict(
86            zip(range(256), charset_encoding["/StandardEncoding"])
87        )
88
89    enc: str | DictionaryObject | NullObject = cast(
90        str | DictionaryObject | NullObject, ft["/Encoding"].get_object()
91    )
92    if isinstance(enc, str):
93        try:
94            # already done : enc = NameObject.unnumber(enc.encode()).decode()
95            # for #xx decoding
96            if enc in charset_encoding:
97                encoding = charset_encoding[enc].copy()
98            elif enc in _predefined_cmap:
99                encoding = _predefined_cmap[enc]
100            elif "-UCS2-" in enc:
101                encoding = "utf-16-be"
102            else:
103                raise KeyError("not found")
104        except KeyError:
105            encoding = (
106                "utf-16-be" if ft.get("/Subtype", "") == "/Type0" else charset_encoding["/StandardEncoding"].copy()
107            )
108            logger_error(
109                "Advanced encoding %(encoding)s not implemented yet, using %(new_encoding)s instead.",
110                source=__name__,
111                encoding=enc,
112                new_encoding=encoding
113            )
114    elif isinstance(enc, DictionaryObject) and "/BaseEncoding" in enc:
115        try:
116            encoding = charset_encoding[cast(str, enc["/BaseEncoding"])].copy()
117        except KeyError:
118            logger_error(
119                "Advanced encoding %(encoding)s not implemented yet",
120                source=__name__, encoding=encoding
121            )
122            encoding = charset_encoding["/StandardEncoding"].copy()
123    else:
124        encoding = charset_encoding["/StandardEncoding"].copy()
125    if isinstance(enc, DictionaryObject) and "/Differences" in enc:
126        x: int = 0
127        o: int | str
128        differences = enc["/Differences"].get_object()
129        if not isinstance(differences, ArrayObject):
130            logger_warning(
131                "Font encoding differences are not an array: %(differences)s",
132                source=__name__,
133                differences=differences,
134            )
135            differences = ArrayObject()
136        for o in differences:
137            if isinstance(o, int):
138                x = o
139            else:  # isinstance(o, str):
140                try:
141                    if x < len(encoding):
142                        encoding[x] = _glyph_name_to_unicode(o) or o  # type: ignore[index]
143                except MemoryError:  # pragma: no cover  # TODO: BLE001
144                    raise
145                except Exception:
146                    encoding[x] = o  # type: ignore[index]
147                x += 1
148    if isinstance(encoding, list):
149        encoding = dict(zip(range(256), encoding))
150    return encoding
151
152
153def _parse_to_unicode(
154    ft: DictionaryObject
155) -> tuple[dict[Any, Any], list[int]]:
156    from .generic._font import HAS_FONTTOOLS  # noqa: PLC0415
157
158    # We store all character mappings in map_dict. In map_dict[-1] we store the byte length
159    # of the character codes (or CIDs) encoded inside the ToUnicode stream.
160    map_dict: dict[Any, Any] = {}
161
162    # We provide the list of cmap keys in int_entry to correct encoding later on in get_encoding().
163    int_entry: list[int] = []
164
165    if "/ToUnicode" not in ft:
166        if ft.get("/Subtype", "") == "/Type1":
167            font_descriptor = ft.get("/FontDescriptor")
168            if not font_descriptor:
169                return map_dict, int_entry
170
171            # We try to read encoding from an embedded font file, if we can. See Table 126 about embedded font
172            # file organization in the PDF specification 1.7 for details.
173            font_file_handlers = (
174                # A normal Type1 font file, can be part of a Type1 or MMType1 font dictionary.
175                (
176                    "/FontFile",
177                    lambda _: True,
178                    _derive_character_map_from_type1_font_file
179                ),
180                # A CFF Type1 font file, as part of a Type1 or MMType1 font dictionary, when subtype is Type1C.
181                (
182                    "/FontFile3",
183                    lambda stream: stream.get("/Subtype") == "/Type1C",
184                    _derive_character_map_from_cff_type1_font_file,
185                )
186            )
187            for font_file, condition, font_file_processor in font_file_handlers:
188                if (
189                    font_file in font_descriptor and
190                    isinstance(font_file_dict := font_descriptor[font_file], StreamObject) and
191                    condition(font_file_dict)
192                ):
193                    if font_file == "/FontFile3" and not HAS_FONTTOOLS:
194                        logger_warning(
195                            (
196                                "fontTools is required to fully parse the encoding of a CFF Type1 font in font "
197                                "dictionary %(ft)s, but is not installed. Consider installing fontTools if you "
198                                "encounter encoding problems."
199                            ),
200                            source=__name__,
201                            ft=ft,
202                        )
203                        return map_dict, int_entry
204
205                    font_file_data = font_file_dict.get_data()
206                    if not font_file_data:
207                        return map_dict, int_entry
208
209                    return _derive_character_map_from_font_file(
210                        font_file_dict, font_file_data, font_file_processor, map_dict, int_entry
211                    )
212
213            return map_dict, int_entry
214
215        return {}, []
216
217    process_rg: bool = False
218    process_char: bool = False
219    multiline_rg: (
220        tuple[int, int] | None
221    ) = None  # tuple = (current_char, remaining size) ; cf #1285 for example of file
222    cm = prepare_cm(ft)
223    for line in cm.split(b"\n"):
224        process_rg, process_char, multiline_rg = process_cm_line(
225            line.strip(b" \t"),
226            process_rg,
227            process_char,
228            multiline_rg,
229            map_dict,
230            int_entry,
231        )
232
233    map_dict.pop(-1, None)  # Don't pass the -1 key, we only used it to temporarily store encoding length
234
235    return map_dict, int_entry
236
237
238def prepare_cm(ft: DictionaryObject) -> bytes:
239    tu = ft["/ToUnicode"]
240    cm: bytes
241    if isinstance(tu, StreamObject):
242        cm = cast(DecodedStreamObject, ft["/ToUnicode"]).get_data()
243    else:  # if (tu is None) or cast(str, tu).startswith("/Identity"):
244        # the full range 0000-FFFF will be processed
245        cm = b"beginbfrange\n<0000> <0001> <0000>\nendbfrange"
246    if isinstance(cm, str):
247        cm = cm.encode()
248    # we need to prepare cm before due to missing return line in pdf printed
249    # to pdf from word
250    cm = (
251        cm.strip()
252        .replace(b"beginbfchar", b"\nbeginbfchar\n")
253        .replace(b"endbfchar", b"\nendbfchar\n")
254        .replace(b"beginbfrange", b"\nbeginbfrange\n")
255        .replace(b"endbfrange", b"\nendbfrange\n")
256        .replace(b"<<", b"\n{\n")  # text between << and >> not used but
257        .replace(b">>", b"\n}\n")  # some solution to find it back
258    )
259    ll = cm.split(b"<")
260    for i in range(len(ll)):
261        j = ll[i].find(b">")
262        if j >= 0:
263            if j == 0:
264                # string is empty: stash a placeholder here (see below)
265                # see https://github.com/py-pdf/pypdf/issues/1111
266                content = b"."
267            else:
268                content = ll[i][:j].replace(b" ", b"")
269            ll[i] = content + b" " + ll[i][j + 1 :]
270    cm = (
271        (b" ".join(ll))
272        .replace(b"[", b" [ ")
273        .replace(b"]", b" ]\n ")
274        .replace(b"\r", b"\n")
275    )
276    return cm
277
278
279def process_cm_line(
280    line: bytes,
281    process_rg: bool,
282    process_char: bool,
283    multiline_rg: tuple[int, int] | None,
284    map_dict: dict[Any, Any],
285    int_entry: list[int],
286) -> tuple[bool, bool, tuple[int, int] | None]:
287    if line == b"" or line[0] == 37:  # 37 = %
288        return process_rg, process_char, multiline_rg
289    line = line.replace(b"\t", b" ")
290    if b"beginbfrange" in line:
291        process_rg = True
292    elif b"endbfrange" in line:
293        process_rg = False
294    elif b"beginbfchar" in line:
295        process_char = True
296    elif b"endbfchar" in line:
297        process_char = False
298    elif process_rg:
299        try:
300            multiline_rg = parse_bfrange(line, map_dict, int_entry, multiline_rg)
301        except (ValueError, IndexError) as error:
302            logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
303    elif process_char:
304        try:
305            parse_bfchar(line, map_dict, int_entry)
306        except ValueError as error:
307            logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
308    return process_rg, process_char, multiline_rg
309
310
311# Usual values should be up to 65_536.
312MAPPING_DICTIONARY_SIZE_LIMIT = 100_000
313
314# Typical /ToUnicode CMaps use 1-4 byte source codes.
315# This is intentionally generous.
316# The actual limit is doubled, as each byte is represented by two hex characters.
317MAX_CMAP_CODE_BYTES = 8
318MAX_CMAP_STRING_BYTES = 512
319MAX_CMAP_CODE_BYTES_LIMIT = MAX_CMAP_CODE_BYTES * 2
320MAX_CMAP_STRING_BYTES_LIMIT = MAX_CMAP_STRING_BYTES * 2
321
322
323def _check_mapping_size(size: int) -> None:
324    if size > MAPPING_DICTIONARY_SIZE_LIMIT:
325        raise LimitReachedError(f"Maximum /ToUnicode size limit reached: {size} > {MAPPING_DICTIONARY_SIZE_LIMIT}.")
326
327
328def _check_token_length(token: bytes, limit: int) -> None:
329    token_length = len(token)
330    if token_length > limit:
331        description = {
332            MAX_CMAP_CODE_BYTES_LIMIT: "code",
333            MAX_CMAP_STRING_BYTES_LIMIT: "string",
334        }.get(limit, "token")
335
336        raise LimitReachedError(
337            f"Maximum /ToUnicode {description} length exceeded: {token_length} > {limit}."
338        )
339
340
341def __parse_bfrange__decode(map_dict: dict[Any, Any], code: int) -> str:
342    # `map_dict[-1]` is the number of bytes each source code occupies. Building
343    # the bytes directly with `int.to_bytes` avoids the hex round-trip of
344    # `unhexlify(b"%%0%dX" % (map_dict[-1] * 2) % code)` (format to hex, parse
345    # the hex back to bytes), which is measurably cheaper for large maps.
346    return code.to_bytes(map_dict[-1], "big").decode(
347        "charmap" if map_dict[-1] == 1 else "utf-16-be",
348        "surrogatepass",
349    )
350
351
352def parse_bfrange(
353    line: bytes,
354    map_dict: dict[Any, Any],
355    int_entry: list[int],
356    multiline_rg: tuple[int, int] | None,
357) -> tuple[int, int] | None:
358    lst = line.split()
359    closure_found = False
360    entry_count = len(int_entry)
361    _check_mapping_size(entry_count)
362    decode_utf16 = partial(bytes.decode, encoding="utf-16-be", errors="surrogatepass")
363    if multiline_rg is not None:
364        a = multiline_rg[0]  # a, b not in the current line
365        b = multiline_rg[1]
366        for sq in lst:
367            if sq == b"]":
368                closure_found = True
369                break
370            _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
371            entry_count += 1
372            _check_mapping_size(entry_count)
373            map_dict[
374                __parse_bfrange__decode(map_dict=map_dict, code=a)
375            ] = decode_utf16(unhexlify(sq))
376            int_entry.append(a)
377            a += 1
378    else:
379        _check_token_length(lst[0], limit=MAX_CMAP_CODE_BYTES_LIMIT)
380        _check_token_length(lst[1], limit=MAX_CMAP_CODE_BYTES_LIMIT)
381        a = int(lst[0], 16)
382        b = int(lst[1], 16)
383        if a < 0 or b < 0:
384            # int(token, 16) accepts a leading "-"; negative range bounds
385            # would leak an OverflowError from the to_bytes encoding below.
386            raise ValueError(f"Negative bfrange source range: {lst[0]!r} {lst[1]!r}")
387        nbi = max(len(lst[0]), len(lst[1]))
388        map_dict[-1] = (nbi + 1) // 2
389        if lst[2] == b"[":
390            for sq in lst[3:]:
391                if sq == b"]":
392                    closure_found = True
393                    break
394                _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
395                entry_count += 1
396                _check_mapping_size(entry_count)
397                map_dict[
398                    __parse_bfrange__decode(map_dict=map_dict, code=a)
399                ] = decode_utf16(unhexlify(sq))
400                int_entry.append(a)
401                a += 1
402        else:  # case without list
403            _check_token_length(lst[2], limit=MAX_CMAP_STRING_BYTES_LIMIT)
404            c = int(lst[2], 16)
405            fmt2 = b"%%0%dX" % max(4, len(lst[2]))
406            closure_found = True
407            range_size = max(0, b - a + 1)
408            _check_mapping_size(entry_count + range_size)  # This can be checked beforehand.
409            while a <= b:
410                destination = unhexlify(fmt2 % c)
411                _check_token_length(destination, limit=MAX_CMAP_CODE_BYTES_LIMIT)
412                map_dict[
413                    __parse_bfrange__decode(map_dict=map_dict, code=a)
414                ] = decode_utf16(destination)
415                int_entry.append(a)
416                a += 1
417                c += 1
418    return None if closure_found else (a, b)
419
420
421def parse_bfchar(line: bytes, map_dict: dict[Any, Any], int_entry: list[int]) -> None:
422    lst = [x for x in line.split(b" ") if x]
423    lst_length = len(lst)
424    if lst_length == 0:
425        logger_warning("Skipping broken line %(line)r: Line is empty.", source=__name__, line=line)
426        return
427    if lst_length % 2:
428        logger_warning("Ignoring final token of odd-length line %(line)r.", source=__name__, line=line)
429
430    new_count = lst_length // 2
431    _check_mapping_size(len(int_entry) + new_count)  # This can be checked beforehand.
432    map_dict[-1] = len(lst[0]) // 2
433
434    for index in range(0, lst_length - 1, 2):
435        source = lst[index]
436        destination = lst[index + 1]
437
438        _check_token_length(source, limit=MAX_CMAP_CODE_BYTES_LIMIT)
439
440        map_to = ""
441        # placeholder (see above) means empty string
442        if destination != b".":
443            _check_token_length(destination, limit=MAX_CMAP_STRING_BYTES_LIMIT)
444            try:
445                map_to = unhexlify(destination).decode(
446                    "charmap" if len(destination) < 4 else "utf-16-be", "surrogatepass"
447                )  # join is here as some cases where the code was split
448            except BinasciiError as exception:
449                logger_warning(
450                    "Got invalid hex string: %(exception)s (%(destination)r)",
451                    source=__name__,
452                    exception=exception,
453                    destination=destination,
454                )
455        map_dict[
456            unhexlify(source).decode(
457                "charmap" if map_dict[-1] == 1 else "utf-16-be", "surrogatepass"
458            )
459        ] = map_to
460        int_entry.append(int(source, 16))
461
462
463def _glyph_name_to_unicode(glyph_name: str) -> str | None:
464    try:
465        return adobe_glyphs[glyph_name]
466    except KeyError:
467        if glyph_name.startswith("/uni"):
468            code_point = glyph_name[4:]
469        elif glyph_name.startswith("/u"):
470            code_point = glyph_name[2:]
471        else:
472            return None
473        try:
474            return chr(int(code_point, 16))
475        except ValueError:  # pragma: no cover
476            return None
477
478
479def _derive_character_map_from_font_file(
480    font_file: StreamObject,
481    font_data: bytes,
482    font_file_processor: Callable[[bytes, dict[Any, Any], list[int]], tuple[dict[Any, Any], list[int]]],
483    map_dict: dict[Any, Any],
484    int_entry: list[int],
485) -> tuple[dict[Any, Any], list[int]]:
486    """
487    Derive the character map from an embedded font program, caching the result on its stream.
488
489    The character map depends only on the decoded font data, and the same font program is reached from every
490    font dictionary, resource name and page referencing its stream. Parsing a CFF font program with fontTools is
491    expensive, so each stream is parsed once and reused until its data changes, which a digest of the data
492    detects (a writer may replace the font program). See #4156.
493    """
494    digest = sha256(font_data).digest()
495    cached = font_file._font_file_character_map
496    if cached is None or cached.digest != digest:
497        cached_map_dict, cached_int_entry = font_file_processor(font_data, {}, [])
498        cached = _FontFileCharacterMap(digest, cached_map_dict, cached_int_entry)
499        font_file._font_file_character_map = cached
500    map_dict.update(cached.map_dict)
501    int_entry.extend(cached.int_entry)
502    return map_dict, int_entry
503
504
505def _derive_character_map_from_cff_type1_font_file(
506    font_data: bytes,
507    map_dict: dict[Any, Any],
508    int_entry: list[int],
509) -> tuple[dict[Any, Any], list[int]]:
510    try:
511        from fontTools.cffLib import CFFFontSet  # noqa: PLC0415
512        cff_set = CFFFontSet()
513        cff_set.decompile(BytesIO(font_data), None)
514        cff_font = cff_set.topDictIndex[0]
515        cff_encoding = cff_font.Encoding
516        # Encoding can fall back to literal strings "StandardEncoding" or "ExpertEncoding", which we do not parse.
517        if isinstance(cff_encoding, str):
518            return map_dict, int_entry
519        for i in range(min(len(cff_encoding), 256)):
520            glyph_name = cff_encoding[i]
521            if not glyph_name or glyph_name == ".notdef":
522                continue
523            if unipoint := _glyph_name_to_unicode(f"/{glyph_name}"):
524                map_dict[chr(i)] = unipoint
525                int_entry.append(i)
526    except MemoryError:  # pragma: no cover  # TODO: BLE001
527        raise
528    except Exception:
529        return map_dict, int_entry
530    else:
531        return map_dict, int_entry
532
533
534def _derive_character_map_from_type1_font_file(
535    font_data: bytes,
536    map_dict: dict[Any, Any],
537    int_entry: list[int],
538) -> tuple[dict[Any, Any], list[int]]:
539    txt = font_data.split(b"eexec\n")[0]  # Only the clear part
540    encoding_part = txt.split(b"/Encoding")
541    if len(encoding_part) < 2:
542        return map_dict, int_entry
543    txt = encoding_part[1]  # To get the encoding part
544    lines = txt.replace(b"\r", b"\n").split(b"\n")
545    for li in lines:
546        if li.startswith(b"dup"):
547            words = [_w for _w in li.split(b" ") if _w != b""]
548            if len(words) < 3 or (len(words) > 3 and words[3] != b"put"):
549                continue
550            try:
551                i = int(words[1])
552            except ValueError:
553                continue
554
555            unipoint = _glyph_name_to_unicode(words[2].decode())
556            if unipoint:
557                map_dict[chr(i)] = unipoint
558                int_entry.append(i)
559
560    return map_dict, int_entry