Source code

Download source archive ↓

src/pypdf/_text_extraction/__init__.py

1"""
2Code related to text extraction.
3
4Some parts are still in _page.py. In doubt, they will stay there.
5"""
6
7import math
8import re
9from collections.abc import Callable, Mapping
10from typing import Any, Literal
11
12from .._utils import RTL_CHARACTER_RANGES, is_char_neutral, is_char_rtl
13from ..generic import ContentStream, DictionaryObject, TextStringObject, encode_pdfdocencoding
14from ..generic._font import Font
15
16CUSTOM_RTL_MIN: str = ""
17CUSTOM_RTL_MAX: str = ""
18CUSTOM_RTL_SPECIAL_CHARS: str = ""
19LAYOUT_NEW_BT_GROUP_SPACE_WIDTHS: int = 5
20UNICODE_LOWER_LIMIT = 0
21UNICODE_UPPER_LIMIT = 0x10FFFF
22RTL_CHARACTER_PATTERN = re.compile("[" + "".join(f"{start}-{end}" for start, end in RTL_CHARACTER_RANGES) + "]")
23
24# The operators that can add text: the text-showing operators (ISO 32000-2:2020,
25# 9.4.3, Table 107) and Do, which can paint a form XObject with text (8.8.1, Table 86).
26# The bytes may also occur in strings, names or inline image data; that only
27# means that the content stream is parsed.
28_TEXT_OPERATORS = (b"Tj", b"TJ", b"Do", b"'", b'"')
29# Searching for their first bytes and comparing the next byte is faster than
30# searching for the alternation Tj|TJ|Do|'|" in content streams with many graphics.
31_TEXT_OPERATOR_START = re.compile(rb"[TD'\"]")
32
33
34class OrientationNotFoundError(Exception):
35    pass
36
37
38def set_custom_rtl(
39    _min: str | int | None = "",
40    _max: str | int | None = "",
41    specials: str | list[int] | None = None,
42) -> tuple[str, str, str]:
43    """
44    Change the Right-To-Left and special characters custom parameters.
45
46    Args:
47        _min: The new minimum value for the range of custom characters that
48            will be written right to left.
49            If set to ``None``, the value will not be changed.
50            If set to a valid integer, it will be converted to its corresponding character.
51            The default value is "", which sets no additional range to be converted.
52        _max: The new maximum value for the range of custom characters that will
53            be written right to left.
54            If set to ``None``, the value will not be changed.
55            If set to a valid integer, it will be converted to its corresponding character.
56            The default value is "", which sets no additional range to be converted.
57        specials: The new list of special characters to be inserted in the
58            current insertion order.
59            If set to ``None``, the current value will not be changed.
60            If set to a string, it will be converted to a list of ASCII codes.
61            The default value is an empty list.
62
63    Returns:
64        A tuple containing the new values for ``CUSTOM_RTL_MIN``,
65        ``CUSTOM_RTL_MAX``, and ``CUSTOM_RTL_SPECIAL_CHARS``.
66
67    """
68    global CUSTOM_RTL_MIN, CUSTOM_RTL_MAX, CUSTOM_RTL_SPECIAL_CHARS
69    if isinstance(_min, int):
70        CUSTOM_RTL_MIN = chr(_min) if UNICODE_LOWER_LIMIT <= _min <= UNICODE_UPPER_LIMIT else ""
71    elif isinstance(_min, str):
72        CUSTOM_RTL_MIN = _min
73    if isinstance(_max, int):
74        CUSTOM_RTL_MAX = chr(_max) if UNICODE_LOWER_LIMIT <= _max <= UNICODE_UPPER_LIMIT else ""
75    elif isinstance(_max, str):
76        CUSTOM_RTL_MAX = _max
77    if isinstance(specials, str):
78        CUSTOM_RTL_SPECIAL_CHARS = specials
79    elif isinstance(specials, list):
80        CUSTOM_RTL_SPECIAL_CHARS = "".join(
81            chr(char) for char in specials if UNICODE_LOWER_LIMIT <= char <= UNICODE_UPPER_LIMIT
82        )
83    return CUSTOM_RTL_MIN, CUSTOM_RTL_MAX, CUSTOM_RTL_SPECIAL_CHARS
84
85
86def mult(
87    m: list[float],
88    n: (
89        list[float] |
90        Mapping[int | Literal["is_text", "is_render"], float | bool]
91    ),
92) -> list[float]:
93    return [
94        m[0] * n[0] + m[1] * n[2],
95        m[0] * n[1] + m[1] * n[3],
96        m[2] * n[0] + m[3] * n[2],
97        m[2] * n[1] + m[3] * n[3],
98        m[4] * n[0] + m[5] * n[2] + n[4],
99        m[4] * n[1] + m[5] * n[3] + n[5],
100    ]
101
102
103def orient(m: list[float]) -> int:
104    if m[3] > 1e-6:
105        return 0
106    if m[3] < -1e-6:
107        return 180
108    if m[1] > 0:
109        return 90
110    return 270
111
112
113def crlf_space_check(  # noqa: PLR0913, PLR0917
114    text: str,
115    cmtm_prev: tuple[list[float], list[float]],
116    cmtm_matrix: tuple[list[float], list[float]],
117    memo_cmtm: tuple[list[float], list[float]],
118    font_resource: DictionaryObject | None,
119    orientations: tuple[int, ...],
120    output: str,
121    font_size: float,
122    visitor_text: Callable[[Any, Any, Any, Any, Any], None] | None,
123    str_widths: float,
124    spacewidth: float,
125    str_height: float,
126    line_span: tuple[int, float, float] | None,
127) -> tuple[str, str, list[float], list[float], tuple[int, float, float] | None]:
128    cm_prev = cmtm_prev[0]
129    tm_prev = cmtm_prev[1]
130    cm_matrix = cmtm_matrix[0]
131    tm_matrix = cmtm_matrix[1]
132    memo_cm = memo_cmtm[0]
133    memo_tm = memo_cmtm[1]
134
135    m_prev = mult(tm_prev, cm_prev)
136    m = mult(tm_matrix, cm_matrix)
137    orientation = orient(m)
138    delta_x = m[4] - m_prev[4]
139    delta_y = m[5] - m_prev[5]
140    # Table 108 of the 1.7 reference ("Text positioning operators")
141    # delta_x/delta_y are expressed in the coordinate system produced by
142    # text matrix x current transformation matrix, so the scaling factors
143    # they get compared against have to be taken from the same combined
144    # matrices instead of the text matrices alone.
145    scale_prev_x = math.sqrt(m_prev[0]**2 + m_prev[1]**2)
146    scale_prev_y = math.sqrt(m_prev[2]**2 + m_prev[3]**2)
147    scale_y = math.sqrt(m[2]**2 + m[3]**2)
148    cm_prev = m
149
150    if orientation not in orientations:
151        raise OrientationNotFoundError
152    if orientation in (0, 180):
153        moved_height: float = delta_y
154        moved_width: float = delta_x
155    elif orientation in (90, 270):
156        moved_height = delta_x
157        moved_width = delta_y
158    # Keep track of the baseline range of the current extracted line. A centered
159    # row label can fall between the two baselines of a wrapped cell (#4130), so
160    # comparing only with the immediately preceding fragment can incorrectly
161    # insert a line break. Use the established baseline range for that case.
162    axis_index = 5 if orientation in (0, 180) else 4
163    axis = m[axis_index]
164    if line_span is None or line_span[0] != axis_index:
165        distance = abs(moved_height)
166    else:
167        lower, upper = line_span[1], line_span[2]
168        distance = 0.0 if lower <= axis <= upper else min(abs(axis - lower), abs(axis - upper))
169    last = (output + text)[-1:]
170    threshold = 0.8 * min(str_height * scale_prev_y, font_size * scale_y)
171    if distance > threshold:
172        if last not in ("", "\n"):
173            output += text + "\n"
174            if visitor_text is not None:
175                visitor_text(
176                    text + "\n",
177                    memo_cm,
178                    memo_tm,
179                    font_resource,
180                    font_size,
181                )
182            text = ""
183        line_span = (axis_index, axis, axis)
184    else:
185        # A fragment kept on this line only by the line span moved to another
186        # baseline, so comparing its horizontal gap with the previous fragment
187        # would miss the word separation (#4254).
188        moved_to_other_baseline = abs(moved_height) > threshold
189        if (
190            (moved_to_other_baseline or moved_width >= (spacewidth + str_widths) * scale_prev_x)
191            and last not in ("", " ")
192        ):
193            text += " "
194        if last in ("", "\n") or line_span is None or line_span[0] != axis_index:
195            line_span = (axis_index, axis, axis)
196        else:
197            line_span = (axis_index, min(line_span[1], axis), max(line_span[2], axis))
198    tm_prev = tm_matrix.copy()
199    cm_prev = cm_matrix.copy()
200    return text, output, cm_prev, tm_prev, line_span
201
202
203def get_text_operands(
204    operands: list[str | TextStringObject],
205    cm_matrix: list[float],
206    tm_matrix: list[float],
207    font: Font,
208    orientations: tuple[int, ...]
209) -> tuple[str, bool, float]:
210    text: str = ""
211    is_str_operands = False
212    widths: float = 0.0
213    width_cache: dict[str, float] = {}
214    m = mult(tm_matrix, cm_matrix)
215    orientation = orient(m)
216    raw_characters: str = ""
217    if orientation in orientations and len(operands) > 0:
218        if isinstance(operands[0], str):
219            text = operands[0]
220            is_str_operands = True
221        else:
222            text = ""
223            tt: bytes = (
224                encode_pdfdocencoding(operands[0])
225                if isinstance(operands[0], str)
226                else operands[0]
227            )
228            if isinstance(font.encoding, str):  # Apply named encoding
229                try:
230                    text = tt.decode(font.encoding, "surrogatepass")
231                except UnicodeDecodeError:
232                    # Fallback for odd byte counts or unmapped 16-bit GIDs/CIDs
233                    text = tt.decode(font.encoding, "replace")
234            else:  # Apply dict encoding
235                text = "".join(font.encoding.get(x, chr(x)) for x in tt)
236                raw_characters = "".join(chr(byte) for byte in tt)
237        for char in (raw_characters or text):
238            if char == font.space_char:
239                widths += font.space_width
240            else:
241                if char not in width_cache:
242                    width_cache[char] = font.get_text_width(char)
243                widths += width_cache[char]
244
245    width_cache.clear()
246    return (text, is_str_operands, widths)
247
248
249def get_display_str(
250    text: str,
251    cm_matrix: list[float],
252    tm_matrix: list[float],
253    font_resource: DictionaryObject | None,
254    font: Font,
255    text_operands: str,
256    font_size: float,
257    rtl_dir: bool,
258    visitor_text: Callable[[Any, Any, Any, Any, Any], None] | None
259) -> tuple[str, bool, str]:
260    """
261    Add the characters of ``text_operands`` to ``text`` in display order.
262
263    Returns:
264        A tuple containing the current text run, its direction, and any text runs completed
265        because of a direction change. The completed runs precede the current run in display order.
266
267    """
268    # "\u0590 - \u08FF \uFB50 - \uFDFF"
269    characters = [font.character_map.get(raw_character, raw_character) for raw_character in text_operands]
270    if not rtl_dir and not (CUSTOM_RTL_MIN and CUSTOM_RTL_MAX):
271        # Without a right-to-left character, all characters are appended in order.
272        appended_text = "".join(characters)
273        if RTL_CHARACTER_PATTERN.search(appended_text) is None:
274            return text + appended_text, rtl_dir, ""
275
276    completed_text: list[str] = []
277    neutral_cache: dict[str, bool] = {}
278    rtl_cache: dict[str, bool] = {}
279    # Characters are only ever prepended in a right-to-left run and appended in a
280    # left-to-right run. Collect them and join once, as building the string one
281    # prepended character at a time takes quadratic time.
282    run_parts = [text]
283
284    def clear_character_caches() -> None:
285        neutral_cache.clear()
286        rtl_cache.clear()
287
288    def get_run_text() -> str:
289        return "".join(reversed(run_parts)) if rtl_dir else "".join(run_parts)
290
291    for x in characters:
292        # Test whether x is a sequence of bytes; ex: habibi.pdf
293        if len(x) == 1:
294            if x not in neutral_cache:
295                neutral_cache[x] = is_char_neutral(x, CUSTOM_RTL_SPECIAL_CHARS)
296            if neutral_cache[x]:
297                # Cases where the current inserting order is kept
298                run_parts.append(x)
299            else:
300                if x not in rtl_cache:
301                    rtl_cache[x] = is_char_rtl(x, CUSTOM_RTL_MIN, CUSTOM_RTL_MAX)
302                if rtl_cache[x] != rtl_dir:
303                    # The direction changes: right-to-left or left-to-right characters
304                    text = get_run_text()
305                    rtl_dir = rtl_cache[x]
306                    if visitor_text is not None:
307                        visitor_text(text, cm_matrix, tm_matrix, font_resource, font_size)
308                        clear_character_caches()
309                    completed_text.append(text)
310                    run_parts = []
311                run_parts.append(x)
312        else:
313            # Treat a sequence of bytes as a neutral character.
314            run_parts.append(x)
315    return get_run_text(), rtl_dir, "".join(completed_text)
316
317
318def _has_text_operator(data: bytes) -> bool:
319    """Return whether the bytes contain an operator that can add text."""
320    for match in _TEXT_OPERATOR_START.finditer(data):
321        start = match.start()
322        if data[start : start + 2] in _TEXT_OPERATORS or match[0] in _TEXT_OPERATORS:
323            return True
324    return False
325
326
327def _get_content_stream_for_text(
328    obj: DictionaryObject, pdf: Any, content_key: str | None, *, has_visitor: bool
329) -> ContentStream | None:
330    """
331    Return the content stream to extract text from, or None if it cannot give any text.
332
333    Content without text, e.g. a figure, gives no text either. It is only parsed if
334    a visitor has to see its operators.
335    """
336    try:
337        content = obj[content_key].get_object() if isinstance(content_key, str) else obj
338        if not isinstance(content, ContentStream):
339            content = ContentStream(content, pdf, "bytes")
340    except (AttributeError, KeyError):  # no content can be extracted (certainly empty page)
341        return None
342    if not has_visitor and not _has_text_operator(content.get_data()):
343        return None
344    return content