src/pypdf/_text_extraction/__init__.py
1"""
2Code related to text extraction.
3
4Some parts are still in _page.py. In doubt, they will stay there.
5"""
6
7import math
8import re
9from collections.abc import Callable, Mapping
10from typing import Any, Literal
11
12from .._utils import RTL_CHARACTER_RANGES, is_char_neutral, is_char_rtl
13from ..generic import ContentStream, DictionaryObject, TextStringObject, encode_pdfdocencoding
14from ..generic._font import Font
15
16CUSTOM_RTL_MIN: str = ""
17CUSTOM_RTL_MAX: str = ""
18CUSTOM_RTL_SPECIAL_CHARS: str = ""
19LAYOUT_NEW_BT_GROUP_SPACE_WIDTHS: int = 5
20UNICODE_LOWER_LIMIT = 0
21UNICODE_UPPER_LIMIT = 0x10FFFF
22RTL_CHARACTER_PATTERN = re.compile("[" + "".join(f"{start}-{end}" for start, end in RTL_CHARACTER_RANGES) + "]")
23
24# The operators that can add text: the text-showing operators (ISO 32000-2:2020,
25# 9.4.3, Table 107) and Do, which can paint a form XObject with text (8.8.1, Table 86).
26# The bytes may also occur in strings, names or inline image data; that only
27# means that the content stream is parsed.
28_TEXT_OPERATORS = (b"Tj", b"TJ", b"Do", b"'", b'"')
29# Searching for their first bytes and comparing the next byte is faster than
30# searching for the alternation Tj|TJ|Do|'|" in content streams with many graphics.
31_TEXT_OPERATOR_START = re.compile(rb"[TD'\"]")
32
33
34class OrientationNotFoundError(Exception):
35 pass
36
37
38def set_custom_rtl(
39 _min: str | int | None = "",
40 _max: str | int | None = "",
41 specials: str | list[int] | None = None,
42) -> tuple[str, str, str]:
43 """
44 Change the Right-To-Left and special characters custom parameters.
45
46 Args:
47 _min: The new minimum value for the range of custom characters that
48 will be written right to left.
49 If set to ``None``, the value will not be changed.
50 If set to a valid integer, it will be converted to its corresponding character.
51 The default value is "", which sets no additional range to be converted.
52 _max: The new maximum value for the range of custom characters that will
53 be written right to left.
54 If set to ``None``, the value will not be changed.
55 If set to a valid integer, it will be converted to its corresponding character.
56 The default value is "", which sets no additional range to be converted.
57 specials: The new list of special characters to be inserted in the
58 current insertion order.
59 If set to ``None``, the current value will not be changed.
60 If set to a string, it will be converted to a list of ASCII codes.
61 The default value is an empty list.
62
63 Returns:
64 A tuple containing the new values for ``CUSTOM_RTL_MIN``,
65 ``CUSTOM_RTL_MAX``, and ``CUSTOM_RTL_SPECIAL_CHARS``.
66
67 """
68 global CUSTOM_RTL_MIN, CUSTOM_RTL_MAX, CUSTOM_RTL_SPECIAL_CHARS
69 if isinstance(_min, int):
70 CUSTOM_RTL_MIN = chr(_min) if UNICODE_LOWER_LIMIT <= _min <= UNICODE_UPPER_LIMIT else ""
71 elif isinstance(_min, str):
72 CUSTOM_RTL_MIN = _min
73 if isinstance(_max, int):
74 CUSTOM_RTL_MAX = chr(_max) if UNICODE_LOWER_LIMIT <= _max <= UNICODE_UPPER_LIMIT else ""
75 elif isinstance(_max, str):
76 CUSTOM_RTL_MAX = _max
77 if isinstance(specials, str):
78 CUSTOM_RTL_SPECIAL_CHARS = specials
79 elif isinstance(specials, list):
80 CUSTOM_RTL_SPECIAL_CHARS = "".join(
81 chr(char) for char in specials if UNICODE_LOWER_LIMIT <= char <= UNICODE_UPPER_LIMIT
82 )
83 return CUSTOM_RTL_MIN, CUSTOM_RTL_MAX, CUSTOM_RTL_SPECIAL_CHARS
84
85
86def mult(
87 m: list[float],
88 n: (
89 list[float] |
90 Mapping[int | Literal["is_text", "is_render"], float | bool]
91 ),
92) -> list[float]:
93 return [
94 m[0] * n[0] + m[1] * n[2],
95 m[0] * n[1] + m[1] * n[3],
96 m[2] * n[0] + m[3] * n[2],
97 m[2] * n[1] + m[3] * n[3],
98 m[4] * n[0] + m[5] * n[2] + n[4],
99 m[4] * n[1] + m[5] * n[3] + n[5],
100 ]
101
102
103def orient(m: list[float]) -> int:
104 if m[3] > 1e-6:
105 return 0
106 if m[3] < -1e-6:
107 return 180
108 if m[1] > 0:
109 return 90
110 return 270
111
112
113def crlf_space_check( # noqa: PLR0913, PLR0917
114 text: str,
115 cmtm_prev: tuple[list[float], list[float]],
116 cmtm_matrix: tuple[list[float], list[float]],
117 memo_cmtm: tuple[list[float], list[float]],
118 font_resource: DictionaryObject | None,
119 orientations: tuple[int, ...],
120 output: str,
121 font_size: float,
122 visitor_text: Callable[[Any, Any, Any, Any, Any], None] | None,
123 str_widths: float,
124 spacewidth: float,
125 str_height: float,
126 line_span: tuple[int, float, float] | None,
127) -> tuple[str, str, list[float], list[float], tuple[int, float, float] | None]:
128 cm_prev = cmtm_prev[0]
129 tm_prev = cmtm_prev[1]
130 cm_matrix = cmtm_matrix[0]
131 tm_matrix = cmtm_matrix[1]
132 memo_cm = memo_cmtm[0]
133 memo_tm = memo_cmtm[1]
134
135 m_prev = mult(tm_prev, cm_prev)
136 m = mult(tm_matrix, cm_matrix)
137 orientation = orient(m)
138 delta_x = m[4] - m_prev[4]
139 delta_y = m[5] - m_prev[5]
140 # Table 108 of the 1.7 reference ("Text positioning operators")
141 # delta_x/delta_y are expressed in the coordinate system produced by
142 # text matrix x current transformation matrix, so the scaling factors
143 # they get compared against have to be taken from the same combined
144 # matrices instead of the text matrices alone.
145 scale_prev_x = math.sqrt(m_prev[0]**2 + m_prev[1]**2)
146 scale_prev_y = math.sqrt(m_prev[2]**2 + m_prev[3]**2)
147 scale_y = math.sqrt(m[2]**2 + m[3]**2)
148 cm_prev = m
149
150 if orientation not in orientations:
151 raise OrientationNotFoundError
152 if orientation in (0, 180):
153 moved_height: float = delta_y
154 moved_width: float = delta_x
155 elif orientation in (90, 270):
156 moved_height = delta_x
157 moved_width = delta_y
158 # Keep track of the baseline range of the current extracted line. A centered
159 # row label can fall between the two baselines of a wrapped cell (#4130), so
160 # comparing only with the immediately preceding fragment can incorrectly
161 # insert a line break. Use the established baseline range for that case.
162 axis_index = 5 if orientation in (0, 180) else 4
163 axis = m[axis_index]
164 if line_span is None or line_span[0] != axis_index:
165 distance = abs(moved_height)
166 else:
167 lower, upper = line_span[1], line_span[2]
168 distance = 0.0 if lower <= axis <= upper else min(abs(axis - lower), abs(axis - upper))
169 last = (output + text)[-1:]
170 threshold = 0.8 * min(str_height * scale_prev_y, font_size * scale_y)
171 if distance > threshold:
172 if last not in ("", "\n"):
173 output += text + "\n"
174 if visitor_text is not None:
175 visitor_text(
176 text + "\n",
177 memo_cm,
178 memo_tm,
179 font_resource,
180 font_size,
181 )
182 text = ""
183 line_span = (axis_index, axis, axis)
184 else:
185 # A fragment kept on this line only by the line span moved to another
186 # baseline, so comparing its horizontal gap with the previous fragment
187 # would miss the word separation (#4254).
188 moved_to_other_baseline = abs(moved_height) > threshold
189 if (
190 (moved_to_other_baseline or moved_width >= (spacewidth + str_widths) * scale_prev_x)
191 and last not in ("", " ")
192 ):
193 text += " "
194 if last in ("", "\n") or line_span is None or line_span[0] != axis_index:
195 line_span = (axis_index, axis, axis)
196 else:
197 line_span = (axis_index, min(line_span[1], axis), max(line_span[2], axis))
198 tm_prev = tm_matrix.copy()
199 cm_prev = cm_matrix.copy()
200 return text, output, cm_prev, tm_prev, line_span
201
202
203def get_text_operands(
204 operands: list[str | TextStringObject],
205 cm_matrix: list[float],
206 tm_matrix: list[float],
207 font: Font,
208 orientations: tuple[int, ...]
209) -> tuple[str, bool, float]:
210 text: str = ""
211 is_str_operands = False
212 widths: float = 0.0
213 width_cache: dict[str, float] = {}
214 m = mult(tm_matrix, cm_matrix)
215 orientation = orient(m)
216 raw_characters: str = ""
217 if orientation in orientations and len(operands) > 0:
218 if isinstance(operands[0], str):
219 text = operands[0]
220 is_str_operands = True
221 else:
222 text = ""
223 tt: bytes = (
224 encode_pdfdocencoding(operands[0])
225 if isinstance(operands[0], str)
226 else operands[0]
227 )
228 if isinstance(font.encoding, str): # Apply named encoding
229 try:
230 text = tt.decode(font.encoding, "surrogatepass")
231 except UnicodeDecodeError:
232 # Fallback for odd byte counts or unmapped 16-bit GIDs/CIDs
233 text = tt.decode(font.encoding, "replace")
234 else: # Apply dict encoding
235 text = "".join(font.encoding.get(x, chr(x)) for x in tt)
236 raw_characters = "".join(chr(byte) for byte in tt)
237 for char in (raw_characters or text):
238 if char == font.space_char:
239 widths += font.space_width
240 else:
241 if char not in width_cache:
242 width_cache[char] = font.get_text_width(char)
243 widths += width_cache[char]
244
245 width_cache.clear()
246 return (text, is_str_operands, widths)
247
248
249def get_display_str(
250 text: str,
251 cm_matrix: list[float],
252 tm_matrix: list[float],
253 font_resource: DictionaryObject | None,
254 font: Font,
255 text_operands: str,
256 font_size: float,
257 rtl_dir: bool,
258 visitor_text: Callable[[Any, Any, Any, Any, Any], None] | None
259) -> tuple[str, bool, str]:
260 """
261 Add the characters of ``text_operands`` to ``text`` in display order.
262
263 Returns:
264 A tuple containing the current text run, its direction, and any text runs completed
265 because of a direction change. The completed runs precede the current run in display order.
266
267 """
268 # "\u0590 - \u08FF \uFB50 - \uFDFF"
269 characters = [font.character_map.get(raw_character, raw_character) for raw_character in text_operands]
270 if not rtl_dir and not (CUSTOM_RTL_MIN and CUSTOM_RTL_MAX):
271 # Without a right-to-left character, all characters are appended in order.
272 appended_text = "".join(characters)
273 if RTL_CHARACTER_PATTERN.search(appended_text) is None:
274 return text + appended_text, rtl_dir, ""
275
276 completed_text: list[str] = []
277 neutral_cache: dict[str, bool] = {}
278 rtl_cache: dict[str, bool] = {}
279 # Characters are only ever prepended in a right-to-left run and appended in a
280 # left-to-right run. Collect them and join once, as building the string one
281 # prepended character at a time takes quadratic time.
282 run_parts = [text]
283
284 def clear_character_caches() -> None:
285 neutral_cache.clear()
286 rtl_cache.clear()
287
288 def get_run_text() -> str:
289 return "".join(reversed(run_parts)) if rtl_dir else "".join(run_parts)
290
291 for x in characters:
292 # Test whether x is a sequence of bytes; ex: habibi.pdf
293 if len(x) == 1:
294 if x not in neutral_cache:
295 neutral_cache[x] = is_char_neutral(x, CUSTOM_RTL_SPECIAL_CHARS)
296 if neutral_cache[x]:
297 # Cases where the current inserting order is kept
298 run_parts.append(x)
299 else:
300 if x not in rtl_cache:
301 rtl_cache[x] = is_char_rtl(x, CUSTOM_RTL_MIN, CUSTOM_RTL_MAX)
302 if rtl_cache[x] != rtl_dir:
303 # The direction changes: right-to-left or left-to-right characters
304 text = get_run_text()
305 rtl_dir = rtl_cache[x]
306 if visitor_text is not None:
307 visitor_text(text, cm_matrix, tm_matrix, font_resource, font_size)
308 clear_character_caches()
309 completed_text.append(text)
310 run_parts = []
311 run_parts.append(x)
312 else:
313 # Treat a sequence of bytes as a neutral character.
314 run_parts.append(x)
315 return get_run_text(), rtl_dir, "".join(completed_text)
316
317
318def _has_text_operator(data: bytes) -> bool:
319 """Return whether the bytes contain an operator that can add text."""
320 for match in _TEXT_OPERATOR_START.finditer(data):
321 start = match.start()
322 if data[start : start + 2] in _TEXT_OPERATORS or match[0] in _TEXT_OPERATORS:
323 return True
324 return False
325
326
327def _get_content_stream_for_text(
328 obj: DictionaryObject, pdf: Any, content_key: str | None, *, has_visitor: bool
329) -> ContentStream | None:
330 """
331 Return the content stream to extract text from, or None if it cannot give any text.
332
333 Content without text, e.g. a figure, gives no text either. It is only parsed if
334 a visitor has to see its operators.
335 """
336 try:
337 content = obj[content_key].get_object() if isinstance(content_key, str) else obj
338 if not isinstance(content, ContentStream):
339 content = ContentStream(content, pdf, "bytes")
340 except (AttributeError, KeyError): # no content can be extracted (certainly empty page)
341 return None
342 if not has_visitor and not _has_text_operator(content.get_data()):
343 return None
344 return content