Source code

Download source archive ↓

src/rdflib/_networking.py

1from __future__ import annotations
2
3import string
4import sys
5from typing import Dict
6from urllib.error import HTTPError
7from urllib.parse import quote as urlquote
8from urllib.parse import urljoin, urlsplit
9from urllib.request import HTTPRedirectHandler, Request, urlopen
10from urllib.response import addinfourl
11
12
13def _make_redirect_request(request: Request, http_error: HTTPError) -> Request:
14    """Create a new request object for a redirected request.
15
16    The logic is based on [HTTPRedirectHandler](https://github.com/python/cpython/blob/b58bc8c2a9a316891a5ea1a0487aebfc86c2793a/Lib/urllib/request.py#L641-L751) from urllib.request.
17
18    Args:
19        request: The original request that resulted in the redirect.
20        http_error: The response to the original request that indicates a
21            redirect should occur and contains the new location.
22
23    Returns:
24        A new request object to the location indicated by the response.
25
26    Raises:
27        HTTPError: the supplied `http_error` if the redirect request
28            cannot be created.
29        ValueError: If the response code is None.
30        ValueError: If the response does not contain a `Location` header
31            or the `Location` header is not a string.
32        HTTPError: If the scheme of the new location is not `http`,
33            `https`, or `ftp`.
34        HTTPError: If there are too many redirects or a redirect loop.
35    """
36    new_url = http_error.headers.get("Location")
37    if new_url is None:
38        raise http_error
39    if not isinstance(new_url, str):
40        raise ValueError(f"Location header {new_url!r} is not a string")
41
42    new_url_parts = urlsplit(new_url)
43
44    # For security reasons don't allow redirection to anything other than http,
45    # https or ftp.
46    if new_url_parts.scheme not in ("http", "https", "ftp", ""):
47        raise HTTPError(
48            new_url,
49            http_error.code,
50            f"{http_error.reason} - Redirection to url {new_url!r} is not allowed",
51            http_error.headers,
52            http_error.fp,
53        )
54
55    # http.client.parse_headers() decodes as ISO-8859-1.  Recover the original
56    # bytes and percent-encode non-ASCII bytes, and any special characters such
57    # as the space.
58    new_url = urlquote(new_url, encoding="iso-8859-1", safe=string.punctuation)
59    new_url = urljoin(request.full_url, new_url)
60
61    # XXX Probably want to forget about the state of the current
62    # request, although that might interact poorly with other
63    # handlers that also use handler-specific request attributes
64    content_headers = ("content-length", "content-type")
65    newheaders = {
66        k: v for k, v in request.headers.items() if k.lower() not in content_headers
67    }
68    new_request = Request(
69        new_url,
70        headers=newheaders,
71        origin_req_host=request.origin_req_host,
72        unverifiable=True,
73    )
74
75    visited: Dict[str, int]
76    if hasattr(request, "redirect_dict"):
77        visited = request.redirect_dict
78        if (
79            visited.get(new_url, 0) >= HTTPRedirectHandler.max_repeats
80            or len(visited) >= HTTPRedirectHandler.max_redirections
81        ):
82            raise HTTPError(
83                request.full_url,
84                http_error.code,
85                HTTPRedirectHandler.inf_msg + http_error.reason,
86                http_error.headers,
87                http_error.fp,
88            )
89    else:
90        visited = {}
91        setattr(request, "redirect_dict", visited)
92
93    setattr(new_request, "redirect_dict", visited)
94    visited[new_url] = visited.get(new_url, 0) + 1
95    return new_request
96
97
98def _urlopen(request: Request) -> addinfourl:
99    """This is a shim for `urlopen` that handles HTTP redirects with status code
100    308 (Permanent Redirect).
101
102    This function should be removed once all supported versions of Python
103    handles the 308 HTTP status code.
104
105    Args:
106        request: The request to open.
107
108    Returns:
109        The response to the request.
110    """
111    try:
112        return urlopen(request)
113    except HTTPError as error:
114        if error.code == 308 and sys.version_info < (3, 11):
115            # HTTP response code 308 (Permanent Redirect) is not supported by python
116            # versions older than 3.11. See <https://bugs.python.org/issue40321> and
117            # <https://github.com/python/cpython/issues/84501> for more details.
118            # This custom error handling should be removed once all supported
119            # versions of Python handles 308.
120            new_request = _make_redirect_request(request, error)
121            return _urlopen(new_request)
122        else:
123            raise