Source code

Download source archive ↓

src/rdflib/plugins/shared/jsonld/util.py

1# https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/util.py
2from __future__ import annotations
3
4import json
5import pathlib
6from html.parser import HTMLParser
7from io import StringIO, TextIOBase, TextIOWrapper
8from typing import IO, TYPE_CHECKING, Any, Dict, List, Optional, TextIO, Tuple, Union
9
10if TYPE_CHECKING:
11    import json
12else:
13    try:
14        import json
15
16        assert json  # workaround for pyflakes issue #13
17    except ImportError:
18        import simplejson as json
19
20from posixpath import normpath, sep
21from typing import TYPE_CHECKING, cast
22from urllib.parse import urljoin, urlsplit, urlunsplit
23
24try:
25    import orjson
26
27    _HAS_ORJSON = True
28except ImportError:
29    orjson = None  # type: ignore[assignment, unused-ignore]
30    _HAS_ORJSON = False
31
32
33from rdflib.parser import (
34    BytesIOWrapper,
35    InputSource,
36    PythonInputSource,
37    StringInputSource,
38    URLInputSource,
39    create_input_source,
40)
41
42
43def source_to_json(
44    source: Optional[
45        Union[IO[bytes], TextIO, InputSource, str, bytes, pathlib.PurePath]
46    ],
47    fragment_id: Optional[str] = None,
48    extract_all_scripts: Optional[bool] = False,
49) -> Tuple[Union[Dict, List[Dict]], Any]:
50    """Extract JSON from a source document.
51
52    The source document can be JSON or HTML with embedded JSON script elements (type attribute = "application/ld+json").
53    To process as HTML `source.content_type` must be set to "text/html" or "application/xhtml+xml".
54
55    Args:
56        source: the input source document (JSON or HTML)
57        fragment_id: if source is an HTML document then extract only the script element with matching id attribute, defaults to None
58        extract_all_scripts: if source is an HTML document then extract all script elements (unless fragment_id is provided), defaults to False (extract only the first script element)
59
60    Returns:
61        Tuple with the extracted JSON document and value of the HTML base element
62    """
63
64    if isinstance(source, PythonInputSource):
65        return source.data, None
66
67    if isinstance(source, StringInputSource):
68        # A StringInputSource is assumed to be never a HTMLJSON doc
69        html_base: Any = None
70        # We can get the original string from the StringInputSource
71        # It's hidden in the BytesIOWrapper 'wrapped' attribute
72        b_stream = source.getByteStream()
73        original_string: Optional[str] = None
74        json_dict: Union[Dict, List[Dict]]
75        if isinstance(b_stream, BytesIOWrapper):
76            wrapped_inner = cast(Union[str, StringIO, TextIOBase], b_stream.wrapped)
77            if isinstance(wrapped_inner, str):
78                original_string = wrapped_inner
79            elif isinstance(wrapped_inner, StringIO):
80                original_string = wrapped_inner.getvalue()
81        if _HAS_ORJSON:
82            if original_string is not None:
83                json_dict = orjson.loads(original_string)
84            elif isinstance(b_stream, BytesIOWrapper):
85                # use the CharacterStream instead
86                c_stream = source.getCharacterStream()
87                json_dict = orjson.loads(c_stream.read())
88            else:
89                # orjson assumes its in utf-8 encoding so
90                # don't bother to check the source.getEncoding()
91                json_dict = orjson.loads(b_stream.read())
92        else:
93            if original_string is not None:
94                json_dict = json.loads(original_string)
95            else:
96                json_dict = json.load(source.getCharacterStream())
97        return json_dict, html_base
98
99    # TODO: conneg for JSON (fix support in rdflib's URLInputSource!)
100    source = create_input_source(source, format="json-ld")
101    try:
102        content_type = source.content_type
103    except (AttributeError, LookupError):
104        content_type = None
105
106    is_html = content_type is not None and content_type.lower() in (
107        "text/html",
108        "application/xhtml+xml",
109    )
110    if is_html:
111        html_docparser: Optional[HTMLJSONParser] = HTMLJSONParser(
112            fragment_id=fragment_id, extract_all_scripts=extract_all_scripts
113        )
114    else:
115        html_docparser = None
116    try:
117        b_stream = source.getByteStream()
118    except (AttributeError, LookupError):
119        b_stream = None
120    try:
121        c_stream = source.getCharacterStream()
122    except (AttributeError, LookupError):
123        c_stream = None
124    if b_stream is None and c_stream is None:
125        raise ValueError(
126            f"Source does not have a character stream or a byte stream and cannot be used {type(source)}"
127        )
128    try:
129        b_encoding: Optional[str] = None if b_stream is None else source.getEncoding()
130    except (AttributeError, LookupError):
131        b_encoding = None
132    underlying_string: Optional[str] = None
133    if b_stream is not None and isinstance(b_stream, BytesIOWrapper):
134        # Try to find an underlying wrapped Unicode string to use?
135        wrapped_inner = b_stream.wrapped
136        if isinstance(wrapped_inner, str):
137            underlying_string = wrapped_inner
138        elif isinstance(wrapped_inner, StringIO):
139            underlying_string = wrapped_inner.getvalue()
140    try:
141        if is_html and html_docparser is not None:
142            # Offload parsing to the HTMLJSONParser
143            if underlying_string is not None:
144                html_string: str = underlying_string
145            elif c_stream is not None:
146                html_string = c_stream.read()
147            else:
148                if TYPE_CHECKING:
149                    assert b_stream is not None
150                if b_encoding is None:
151                    b_encoding = "utf-8"
152                html_string = TextIOWrapper(b_stream, encoding=b_encoding).read()
153            html_docparser.feed(html_string)
154            json_dict, html_base = html_docparser.get_json(), html_docparser.get_base()
155        elif _HAS_ORJSON:
156            html_base = None
157            if underlying_string is not None:
158                json_dict = orjson.loads(underlying_string)
159            elif (
160                (b_stream is not None and isinstance(b_stream, BytesIOWrapper))
161                or b_stream is None
162            ) and c_stream is not None:
163                # use the CharacterStream instead
164                json_dict = orjson.loads(c_stream.read())
165            else:
166                if TYPE_CHECKING:
167                    assert b_stream is not None
168                # b_stream is not None
169                json_dict = orjson.loads(b_stream.read())
170        else:
171            html_base = None
172            if underlying_string is not None:
173                return json.loads(underlying_string)
174            if c_stream is not None:
175                use_stream = c_stream
176            else:
177                if TYPE_CHECKING:
178                    assert b_stream is not None
179                # b_stream is not None
180                if b_encoding is None:
181                    b_encoding = "utf-8"
182                use_stream = TextIOWrapper(b_stream, encoding=b_encoding)
183            json_dict = json.load(use_stream)
184        return json_dict, html_base
185    finally:
186        if b_stream is not None:
187            try:
188                b_stream.close()
189            except AttributeError:
190                pass
191        if c_stream is not None:
192            try:
193                c_stream.close()
194            except AttributeError:
195                pass
196
197
198VOCAB_DELIMS = ("#", "/", ":")
199
200
201def split_iri(iri: str) -> Tuple[str, Optional[str]]:
202    for delim in VOCAB_DELIMS:
203        at = iri.rfind(delim)
204        if at > -1:
205            return iri[: at + 1], iri[at + 1 :]
206    return iri, None
207
208
209def norm_url(base: str, url: str) -> str:
210    """
211    ```python
212    >>> norm_url('http://example.org/', '/one')
213    'http://example.org/one'
214    >>> norm_url('http://example.org/', '/one#')
215    'http://example.org/one#'
216    >>> norm_url('http://example.org/one', 'two')
217    'http://example.org/two'
218    >>> norm_url('http://example.org/one/', 'two')
219    'http://example.org/one/two'
220    >>> norm_url('http://example.org/', 'http://example.net/one')
221    'http://example.net/one'
222    >>> norm_url('http://example.org/', 'http://example.org//one')
223    'http://example.org//one'
224
225    ```
226    """
227    if "://" in url:
228        return url
229
230    # Fix for URNs
231    parsed_base = urlsplit(base)
232    parsed_url = urlsplit(url)
233    if parsed_url.scheme:
234        # Assume full URL
235        return url
236    if parsed_base.scheme in ("urn", "urn-x"):
237        # No scheme -> assume relative and join paths
238        base_path_parts = parsed_base.path.split("/", 1)
239        base_path = "/" + (base_path_parts[1] if len(base_path_parts) > 1 else "")
240        joined_path = urljoin(base_path, parsed_url.path)
241        fragment = f"#{parsed_url.fragment}" if parsed_url.fragment else ""
242        result = f"{parsed_base.scheme}:{base_path_parts[0]}{joined_path}{fragment}"
243    else:
244        parts = urlsplit(urljoin(base, url))
245        path = normpath(parts[2])
246        if sep != "/":
247            path = "/".join(path.split(sep))
248        if parts[2].endswith("/") and not path.endswith("/"):
249            path += "/"
250        result = urlunsplit(parts[0:2] + (path,) + parts[3:])
251    if url.endswith("#") and not result.endswith("#"):
252        result += "#"
253    return result
254
255
256# type error: Missing return statement
257def context_from_urlinputsource(source: URLInputSource) -> Optional[str]:  # type: ignore[return]
258    """
259    Please note that JSON-LD documents served with the `application/ld+json` media type
260    MUST have all context information, including references to external contexts,
261    within the body of the document. Contexts linked via a
262    http://www.w3.org/ns/json-ld#context HTTP Link Header MUST be
263    ignored for such documents.
264    """
265    if source.content_type != "application/ld+json":
266        try:
267            # source.links is the new way of getting Link headers from URLInputSource
268            links = source.links
269        except AttributeError:
270            # type error: Return value expected
271            return  # type: ignore[return-value]
272        for link in links:
273            if ' rel="http://www.w3.org/ns/json-ld#context"' in link:
274                i, j = link.index("<"), link.index(">")
275                if i > -1 and j > -1:
276                    # type error: Value of type variable "AnyStr" of "urljoin" cannot be "Optional[str]"
277                    return urljoin(source.url, link[i + 1 : j])  # type: ignore[type-var]
278
279
280__all__ = [
281    "json",
282    "source_to_json",
283    "split_iri",
284    "norm_url",
285    "context_from_urlinputsource",
286    "orjson",
287    "_HAS_ORJSON",
288]
289
290
291class HTMLJSONParser(HTMLParser):
292    def __init__(
293        self,
294        fragment_id: Optional[str] = None,
295        extract_all_scripts: Optional[bool] = False,
296    ):
297        super().__init__()
298        self.fragment_id = fragment_id
299        self.json: List[Dict] = []
300        self.contains_json = False
301        self.fragment_id_does_not_match = False
302        self.base = None
303        self.extract_all_scripts = extract_all_scripts
304        self.script_count = 0
305
306    def handle_starttag(self, tag, attrs):
307        self.contains_json = False
308        self.fragment_id_does_not_match = False
309
310        # Only set self. contains_json to True if the
311        # type is 'application/ld+json'
312        if tag == "script":
313            for attr, value in attrs:
314                if attr == "type" and value == "application/ld+json":
315                    self.contains_json = True
316                elif attr == "id" and self.fragment_id and value != self.fragment_id:
317                    self.fragment_id_does_not_match = True
318
319        elif tag == "base":
320            for attr, value in attrs:
321                if attr == "href":
322                    self.base = value
323
324    def handle_data(self, data):
325        # Only do something when we know the context is a
326        # script element containing application/ld+json
327
328        if self.contains_json is True and self.fragment_id_does_not_match is False:
329
330            if not self.extract_all_scripts and self.script_count > 0:
331                return
332
333            if data.strip() == "":
334                # skip empty data elements
335                return
336
337            # Try to parse the json
338            if _HAS_ORJSON:
339                # orjson can load a unicode string
340                # if that's the only thing we have,
341                # its not worth encoding it to bytes
342                parsed = orjson.loads(data)
343            else:
344                parsed = json.loads(data)
345
346            # Add to the result document
347            if isinstance(parsed, list):
348                self.json.extend(parsed)
349            else:
350                self.json.append(parsed)
351
352            self.script_count += 1
353
354    def get_json(self) -> List[Dict]:
355        return self.json
356
357    def get_base(self):
358        return self.base