src/rdflib/plugins/shared/jsonld/util.py
1# https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/util.py
2from __future__ import annotations
3
4import json
5import pathlib
6from html.parser import HTMLParser
7from io import StringIO, TextIOBase, TextIOWrapper
8from typing import IO, TYPE_CHECKING, Any, Dict, List, Optional, TextIO, Tuple, Union
9
10if TYPE_CHECKING:
11 import json
12else:
13 try:
14 import json
15
16 assert json # workaround for pyflakes issue #13
17 except ImportError:
18 import simplejson as json
19
20from posixpath import normpath, sep
21from typing import TYPE_CHECKING, cast
22from urllib.parse import urljoin, urlsplit, urlunsplit
23
24try:
25 import orjson
26
27 _HAS_ORJSON = True
28except ImportError:
29 orjson = None # type: ignore[assignment, unused-ignore]
30 _HAS_ORJSON = False
31
32
33from rdflib.parser import (
34 BytesIOWrapper,
35 InputSource,
36 PythonInputSource,
37 StringInputSource,
38 URLInputSource,
39 create_input_source,
40)
41
42
43def source_to_json(
44 source: Optional[
45 Union[IO[bytes], TextIO, InputSource, str, bytes, pathlib.PurePath]
46 ],
47 fragment_id: Optional[str] = None,
48 extract_all_scripts: Optional[bool] = False,
49) -> Tuple[Union[Dict, List[Dict]], Any]:
50 """Extract JSON from a source document.
51
52 The source document can be JSON or HTML with embedded JSON script elements (type attribute = "application/ld+json").
53 To process as HTML `source.content_type` must be set to "text/html" or "application/xhtml+xml".
54
55 Args:
56 source: the input source document (JSON or HTML)
57 fragment_id: if source is an HTML document then extract only the script element with matching id attribute, defaults to None
58 extract_all_scripts: if source is an HTML document then extract all script elements (unless fragment_id is provided), defaults to False (extract only the first script element)
59
60 Returns:
61 Tuple with the extracted JSON document and value of the HTML base element
62 """
63
64 if isinstance(source, PythonInputSource):
65 return source.data, None
66
67 if isinstance(source, StringInputSource):
68 # A StringInputSource is assumed to be never a HTMLJSON doc
69 html_base: Any = None
70 # We can get the original string from the StringInputSource
71 # It's hidden in the BytesIOWrapper 'wrapped' attribute
72 b_stream = source.getByteStream()
73 original_string: Optional[str] = None
74 json_dict: Union[Dict, List[Dict]]
75 if isinstance(b_stream, BytesIOWrapper):
76 wrapped_inner = cast(Union[str, StringIO, TextIOBase], b_stream.wrapped)
77 if isinstance(wrapped_inner, str):
78 original_string = wrapped_inner
79 elif isinstance(wrapped_inner, StringIO):
80 original_string = wrapped_inner.getvalue()
81 if _HAS_ORJSON:
82 if original_string is not None:
83 json_dict = orjson.loads(original_string)
84 elif isinstance(b_stream, BytesIOWrapper):
85 # use the CharacterStream instead
86 c_stream = source.getCharacterStream()
87 json_dict = orjson.loads(c_stream.read())
88 else:
89 # orjson assumes its in utf-8 encoding so
90 # don't bother to check the source.getEncoding()
91 json_dict = orjson.loads(b_stream.read())
92 else:
93 if original_string is not None:
94 json_dict = json.loads(original_string)
95 else:
96 json_dict = json.load(source.getCharacterStream())
97 return json_dict, html_base
98
99 # TODO: conneg for JSON (fix support in rdflib's URLInputSource!)
100 source = create_input_source(source, format="json-ld")
101 try:
102 content_type = source.content_type
103 except (AttributeError, LookupError):
104 content_type = None
105
106 is_html = content_type is not None and content_type.lower() in (
107 "text/html",
108 "application/xhtml+xml",
109 )
110 if is_html:
111 html_docparser: Optional[HTMLJSONParser] = HTMLJSONParser(
112 fragment_id=fragment_id, extract_all_scripts=extract_all_scripts
113 )
114 else:
115 html_docparser = None
116 try:
117 b_stream = source.getByteStream()
118 except (AttributeError, LookupError):
119 b_stream = None
120 try:
121 c_stream = source.getCharacterStream()
122 except (AttributeError, LookupError):
123 c_stream = None
124 if b_stream is None and c_stream is None:
125 raise ValueError(
126 f"Source does not have a character stream or a byte stream and cannot be used {type(source)}"
127 )
128 try:
129 b_encoding: Optional[str] = None if b_stream is None else source.getEncoding()
130 except (AttributeError, LookupError):
131 b_encoding = None
132 underlying_string: Optional[str] = None
133 if b_stream is not None and isinstance(b_stream, BytesIOWrapper):
134 # Try to find an underlying wrapped Unicode string to use?
135 wrapped_inner = b_stream.wrapped
136 if isinstance(wrapped_inner, str):
137 underlying_string = wrapped_inner
138 elif isinstance(wrapped_inner, StringIO):
139 underlying_string = wrapped_inner.getvalue()
140 try:
141 if is_html and html_docparser is not None:
142 # Offload parsing to the HTMLJSONParser
143 if underlying_string is not None:
144 html_string: str = underlying_string
145 elif c_stream is not None:
146 html_string = c_stream.read()
147 else:
148 if TYPE_CHECKING:
149 assert b_stream is not None
150 if b_encoding is None:
151 b_encoding = "utf-8"
152 html_string = TextIOWrapper(b_stream, encoding=b_encoding).read()
153 html_docparser.feed(html_string)
154 json_dict, html_base = html_docparser.get_json(), html_docparser.get_base()
155 elif _HAS_ORJSON:
156 html_base = None
157 if underlying_string is not None:
158 json_dict = orjson.loads(underlying_string)
159 elif (
160 (b_stream is not None and isinstance(b_stream, BytesIOWrapper))
161 or b_stream is None
162 ) and c_stream is not None:
163 # use the CharacterStream instead
164 json_dict = orjson.loads(c_stream.read())
165 else:
166 if TYPE_CHECKING:
167 assert b_stream is not None
168 # b_stream is not None
169 json_dict = orjson.loads(b_stream.read())
170 else:
171 html_base = None
172 if underlying_string is not None:
173 return json.loads(underlying_string)
174 if c_stream is not None:
175 use_stream = c_stream
176 else:
177 if TYPE_CHECKING:
178 assert b_stream is not None
179 # b_stream is not None
180 if b_encoding is None:
181 b_encoding = "utf-8"
182 use_stream = TextIOWrapper(b_stream, encoding=b_encoding)
183 json_dict = json.load(use_stream)
184 return json_dict, html_base
185 finally:
186 if b_stream is not None:
187 try:
188 b_stream.close()
189 except AttributeError:
190 pass
191 if c_stream is not None:
192 try:
193 c_stream.close()
194 except AttributeError:
195 pass
196
197
198VOCAB_DELIMS = ("#", "/", ":")
199
200
201def split_iri(iri: str) -> Tuple[str, Optional[str]]:
202 for delim in VOCAB_DELIMS:
203 at = iri.rfind(delim)
204 if at > -1:
205 return iri[: at + 1], iri[at + 1 :]
206 return iri, None
207
208
209def norm_url(base: str, url: str) -> str:
210 """
211 ```python
212 >>> norm_url('http://example.org/', '/one')
213 'http://example.org/one'
214 >>> norm_url('http://example.org/', '/one#')
215 'http://example.org/one#'
216 >>> norm_url('http://example.org/one', 'two')
217 'http://example.org/two'
218 >>> norm_url('http://example.org/one/', 'two')
219 'http://example.org/one/two'
220 >>> norm_url('http://example.org/', 'http://example.net/one')
221 'http://example.net/one'
222 >>> norm_url('http://example.org/', 'http://example.org//one')
223 'http://example.org//one'
224
225 ```
226 """
227 if "://" in url:
228 return url
229
230 # Fix for URNs
231 parsed_base = urlsplit(base)
232 parsed_url = urlsplit(url)
233 if parsed_url.scheme:
234 # Assume full URL
235 return url
236 if parsed_base.scheme in ("urn", "urn-x"):
237 # No scheme -> assume relative and join paths
238 base_path_parts = parsed_base.path.split("/", 1)
239 base_path = "/" + (base_path_parts[1] if len(base_path_parts) > 1 else "")
240 joined_path = urljoin(base_path, parsed_url.path)
241 fragment = f"#{parsed_url.fragment}" if parsed_url.fragment else ""
242 result = f"{parsed_base.scheme}:{base_path_parts[0]}{joined_path}{fragment}"
243 else:
244 parts = urlsplit(urljoin(base, url))
245 path = normpath(parts[2])
246 if sep != "/":
247 path = "/".join(path.split(sep))
248 if parts[2].endswith("/") and not path.endswith("/"):
249 path += "/"
250 result = urlunsplit(parts[0:2] + (path,) + parts[3:])
251 if url.endswith("#") and not result.endswith("#"):
252 result += "#"
253 return result
254
255
256# type error: Missing return statement
257def context_from_urlinputsource(source: URLInputSource) -> Optional[str]: # type: ignore[return]
258 """
259 Please note that JSON-LD documents served with the `application/ld+json` media type
260 MUST have all context information, including references to external contexts,
261 within the body of the document. Contexts linked via a
262 http://www.w3.org/ns/json-ld#context HTTP Link Header MUST be
263 ignored for such documents.
264 """
265 if source.content_type != "application/ld+json":
266 try:
267 # source.links is the new way of getting Link headers from URLInputSource
268 links = source.links
269 except AttributeError:
270 # type error: Return value expected
271 return # type: ignore[return-value]
272 for link in links:
273 if ' rel="http://www.w3.org/ns/json-ld#context"' in link:
274 i, j = link.index("<"), link.index(">")
275 if i > -1 and j > -1:
276 # type error: Value of type variable "AnyStr" of "urljoin" cannot be "Optional[str]"
277 return urljoin(source.url, link[i + 1 : j]) # type: ignore[type-var]
278
279
280__all__ = [
281 "json",
282 "source_to_json",
283 "split_iri",
284 "norm_url",
285 "context_from_urlinputsource",
286 "orjson",
287 "_HAS_ORJSON",
288]
289
290
291class HTMLJSONParser(HTMLParser):
292 def __init__(
293 self,
294 fragment_id: Optional[str] = None,
295 extract_all_scripts: Optional[bool] = False,
296 ):
297 super().__init__()
298 self.fragment_id = fragment_id
299 self.json: List[Dict] = []
300 self.contains_json = False
301 self.fragment_id_does_not_match = False
302 self.base = None
303 self.extract_all_scripts = extract_all_scripts
304 self.script_count = 0
305
306 def handle_starttag(self, tag, attrs):
307 self.contains_json = False
308 self.fragment_id_does_not_match = False
309
310 # Only set self. contains_json to True if the
311 # type is 'application/ld+json'
312 if tag == "script":
313 for attr, value in attrs:
314 if attr == "type" and value == "application/ld+json":
315 self.contains_json = True
316 elif attr == "id" and self.fragment_id and value != self.fragment_id:
317 self.fragment_id_does_not_match = True
318
319 elif tag == "base":
320 for attr, value in attrs:
321 if attr == "href":
322 self.base = value
323
324 def handle_data(self, data):
325 # Only do something when we know the context is a
326 # script element containing application/ld+json
327
328 if self.contains_json is True and self.fragment_id_does_not_match is False:
329
330 if not self.extract_all_scripts and self.script_count > 0:
331 return
332
333 if data.strip() == "":
334 # skip empty data elements
335 return
336
337 # Try to parse the json
338 if _HAS_ORJSON:
339 # orjson can load a unicode string
340 # if that's the only thing we have,
341 # its not worth encoding it to bytes
342 parsed = orjson.loads(data)
343 else:
344 parsed = json.loads(data)
345
346 # Add to the result document
347 if isinstance(parsed, list):
348 self.json.extend(parsed)
349 else:
350 self.json.append(parsed)
351
352 self.script_count += 1
353
354 def get_json(self) -> List[Dict]:
355 return self.json
356
357 def get_base(self):
358 return self.base