src/rdflib/plugins/parsers/jsonld.py
1"""
2This parser will interpret a JSON-LD document as an RDF Graph. See http://json-ld.org/
3
4Example:
5 ```python
6 >>> from rdflib import Graph, URIRef, Literal
7 >>> test_json = '''
8 ... {
9 ... "@context": {
10 ... "dc": "http://purl.org/dc/terms/",
11 ... "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
12 ... "rdfs": "http://www.w3.org/2000/01/rdf-schema#"
13 ... },
14 ... "@id": "http://example.org/about",
15 ... "dc:title": {
16 ... "@language": "en",
17 ... "@value": "Someone's Homepage"
18 ... }
19 ... }
20 ... '''
21 >>> g = Graph().parse(data=test_json, format='json-ld')
22 >>> list(g) == [(URIRef('http://example.org/about'),
23 ... URIRef('http://purl.org/dc/terms/title'),
24 ... Literal("Someone's Homepage", lang='en'))]
25 True
26
27 ```
28"""
29
30# From: https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/parser.py
31
32# NOTE: This code reads the entire JSON object into memory before parsing, but
33# we should consider streaming the input to deal with arbitrarily large graphs.
34from __future__ import annotations
35
36import secrets
37import warnings
38from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
39
40import rdflib.parser
41from rdflib.graph import ConjunctiveGraph, Graph
42from rdflib.namespace import RDF, XSD
43from rdflib.parser import InputSource, URLInputSource
44from rdflib.term import BNode, IdentifiedNode, Literal, Node, URIRef
45
46from ..shared.jsonld.context import UNDEF, Context, Term
47from ..shared.jsonld.keys import (
48 CONTEXT,
49 GRAPH,
50 ID,
51 INCLUDED,
52 INDEX,
53 JSON,
54 LANG,
55 LIST,
56 NEST,
57 NONE,
58 REV,
59 SET,
60 TYPE,
61 VALUE,
62 VOCAB,
63)
64from ..shared.jsonld.util import (
65 _HAS_ORJSON,
66 VOCAB_DELIMS,
67 context_from_urlinputsource,
68 json,
69 orjson,
70 source_to_json,
71)
72
73__all__ = ["JsonLDParser", "to_rdf"]
74
75TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB) # type: ignore[call-arg]
76
77ALLOW_LISTS_OF_LISTS = True # NOTE: Not allowed in JSON-LD 1.0
78
79
80class JsonLDParser(rdflib.parser.Parser):
81 def __init__(self):
82 super(JsonLDParser, self).__init__()
83
84 def parse(
85 self,
86 source: InputSource,
87 sink: Graph,
88 version: float = 1.1,
89 skolemize: bool = False,
90 encoding: Optional[str] = "utf-8",
91 base: Optional[str] = None,
92 context: Optional[
93 Union[
94 List[Union[Dict[str, Any], str, None]],
95 Dict[str, Any],
96 str,
97 ]
98 ] = None,
99 generalized_rdf: Optional[bool] = False,
100 extract_all_scripts: Optional[bool] = False,
101 **kwargs: Any,
102 ) -> None:
103 """Parse JSON-LD from a source document.
104
105 The source document can be JSON or HTML with embedded JSON script
106 elements (type attribute = `application/ld+json`). To process as HTML
107 `source.content_type` must be set to "text/html" or
108 `application/xhtml+xml.
109
110 Args:
111 source: InputSource with JSON-formatted data (JSON or HTML)
112 sink: Graph to receive the parsed triples
113 version: parse as JSON-LD version, defaults to 1.1
114 skolemize: whether to skolemize blank nodes, defaults to False
115 encoding: character encoding of the JSON (should be "utf-8"
116 base: JSON-LD [Base IRI](https://www.w3.org/TR/json-ld/#base-iri), defaults to None
117 context: JSON-LD [Context](https://www.w3.org/TR/json-ld/#the-context), defaults to None
118 generalized_rdf: parse as [Generalized RDF](https://www.w3.org/TR/json-ld/#relationship-to-rdf), defaults to False
119 extract_all_scripts: if source is an HTML document then extract
120 script element). This is ignored if `source.system_id` contains
121 a fragment identifier, in which case only the script element with
122 matching id attribute is extracted.
123 """
124 if encoding not in ("utf-8", "utf-16"):
125 warnings.warn(
126 "JSON should be encoded as unicode. "
127 "Given encoding was: %s" % encoding
128 )
129
130 if not base:
131 base = sink.absolutize(source.getPublicId() or source.getSystemId() or "")
132
133 context_data = context
134 if not context_data and hasattr(source, "url") and hasattr(source, "links"):
135 if TYPE_CHECKING:
136 assert isinstance(source, URLInputSource)
137 context_data = context_from_urlinputsource(source)
138
139 try:
140 version = float(version)
141 except ValueError:
142 version = 1.1
143
144 # Get the optional fragment identifier
145 try:
146 fragment_id = URIRef(source.getSystemId()).fragment
147 except Exception:
148 fragment_id = None
149
150 data, html_base = source_to_json(source, fragment_id, extract_all_scripts)
151 if html_base is not None:
152 base = URIRef(html_base, base=base)
153
154 # NOTE: A ConjunctiveGraph parses into a Graph sink, so no sink will be
155 # context_aware. Keeping this check in case RDFLib is changed, or
156 # someone passes something context_aware to this parser directly.
157 conj_sink: Graph
158 if not sink.context_aware:
159 conj_sink = ConjunctiveGraph(store=sink.store, identifier=sink.identifier)
160 else:
161 conj_sink = sink
162
163 to_rdf(
164 data,
165 conj_sink,
166 base,
167 context_data,
168 version,
169 bool(generalized_rdf),
170 skolemize=skolemize,
171 )
172
173
174def to_rdf(
175 data: Any,
176 dataset: Graph,
177 base: Optional[str] = None,
178 context_data: Optional[
179 Union[
180 List[Union[Dict[str, Any], str, None]],
181 Dict[str, Any],
182 str,
183 ]
184 ] = None,
185 version: Optional[float] = None,
186 generalized_rdf: bool = False,
187 allow_lists_of_lists: Optional[bool] = None,
188 skolemize: bool = False,
189):
190 # TODO: docstring w. args and return value
191 context = Context(base=base, version=version)
192 if context_data:
193 context.load(context_data)
194 parser = Parser(
195 generalized_rdf=generalized_rdf,
196 allow_lists_of_lists=allow_lists_of_lists,
197 skolemize=skolemize,
198 )
199 return parser.parse(data, context, dataset)
200
201
202class Parser:
203 def __init__(
204 self,
205 generalized_rdf: bool = False,
206 allow_lists_of_lists: Optional[bool] = None,
207 skolemize: bool = False,
208 ):
209 self.skolemize = skolemize
210 self.generalized_rdf = generalized_rdf
211 self.allow_lists_of_lists = (
212 allow_lists_of_lists
213 if allow_lists_of_lists is not None
214 else ALLOW_LISTS_OF_LISTS
215 )
216 self.invalid_uri_to_bnode: dict[str, BNode] = {}
217
218 def parse(self, data: Any, context: Context, dataset: Graph) -> Graph:
219 topcontext = False
220 resources: Union[Dict[str, Any], List[Any]]
221 if isinstance(data, list):
222 resources = data
223 elif isinstance(data, dict):
224 local_context = data.get(CONTEXT)
225 if local_context:
226 context.load(local_context, context.base)
227 topcontext = True
228 resources = data
229 # type error: Subclass of "Dict[str, Any]" and "List[Any]" cannot exist: would have incompatible method signatures
230 if not isinstance(resources, list): # type: ignore[unreachable]
231 resources = [resources]
232
233 if context.vocab:
234 dataset.bind(None, context.vocab)
235 for name, term in context.terms.items():
236 if term.id and term.id.endswith(VOCAB_DELIMS):
237 dataset.bind(name, term.id)
238
239 # type error: "Graph" has no attribute "default_context"
240 graph = dataset.default_context if dataset.context_aware else dataset # type: ignore[attr-defined]
241
242 for node in resources:
243 self._add_to_graph(dataset, graph, context, node, topcontext)
244
245 return graph
246
247 def _add_to_graph(
248 self,
249 dataset: Graph,
250 graph: Graph,
251 context: Context,
252 node: Any,
253 topcontext: bool = False,
254 ) -> Optional[Node]:
255 if not isinstance(node, dict) or context.get_value(node):
256 # type error: Return value expected
257 return # type: ignore[return-value]
258
259 if CONTEXT in node and not topcontext:
260 local_context = node[CONTEXT]
261 if local_context:
262 context = context.subcontext(local_context)
263 else:
264 context = Context(base=context.doc_base)
265
266 # type error: Incompatible types in assignment (expression has type "Optional[Context]", variable has type "Context")
267 context = context.get_context_for_type(node) # type: ignore[assignment]
268
269 id_val = context.get_id(node)
270
271 if id_val is None:
272 nested_id = self._get_nested_id(context, node)
273 if nested_id is not None and len(nested_id) > 0:
274 id_val = nested_id
275
276 if isinstance(id_val, str):
277 subj = self._to_rdf_id(context, id_val)
278 else:
279 subj = BNode()
280 if self.skolemize:
281 subj = subj.skolemize()
282
283 if subj is None:
284 return None
285
286 # NOTE: crude way to signify that this node might represent a named graph
287 no_id = id_val is None
288
289 for key, obj in node.items():
290 if key == CONTEXT or key in context.get_keys(ID):
291 continue
292
293 if key == REV or key in context.get_keys(REV):
294 for rkey, robj in obj.items():
295 self._key_to_graph(
296 dataset,
297 graph,
298 context,
299 subj,
300 rkey,
301 robj,
302 reverse=True,
303 no_id=no_id,
304 )
305 else:
306 self._key_to_graph(dataset, graph, context, subj, key, obj, no_id=no_id)
307
308 return subj
309
310 # type error: Missing return statement
311 def _get_nested_id(self, context: Context, node: Dict[str, Any]) -> Optional[str]: # type: ignore[return]
312 for key, obj in node.items():
313 if context.version >= 1.1 and key in context.get_keys(NEST):
314 term = context.terms.get(key)
315 if term and term.id is None:
316 continue
317 objs = obj if isinstance(obj, list) else [obj]
318 for obj in objs:
319 if not isinstance(obj, dict):
320 continue
321 id_val = context.get_id(obj)
322 if not id_val:
323 subcontext = context.get_context_for_term(
324 context.terms.get(key)
325 )
326 id_val = self._get_nested_id(subcontext, obj)
327 if isinstance(id_val, str):
328 return id_val
329
330 def _key_to_graph(
331 self,
332 dataset: Graph,
333 graph: Graph,
334 context: Context,
335 subj: Node,
336 key: str,
337 obj: Any,
338 reverse: bool = False,
339 no_id: bool = False,
340 ) -> None:
341 if isinstance(obj, list):
342 obj_nodes = obj
343 else:
344 obj_nodes = [obj]
345
346 term = context.terms.get(key)
347 if term:
348 term_id = term.id
349 if term.type == JSON:
350 obj_nodes = [self._to_typed_json_value(obj)]
351 elif LIST in term.container:
352 obj_nodes = [self._expand_nested_list(obj_nodes)]
353 elif isinstance(obj, dict):
354 obj_nodes = self._parse_container(context, term, obj)
355 else:
356 term_id = None
357
358 if TYPE in (key, term_id):
359 term = TYPE_TERM
360
361 if GRAPH in (key, term_id):
362 if dataset.context_aware and not no_id:
363 if TYPE_CHECKING:
364 assert isinstance(dataset, ConjunctiveGraph)
365 # type error: Argument 1 to "get_context" of "ConjunctiveGraph" has incompatible type "Node"; expected "Union[IdentifiedNode, str, None]"
366 subgraph = dataset.get_context(subj) # type: ignore[arg-type]
367 else:
368 subgraph = graph
369 for onode in obj_nodes:
370 self._add_to_graph(dataset, subgraph, context, onode)
371 return
372
373 if SET in (key, term_id):
374 for onode in obj_nodes:
375 self._add_to_graph(dataset, graph, context, onode)
376 return
377
378 if INCLUDED in (key, term_id):
379 for onode in obj_nodes:
380 self._add_to_graph(dataset, graph, context, onode)
381 return
382
383 if context.version >= 1.1 and key in context.get_keys(NEST):
384 term = context.terms.get(key)
385 if term and term.id is None:
386 return
387 objs = obj if isinstance(obj, list) else [obj]
388 for obj in objs:
389 if not isinstance(obj, dict):
390 continue
391 for nkey, nobj in obj.items():
392 # NOTE: we've already captured subject
393 if nkey in context.get_keys(ID):
394 continue
395 subcontext = context.get_context_for_type(obj)
396 # type error: Argument 3 to "_key_to_graph" of "Parser" has incompatible type "Optional[Context]"; expected "Context"
397 self._key_to_graph(dataset, graph, subcontext, subj, nkey, nobj) # type: ignore[arg-type]
398 return
399
400 pred_uri = term.id if term else context.expand(key)
401
402 context = context.get_context_for_term(term)
403
404 # Flatten deep nested lists
405 def flatten(n: Iterable[Any]) -> List[Any]:
406 flattened = []
407 for obj in n:
408 if isinstance(obj, dict):
409 objs = context.get_set(obj)
410 if objs is not None:
411 obj = objs
412 if isinstance(obj, list):
413 flattened += flatten(obj)
414 continue
415 flattened.append(obj)
416 return flattened
417
418 obj_nodes = flatten(obj_nodes)
419
420 if not pred_uri:
421 return
422
423 if term and term.reverse:
424 reverse = not reverse
425
426 pred: IdentifiedNode
427 bid = self._get_bnodeid(pred_uri)
428 if bid:
429 if not self.generalized_rdf:
430 return
431 pred = BNode(bid)
432 if self.skolemize:
433 pred = pred.skolemize()
434 else:
435 pred = URIRef(pred_uri)
436
437 for obj_node in obj_nodes:
438 obj = self._to_object(dataset, graph, context, term, obj_node)
439 if obj is None:
440 continue
441 if reverse:
442 graph.add((obj, pred, subj))
443 else:
444 graph.add((subj, pred, obj))
445
446 def _parse_container(
447 self, context: Context, term: Term, obj: Dict[str, Any]
448 ) -> List[Any]:
449 if LANG in term.container:
450 obj_nodes = []
451 for lang, values in obj.items():
452 if not isinstance(values, list):
453 values = [values]
454 if lang in context.get_keys(NONE):
455 obj_nodes += values
456 else:
457 for v in values:
458 obj_nodes.append((v, lang))
459 return obj_nodes
460
461 v11 = context.version >= 1.1
462
463 if v11 and GRAPH in term.container and ID in term.container:
464 return [
465 (
466 dict({GRAPH: o})
467 if k in context.get_keys(NONE)
468 else dict({ID: k, GRAPH: o}) if isinstance(o, dict) else o
469 )
470 for k, o in obj.items()
471 ]
472
473 elif v11 and GRAPH in term.container and INDEX in term.container:
474 return [dict({GRAPH: o}) for k, o in obj.items()]
475
476 elif v11 and GRAPH in term.container:
477 return [dict({GRAPH: obj})]
478
479 elif v11 and ID in term.container:
480 return [
481 (
482 dict({ID: k}, **o)
483 if isinstance(o, dict) and k not in context.get_keys(NONE)
484 else o
485 )
486 for k, o in obj.items()
487 ]
488
489 elif v11 and TYPE in term.container:
490 return [
491 (
492 self._add_type(
493 context,
494 (
495 {ID: context.expand(o) if term.type == VOCAB else o}
496 if isinstance(o, str)
497 else o
498 ),
499 k,
500 )
501 if isinstance(o, (dict, str)) and k not in context.get_keys(NONE)
502 else o
503 )
504 for k, o in obj.items()
505 ]
506
507 elif INDEX in term.container:
508 obj_nodes = []
509 for key, nodes in obj.items():
510 if not isinstance(nodes, list):
511 nodes = [nodes]
512 for node in nodes:
513 if v11 and term.index and key not in context.get_keys(NONE):
514 if not isinstance(node, dict):
515 node = {ID: node}
516 values = node.get(term.index, [])
517 if not isinstance(values, list):
518 values = [values]
519 values.append(key)
520 node[term.index] = values
521 obj_nodes.append(node)
522 return obj_nodes
523
524 return [obj]
525
526 @staticmethod
527 def _add_type(context: Context, o: Dict[str, Any], k: str) -> Dict[str, Any]:
528 otype = context.get_type(o) or []
529 if otype and not isinstance(otype, list):
530 otype = [otype]
531 otype.append(k)
532 o[TYPE] = otype
533 return o
534
535 def _to_object(
536 self,
537 dataset: Graph,
538 graph: Graph,
539 context: Context,
540 term: Optional[Term],
541 node: Any,
542 inlist: bool = False,
543 ) -> Optional[Node]:
544 if isinstance(node, tuple):
545 value, lang = node
546 if value is None:
547 # type error: Return value expected
548 return # type: ignore[return-value]
549 if lang and " " in lang:
550 # type error: Return value expected
551 return # type: ignore[return-value]
552 return Literal(value, lang=lang)
553
554 if isinstance(node, dict):
555 node_list = context.get_list(node)
556 if node_list is not None:
557 if inlist and not self.allow_lists_of_lists:
558 # type error: Return value expected
559 return # type: ignore[return-value]
560 listref = self._add_list(dataset, graph, context, term, node_list)
561 if listref:
562 return listref
563
564 else: # expand compacted value
565 if term and term.type:
566 if term.type == JSON:
567 node = self._to_typed_json_value(node)
568 elif node is None:
569 # type error: Return value expected
570 return # type: ignore[return-value]
571 elif term.type == ID and isinstance(node, str):
572 node = {ID: context.resolve(node)}
573 elif term.type == VOCAB and isinstance(node, str):
574 node = {ID: context.expand(node) or context.resolve_iri(node)}
575 else:
576 node = {TYPE: term.type, VALUE: node}
577 else:
578 if node is None:
579 # type error: Return value expected
580 return # type: ignore[return-value]
581 if isinstance(node, float):
582 return Literal(node, datatype=XSD.double)
583
584 if term and term.language is not UNDEF:
585 lang = term.language
586 else:
587 lang = context.language
588 return Literal(node, lang=lang)
589
590 lang = context.get_language(node)
591 datatype = not lang and context.get_type(node) or None
592 value = context.get_value(node)
593 # type error: Unsupported operand types for in ("Optional[Any]" and "Generator[str, None, None]")
594 if datatype in context.get_keys(JSON): # type: ignore[operator]
595 node = self._to_typed_json_value(value)
596 datatype = context.get_type(node)
597 value = context.get_value(node)
598
599 if lang or context.get_key(VALUE) in node or VALUE in node:
600 if value is None:
601 return None
602 if lang:
603 if " " in lang:
604 # type error: Return value expected
605 return # type: ignore[return-value]
606 return Literal(value, lang=lang)
607 elif datatype:
608 return Literal(value, datatype=context.expand(datatype))
609 else:
610 return Literal(value)
611 else:
612 return self._add_to_graph(dataset, graph, context, node)
613
614 def _to_rdf_id(self, context: Context, id_val: str) -> Optional[IdentifiedNode]:
615 bid = self._get_bnodeid(id_val)
616 if bid:
617 b = BNode(bid)
618 if self.skolemize:
619 return b.skolemize()
620 return b
621 else:
622 uri = context.resolve(id_val)
623 if not self.generalized_rdf and ":" not in uri:
624 return None
625 node: IdentifiedNode = URIRef(uri)
626 if not str(node):
627 if id_val not in self.invalid_uri_to_bnode:
628 self.invalid_uri_to_bnode[id_val] = BNode(secrets.token_urlsafe(20))
629 node = self.invalid_uri_to_bnode[id_val]
630 return node
631
632 def _get_bnodeid(self, ref: str) -> Optional[str]:
633 if not ref.startswith("_:"):
634 # type error: Return value expected
635 return # type: ignore[return-value]
636 bid = ref.split("_:", 1)[-1]
637 return bid or None
638
639 def _add_list(
640 self,
641 dataset: Graph,
642 graph: Graph,
643 context: Context,
644 term: Optional[Term],
645 node_list: Any,
646 ) -> IdentifiedNode:
647 if not isinstance(node_list, list):
648 node_list = [node_list]
649
650 first_subj: Union[URIRef, BNode] = BNode()
651 if self.skolemize and isinstance(first_subj, BNode):
652 first_subj = first_subj.skolemize()
653
654 rest: Union[URIRef, BNode, None]
655 subj, rest = first_subj, None
656
657 for node in node_list:
658 if node is None:
659 continue
660
661 if rest:
662 # type error: Statement is unreachable
663 graph.add((subj, RDF.rest, rest)) # type: ignore[unreachable]
664 subj = rest
665
666 obj = self._to_object(dataset, graph, context, term, node, inlist=True)
667
668 if obj is None:
669 continue
670
671 graph.add((subj, RDF.first, obj))
672 rest = BNode()
673 if self.skolemize and isinstance(rest, BNode):
674 rest = rest.skolemize()
675
676 if rest:
677 graph.add((subj, RDF.rest, RDF.nil))
678 return first_subj
679 else:
680 return RDF.nil
681
682 @staticmethod
683 def _to_typed_json_value(value: Any) -> Dict[str, str]:
684 if _HAS_ORJSON:
685 val_string: str = orjson.dumps(
686 value,
687 option=orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS,
688 ).decode("utf-8")
689 else:
690 val_string = json.dumps(
691 value, separators=(",", ":"), sort_keys=True, ensure_ascii=False
692 )
693 return {
694 TYPE: RDF.JSON,
695 VALUE: val_string,
696 }
697
698 @classmethod
699 def _expand_nested_list(cls, obj_nodes: List[Any]) -> Dict[str, List[Any]]:
700 result = [
701 cls._expand_nested_list(o) if isinstance(o, list) else o for o in obj_nodes
702 ]
703 return {LIST: result}