Source code

Download source archive ↓

src/rdflib/plugins/parsers/jsonld.py

1"""
2This parser will interpret a JSON-LD document as an RDF Graph. See http://json-ld.org/
3
4Example:
5    ```python
6    >>> from rdflib import Graph, URIRef, Literal
7    >>> test_json = '''
8    ... {
9    ...     "@context": {
10    ...         "dc": "http://purl.org/dc/terms/",
11    ...         "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
12    ...         "rdfs": "http://www.w3.org/2000/01/rdf-schema#"
13    ...     },
14    ...     "@id": "http://example.org/about",
15    ...     "dc:title": {
16    ...         "@language": "en",
17    ...         "@value": "Someone's Homepage"
18    ...     }
19    ... }
20    ... '''
21    >>> g = Graph().parse(data=test_json, format='json-ld')
22    >>> list(g) == [(URIRef('http://example.org/about'),
23    ...     URIRef('http://purl.org/dc/terms/title'),
24    ...     Literal("Someone's Homepage", lang='en'))]
25    True
26
27    ```
28"""
29
30# From: https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/parser.py
31
32# NOTE: This code reads the entire JSON object into memory before parsing, but
33# we should consider streaming the input to deal with arbitrarily large graphs.
34from __future__ import annotations
35
36import secrets
37import warnings
38from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
39
40import rdflib.parser
41from rdflib.graph import ConjunctiveGraph, Graph
42from rdflib.namespace import RDF, XSD
43from rdflib.parser import InputSource, URLInputSource
44from rdflib.term import BNode, IdentifiedNode, Literal, Node, URIRef
45
46from ..shared.jsonld.context import UNDEF, Context, Term
47from ..shared.jsonld.keys import (
48    CONTEXT,
49    GRAPH,
50    ID,
51    INCLUDED,
52    INDEX,
53    JSON,
54    LANG,
55    LIST,
56    NEST,
57    NONE,
58    REV,
59    SET,
60    TYPE,
61    VALUE,
62    VOCAB,
63)
64from ..shared.jsonld.util import (
65    _HAS_ORJSON,
66    VOCAB_DELIMS,
67    context_from_urlinputsource,
68    json,
69    orjson,
70    source_to_json,
71)
72
73__all__ = ["JsonLDParser", "to_rdf"]
74
75TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB)  # type: ignore[call-arg]
76
77ALLOW_LISTS_OF_LISTS = True  # NOTE: Not allowed in JSON-LD 1.0
78
79
80class JsonLDParser(rdflib.parser.Parser):
81    def __init__(self):
82        super(JsonLDParser, self).__init__()
83
84    def parse(
85        self,
86        source: InputSource,
87        sink: Graph,
88        version: float = 1.1,
89        skolemize: bool = False,
90        encoding: Optional[str] = "utf-8",
91        base: Optional[str] = None,
92        context: Optional[
93            Union[
94                List[Union[Dict[str, Any], str, None]],
95                Dict[str, Any],
96                str,
97            ]
98        ] = None,
99        generalized_rdf: Optional[bool] = False,
100        extract_all_scripts: Optional[bool] = False,
101        **kwargs: Any,
102    ) -> None:
103        """Parse JSON-LD from a source document.
104
105        The source document can be JSON or HTML with embedded JSON script
106        elements (type attribute = `application/ld+json`). To process as HTML
107        `source.content_type` must be set to "text/html" or
108        `application/xhtml+xml.
109
110        Args:
111            source: InputSource with JSON-formatted data (JSON or HTML)
112            sink: Graph to receive the parsed triples
113            version: parse as JSON-LD version, defaults to 1.1
114            skolemize: whether to skolemize blank nodes, defaults to False
115            encoding: character encoding of the JSON (should be "utf-8"
116            base: JSON-LD [Base IRI](https://www.w3.org/TR/json-ld/#base-iri), defaults to None
117            context: JSON-LD [Context](https://www.w3.org/TR/json-ld/#the-context), defaults to None
118            generalized_rdf: parse as [Generalized RDF](https://www.w3.org/TR/json-ld/#relationship-to-rdf), defaults to False
119            extract_all_scripts: if source is an HTML document then extract
120                script element). This is ignored if `source.system_id` contains
121                a fragment identifier, in which case only the script element with
122                matching id attribute is extracted.
123        """
124        if encoding not in ("utf-8", "utf-16"):
125            warnings.warn(
126                "JSON should be encoded as unicode. "
127                "Given encoding was: %s" % encoding
128            )
129
130        if not base:
131            base = sink.absolutize(source.getPublicId() or source.getSystemId() or "")
132
133        context_data = context
134        if not context_data and hasattr(source, "url") and hasattr(source, "links"):
135            if TYPE_CHECKING:
136                assert isinstance(source, URLInputSource)
137            context_data = context_from_urlinputsource(source)
138
139        try:
140            version = float(version)
141        except ValueError:
142            version = 1.1
143
144        # Get the optional fragment identifier
145        try:
146            fragment_id = URIRef(source.getSystemId()).fragment
147        except Exception:
148            fragment_id = None
149
150        data, html_base = source_to_json(source, fragment_id, extract_all_scripts)
151        if html_base is not None:
152            base = URIRef(html_base, base=base)
153
154        # NOTE: A ConjunctiveGraph parses into a Graph sink, so no sink will be
155        # context_aware. Keeping this check in case RDFLib is changed, or
156        # someone passes something context_aware to this parser directly.
157        conj_sink: Graph
158        if not sink.context_aware:
159            conj_sink = ConjunctiveGraph(store=sink.store, identifier=sink.identifier)
160        else:
161            conj_sink = sink
162
163        to_rdf(
164            data,
165            conj_sink,
166            base,
167            context_data,
168            version,
169            bool(generalized_rdf),
170            skolemize=skolemize,
171        )
172
173
174def to_rdf(
175    data: Any,
176    dataset: Graph,
177    base: Optional[str] = None,
178    context_data: Optional[
179        Union[
180            List[Union[Dict[str, Any], str, None]],
181            Dict[str, Any],
182            str,
183        ]
184    ] = None,
185    version: Optional[float] = None,
186    generalized_rdf: bool = False,
187    allow_lists_of_lists: Optional[bool] = None,
188    skolemize: bool = False,
189):
190    # TODO: docstring w. args and return value
191    context = Context(base=base, version=version)
192    if context_data:
193        context.load(context_data)
194    parser = Parser(
195        generalized_rdf=generalized_rdf,
196        allow_lists_of_lists=allow_lists_of_lists,
197        skolemize=skolemize,
198    )
199    return parser.parse(data, context, dataset)
200
201
202class Parser:
203    def __init__(
204        self,
205        generalized_rdf: bool = False,
206        allow_lists_of_lists: Optional[bool] = None,
207        skolemize: bool = False,
208    ):
209        self.skolemize = skolemize
210        self.generalized_rdf = generalized_rdf
211        self.allow_lists_of_lists = (
212            allow_lists_of_lists
213            if allow_lists_of_lists is not None
214            else ALLOW_LISTS_OF_LISTS
215        )
216        self.invalid_uri_to_bnode: dict[str, BNode] = {}
217
218    def parse(self, data: Any, context: Context, dataset: Graph) -> Graph:
219        topcontext = False
220        resources: Union[Dict[str, Any], List[Any]]
221        if isinstance(data, list):
222            resources = data
223        elif isinstance(data, dict):
224            local_context = data.get(CONTEXT)
225            if local_context:
226                context.load(local_context, context.base)
227                topcontext = True
228            resources = data
229            # type error: Subclass of "Dict[str, Any]" and "List[Any]" cannot exist: would have incompatible method signatures
230            if not isinstance(resources, list):  # type: ignore[unreachable]
231                resources = [resources]
232
233        if context.vocab:
234            dataset.bind(None, context.vocab)
235        for name, term in context.terms.items():
236            if term.id and term.id.endswith(VOCAB_DELIMS):
237                dataset.bind(name, term.id)
238
239        # type error: "Graph" has no attribute "default_context"
240        graph = dataset.default_context if dataset.context_aware else dataset  # type: ignore[attr-defined]
241
242        for node in resources:
243            self._add_to_graph(dataset, graph, context, node, topcontext)
244
245        return graph
246
247    def _add_to_graph(
248        self,
249        dataset: Graph,
250        graph: Graph,
251        context: Context,
252        node: Any,
253        topcontext: bool = False,
254    ) -> Optional[Node]:
255        if not isinstance(node, dict) or context.get_value(node):
256            # type error: Return value expected
257            return  # type: ignore[return-value]
258
259        if CONTEXT in node and not topcontext:
260            local_context = node[CONTEXT]
261            if local_context:
262                context = context.subcontext(local_context)
263            else:
264                context = Context(base=context.doc_base)
265
266        # type error: Incompatible types in assignment (expression has type "Optional[Context]", variable has type "Context")
267        context = context.get_context_for_type(node)  # type: ignore[assignment]
268
269        id_val = context.get_id(node)
270
271        if id_val is None:
272            nested_id = self._get_nested_id(context, node)
273            if nested_id is not None and len(nested_id) > 0:
274                id_val = nested_id
275
276        if isinstance(id_val, str):
277            subj = self._to_rdf_id(context, id_val)
278        else:
279            subj = BNode()
280            if self.skolemize:
281                subj = subj.skolemize()
282
283        if subj is None:
284            return None
285
286        # NOTE: crude way to signify that this node might represent a named graph
287        no_id = id_val is None
288
289        for key, obj in node.items():
290            if key == CONTEXT or key in context.get_keys(ID):
291                continue
292
293            if key == REV or key in context.get_keys(REV):
294                for rkey, robj in obj.items():
295                    self._key_to_graph(
296                        dataset,
297                        graph,
298                        context,
299                        subj,
300                        rkey,
301                        robj,
302                        reverse=True,
303                        no_id=no_id,
304                    )
305            else:
306                self._key_to_graph(dataset, graph, context, subj, key, obj, no_id=no_id)
307
308        return subj
309
310    # type error: Missing return statement
311    def _get_nested_id(self, context: Context, node: Dict[str, Any]) -> Optional[str]:  # type: ignore[return]
312        for key, obj in node.items():
313            if context.version >= 1.1 and key in context.get_keys(NEST):
314                term = context.terms.get(key)
315                if term and term.id is None:
316                    continue
317                objs = obj if isinstance(obj, list) else [obj]
318                for obj in objs:
319                    if not isinstance(obj, dict):
320                        continue
321                    id_val = context.get_id(obj)
322                    if not id_val:
323                        subcontext = context.get_context_for_term(
324                            context.terms.get(key)
325                        )
326                        id_val = self._get_nested_id(subcontext, obj)
327                    if isinstance(id_val, str):
328                        return id_val
329
330    def _key_to_graph(
331        self,
332        dataset: Graph,
333        graph: Graph,
334        context: Context,
335        subj: Node,
336        key: str,
337        obj: Any,
338        reverse: bool = False,
339        no_id: bool = False,
340    ) -> None:
341        if isinstance(obj, list):
342            obj_nodes = obj
343        else:
344            obj_nodes = [obj]
345
346        term = context.terms.get(key)
347        if term:
348            term_id = term.id
349            if term.type == JSON:
350                obj_nodes = [self._to_typed_json_value(obj)]
351            elif LIST in term.container:
352                obj_nodes = [self._expand_nested_list(obj_nodes)]
353            elif isinstance(obj, dict):
354                obj_nodes = self._parse_container(context, term, obj)
355        else:
356            term_id = None
357
358        if TYPE in (key, term_id):
359            term = TYPE_TERM
360
361        if GRAPH in (key, term_id):
362            if dataset.context_aware and not no_id:
363                if TYPE_CHECKING:
364                    assert isinstance(dataset, ConjunctiveGraph)
365                # type error: Argument 1 to "get_context" of "ConjunctiveGraph" has incompatible type "Node"; expected "Union[IdentifiedNode, str, None]"
366                subgraph = dataset.get_context(subj)  # type: ignore[arg-type]
367            else:
368                subgraph = graph
369            for onode in obj_nodes:
370                self._add_to_graph(dataset, subgraph, context, onode)
371            return
372
373        if SET in (key, term_id):
374            for onode in obj_nodes:
375                self._add_to_graph(dataset, graph, context, onode)
376            return
377
378        if INCLUDED in (key, term_id):
379            for onode in obj_nodes:
380                self._add_to_graph(dataset, graph, context, onode)
381            return
382
383        if context.version >= 1.1 and key in context.get_keys(NEST):
384            term = context.terms.get(key)
385            if term and term.id is None:
386                return
387            objs = obj if isinstance(obj, list) else [obj]
388            for obj in objs:
389                if not isinstance(obj, dict):
390                    continue
391                for nkey, nobj in obj.items():
392                    # NOTE: we've already captured subject
393                    if nkey in context.get_keys(ID):
394                        continue
395                    subcontext = context.get_context_for_type(obj)
396                    # type error: Argument 3 to "_key_to_graph" of "Parser" has incompatible type "Optional[Context]"; expected "Context"
397                    self._key_to_graph(dataset, graph, subcontext, subj, nkey, nobj)  # type: ignore[arg-type]
398            return
399
400        pred_uri = term.id if term else context.expand(key)
401
402        context = context.get_context_for_term(term)
403
404        # Flatten deep nested lists
405        def flatten(n: Iterable[Any]) -> List[Any]:
406            flattened = []
407            for obj in n:
408                if isinstance(obj, dict):
409                    objs = context.get_set(obj)
410                    if objs is not None:
411                        obj = objs
412                if isinstance(obj, list):
413                    flattened += flatten(obj)
414                    continue
415                flattened.append(obj)
416            return flattened
417
418        obj_nodes = flatten(obj_nodes)
419
420        if not pred_uri:
421            return
422
423        if term and term.reverse:
424            reverse = not reverse
425
426        pred: IdentifiedNode
427        bid = self._get_bnodeid(pred_uri)
428        if bid:
429            if not self.generalized_rdf:
430                return
431            pred = BNode(bid)
432            if self.skolemize:
433                pred = pred.skolemize()
434        else:
435            pred = URIRef(pred_uri)
436
437        for obj_node in obj_nodes:
438            obj = self._to_object(dataset, graph, context, term, obj_node)
439            if obj is None:
440                continue
441            if reverse:
442                graph.add((obj, pred, subj))
443            else:
444                graph.add((subj, pred, obj))
445
446    def _parse_container(
447        self, context: Context, term: Term, obj: Dict[str, Any]
448    ) -> List[Any]:
449        if LANG in term.container:
450            obj_nodes = []
451            for lang, values in obj.items():
452                if not isinstance(values, list):
453                    values = [values]
454                if lang in context.get_keys(NONE):
455                    obj_nodes += values
456                else:
457                    for v in values:
458                        obj_nodes.append((v, lang))
459            return obj_nodes
460
461        v11 = context.version >= 1.1
462
463        if v11 and GRAPH in term.container and ID in term.container:
464            return [
465                (
466                    dict({GRAPH: o})
467                    if k in context.get_keys(NONE)
468                    else dict({ID: k, GRAPH: o}) if isinstance(o, dict) else o
469                )
470                for k, o in obj.items()
471            ]
472
473        elif v11 and GRAPH in term.container and INDEX in term.container:
474            return [dict({GRAPH: o}) for k, o in obj.items()]
475
476        elif v11 and GRAPH in term.container:
477            return [dict({GRAPH: obj})]
478
479        elif v11 and ID in term.container:
480            return [
481                (
482                    dict({ID: k}, **o)
483                    if isinstance(o, dict) and k not in context.get_keys(NONE)
484                    else o
485                )
486                for k, o in obj.items()
487            ]
488
489        elif v11 and TYPE in term.container:
490            return [
491                (
492                    self._add_type(
493                        context,
494                        (
495                            {ID: context.expand(o) if term.type == VOCAB else o}
496                            if isinstance(o, str)
497                            else o
498                        ),
499                        k,
500                    )
501                    if isinstance(o, (dict, str)) and k not in context.get_keys(NONE)
502                    else o
503                )
504                for k, o in obj.items()
505            ]
506
507        elif INDEX in term.container:
508            obj_nodes = []
509            for key, nodes in obj.items():
510                if not isinstance(nodes, list):
511                    nodes = [nodes]
512                for node in nodes:
513                    if v11 and term.index and key not in context.get_keys(NONE):
514                        if not isinstance(node, dict):
515                            node = {ID: node}
516                        values = node.get(term.index, [])
517                        if not isinstance(values, list):
518                            values = [values]
519                        values.append(key)
520                        node[term.index] = values
521                    obj_nodes.append(node)
522            return obj_nodes
523
524        return [obj]
525
526    @staticmethod
527    def _add_type(context: Context, o: Dict[str, Any], k: str) -> Dict[str, Any]:
528        otype = context.get_type(o) or []
529        if otype and not isinstance(otype, list):
530            otype = [otype]
531        otype.append(k)
532        o[TYPE] = otype
533        return o
534
535    def _to_object(
536        self,
537        dataset: Graph,
538        graph: Graph,
539        context: Context,
540        term: Optional[Term],
541        node: Any,
542        inlist: bool = False,
543    ) -> Optional[Node]:
544        if isinstance(node, tuple):
545            value, lang = node
546            if value is None:
547                # type error: Return value expected
548                return  # type: ignore[return-value]
549            if lang and " " in lang:
550                # type error: Return value expected
551                return  # type: ignore[return-value]
552            return Literal(value, lang=lang)
553
554        if isinstance(node, dict):
555            node_list = context.get_list(node)
556            if node_list is not None:
557                if inlist and not self.allow_lists_of_lists:
558                    # type error: Return value expected
559                    return  # type: ignore[return-value]
560                listref = self._add_list(dataset, graph, context, term, node_list)
561                if listref:
562                    return listref
563
564        else:  # expand compacted value
565            if term and term.type:
566                if term.type == JSON:
567                    node = self._to_typed_json_value(node)
568                elif node is None:
569                    # type error: Return value expected
570                    return  # type: ignore[return-value]
571                elif term.type == ID and isinstance(node, str):
572                    node = {ID: context.resolve(node)}
573                elif term.type == VOCAB and isinstance(node, str):
574                    node = {ID: context.expand(node) or context.resolve_iri(node)}
575                else:
576                    node = {TYPE: term.type, VALUE: node}
577            else:
578                if node is None:
579                    # type error: Return value expected
580                    return  # type: ignore[return-value]
581                if isinstance(node, float):
582                    return Literal(node, datatype=XSD.double)
583
584                if term and term.language is not UNDEF:
585                    lang = term.language
586                else:
587                    lang = context.language
588                return Literal(node, lang=lang)
589
590        lang = context.get_language(node)
591        datatype = not lang and context.get_type(node) or None
592        value = context.get_value(node)
593        # type error: Unsupported operand types for in ("Optional[Any]" and "Generator[str, None, None]")
594        if datatype in context.get_keys(JSON):  # type: ignore[operator]
595            node = self._to_typed_json_value(value)
596            datatype = context.get_type(node)
597            value = context.get_value(node)
598
599        if lang or context.get_key(VALUE) in node or VALUE in node:
600            if value is None:
601                return None
602            if lang:
603                if " " in lang:
604                    # type error: Return value expected
605                    return  # type: ignore[return-value]
606                return Literal(value, lang=lang)
607            elif datatype:
608                return Literal(value, datatype=context.expand(datatype))
609            else:
610                return Literal(value)
611        else:
612            return self._add_to_graph(dataset, graph, context, node)
613
614    def _to_rdf_id(self, context: Context, id_val: str) -> Optional[IdentifiedNode]:
615        bid = self._get_bnodeid(id_val)
616        if bid:
617            b = BNode(bid)
618            if self.skolemize:
619                return b.skolemize()
620            return b
621        else:
622            uri = context.resolve(id_val)
623            if not self.generalized_rdf and ":" not in uri:
624                return None
625            node: IdentifiedNode = URIRef(uri)
626            if not str(node):
627                if id_val not in self.invalid_uri_to_bnode:
628                    self.invalid_uri_to_bnode[id_val] = BNode(secrets.token_urlsafe(20))
629                node = self.invalid_uri_to_bnode[id_val]
630            return node
631
632    def _get_bnodeid(self, ref: str) -> Optional[str]:
633        if not ref.startswith("_:"):
634            # type error: Return value expected
635            return  # type: ignore[return-value]
636        bid = ref.split("_:", 1)[-1]
637        return bid or None
638
639    def _add_list(
640        self,
641        dataset: Graph,
642        graph: Graph,
643        context: Context,
644        term: Optional[Term],
645        node_list: Any,
646    ) -> IdentifiedNode:
647        if not isinstance(node_list, list):
648            node_list = [node_list]
649
650        first_subj: Union[URIRef, BNode] = BNode()
651        if self.skolemize and isinstance(first_subj, BNode):
652            first_subj = first_subj.skolemize()
653
654        rest: Union[URIRef, BNode, None]
655        subj, rest = first_subj, None
656
657        for node in node_list:
658            if node is None:
659                continue
660
661            if rest:
662                # type error: Statement is unreachable
663                graph.add((subj, RDF.rest, rest))  # type: ignore[unreachable]
664                subj = rest
665
666            obj = self._to_object(dataset, graph, context, term, node, inlist=True)
667
668            if obj is None:
669                continue
670
671            graph.add((subj, RDF.first, obj))
672            rest = BNode()
673            if self.skolemize and isinstance(rest, BNode):
674                rest = rest.skolemize()
675
676        if rest:
677            graph.add((subj, RDF.rest, RDF.nil))
678            return first_subj
679        else:
680            return RDF.nil
681
682    @staticmethod
683    def _to_typed_json_value(value: Any) -> Dict[str, str]:
684        if _HAS_ORJSON:
685            val_string: str = orjson.dumps(
686                value,
687                option=orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS,
688            ).decode("utf-8")
689        else:
690            val_string = json.dumps(
691                value, separators=(",", ":"), sort_keys=True, ensure_ascii=False
692            )
693        return {
694            TYPE: RDF.JSON,
695            VALUE: val_string,
696        }
697
698    @classmethod
699    def _expand_nested_list(cls, obj_nodes: List[Any]) -> Dict[str, List[Any]]:
700        result = [
701            cls._expand_nested_list(o) if isinstance(o, list) else o for o in obj_nodes
702        ]
703        return {LIST: result}