Source code

Download source archive ↓

src/rdflib/plugins/serializers/turtle.py

1"""
2Turtle RDF graph serializer for RDFLib.
3See <http://www.w3.org/TeamSubmission/turtle/> for syntax specification.
4"""
5
6from __future__ import annotations
7
8import re
9import warnings
10from collections import defaultdict
11from typing import (
12    IO,
13    TYPE_CHECKING,
14    Any,
15    DefaultDict,
16    Dict,
17    List,
18    Mapping,
19    Optional,
20    Sequence,
21    Tuple,
22    TypeVar,
23    Union,
24)
25
26from rdflib.exceptions import Error
27from rdflib.graph import Graph
28from rdflib.namespace import RDF, RDFS
29from rdflib.serializer import Serializer
30from rdflib.term import BNode, Literal, Node, URIRef
31
32_StrT = TypeVar("_StrT", bound=str)
33
34if TYPE_CHECKING:
35    from rdflib.graph import _PredicateType, _SubjectType, _TripleType
36
37__all__ = ["RecursiveSerializer", "TurtleSerializer"]
38
39
40class RecursiveSerializer(Serializer):
41    """Base class for recursive serializers."""
42
43    topClasses = [RDFS.Class]
44    predicateOrder = [RDF.type, RDFS.label]
45    maxDepth = 10
46    indentString = "  "
47    roundtrip_prefixes: Tuple[Any, ...] = ()
48    LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX = re.compile(
49        r"%(?![0-9A-Fa-f]{2})"
50    )
51
52    def __init__(self, store: Graph):
53        super(RecursiveSerializer, self).__init__(store)
54        self.stream: Optional[IO[bytes]] = None
55        self.reset()
56
57    def addNamespace(self, prefix: str, uri: URIRef) -> None:
58        if prefix in self.namespaces and self.namespaces[prefix] != uri:
59            raise Exception(
60                "Trying to override namespace prefix %s => %s, but it's already bound to %s"
61                % (prefix, uri, self.namespaces[prefix])
62            )
63        self.namespaces[prefix] = uri
64
65    def checkSubject(self, subject: _SubjectType) -> bool:
66        """Check to see if the subject should be serialized yet"""
67        if (
68            (self.isDone(subject))
69            or (subject not in self._subjects)
70            or ((subject in self._topLevels) and (self.depth > 1))
71            or (isinstance(subject, URIRef) and (self.depth >= self.maxDepth))
72        ):
73            return False
74        return True
75
76    def isDone(self, subject: _SubjectType) -> bool:
77        """Return true if subject is serialized"""
78        return subject in self._serialized
79
80    def orderSubjects(self) -> List[_SubjectType]:
81        seen: Dict[_SubjectType, bool] = {}
82        subjects: List[_SubjectType] = []
83
84        for classURI in self.topClasses:
85            members = list(self.store.subjects(RDF.type, classURI))
86            # type error: All overload variants of "sort" of "list" require at least one argument
87            members.sort()  # type: ignore[call-overload]
88
89            subjects.extend(members)
90            for member in members:
91                self._topLevels[member] = True
92                seen[member] = True
93
94        recursable = [
95            (isinstance(subject, BNode), self._references[subject], subject)
96            for subject in self._subjects
97            if subject not in seen
98        ]
99
100        recursable.sort()
101        subjects.extend([subject for (isbnode, refs, subject) in recursable])
102
103        return subjects
104
105    def preprocess(self) -> None:
106        for triple in self.store.triples((None, None, None)):
107            self.preprocessTriple(triple)
108
109    def preprocessTriple(self, spo: _TripleType) -> None:
110        s, p, o = spo
111        self._references[o] += 1
112        self._subjects[s] = True
113
114    def reset(self) -> None:
115        self.depth = 0
116        # Typed none because nothing is using it ...
117        self.lists: Dict[None, None] = {}
118        self.namespaces: Dict[str, URIRef] = {}
119        self._references: DefaultDict[Node, int] = defaultdict(int)
120        self._serialized: Dict[_SubjectType, bool] = {}
121        self._subjects: Dict[_SubjectType, bool] = {}
122        self._topLevels: Dict[_SubjectType, bool] = {}
123
124        if self.roundtrip_prefixes:
125            if hasattr(self.roundtrip_prefixes, "__iter__"):
126                for prefix, ns in self.store.namespaces():
127                    if prefix in self.roundtrip_prefixes:
128                        self.addNamespace(prefix, ns)
129            else:
130                for prefix, ns in self.store.namespaces():
131                    self.addNamespace(prefix, ns)
132
133    def buildPredicateHash(
134        self, subject: _SubjectType
135    ) -> Mapping[_PredicateType, List[Node]]:
136        """
137        Build a hash key by predicate to a list of objects for the given
138        subject
139        """
140        properties: Dict[_PredicateType, List[Node]] = {}
141        for s, p, o in self.store.triples((subject, None, None)):
142            oList = properties.get(p, [])
143            oList.append(o)
144            properties[p] = oList
145        return properties
146
147    def sortProperties(
148        self, properties: Mapping[_PredicateType, List[Node]]
149    ) -> List[_PredicateType]:
150        """Take a hash from predicate uris to lists of values.
151        Sort the lists of values.  Return a sorted list of properties."""
152        # Sort object lists
153        for prop, objects in properties.items():
154            # type error: All overload variants of "sort" of "list" require at least one argument
155            objects.sort()  # type: ignore[call-overload]
156
157        # Make sorted list of properties
158        propList: List[_PredicateType] = []
159        seen: Dict[_PredicateType, bool] = {}
160        for prop in self.predicateOrder:
161            if (prop in properties) and (prop not in seen):
162                propList.append(prop)
163                seen[prop] = True
164        props = list(properties.keys())
165        # type error: All overload variants of "sort" of "list" require at least one argument
166        props.sort()  # type: ignore[call-overload]
167        for prop in props:
168            if prop not in seen:
169                propList.append(prop)
170                seen[prop] = True
171        return propList
172
173    def subjectDone(self, subject: _SubjectType) -> None:
174        """Mark a subject as done."""
175        self._serialized[subject] = True
176
177    def indent(self, modifier: int = 0) -> str:
178        """Returns indent string multiplied by the depth"""
179        return (self.depth + modifier) * self.indentString
180
181    def write(self, text: str) -> None:
182        """Write text in given encoding."""
183        # type error: Item "None" of "Optional[IO[bytes]]" has no attribute "write"
184        self.stream.write(text.encode(self.encoding, "replace"))  # type: ignore[union-attr]
185
186    def relativize(self, uri: _StrT) -> Union[_StrT, URIRef]:
187        base = self.base
188        if (
189            base is not None
190            and uri.startswith(base)
191            and "#" not in uri.replace(base, "")
192            and "/" not in uri.replace(base, "")
193        ):
194            # type error: Incompatible types in assignment (expression has type "str", variable has type "Node")
195            uri = URIRef(uri.replace(base, "", 1))  # type: ignore[assignment]
196        return uri
197
198
199SUBJECT = 0
200VERB = 1
201OBJECT = 2
202
203_GEN_QNAME_FOR_DT = False
204_SPACIOUS_OUTPUT = False
205
206
207class TurtleSerializer(RecursiveSerializer):
208    """Turtle RDF graph serializer."""
209
210    short_name = "turtle"
211    indentString = "    "
212    LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX = re.compile(
213        r"%(?![0-9A-Fa-f]{2})"
214    )
215
216    def __init__(self, store: Graph):
217        self._ns_rewrite: Dict[str, str] = {}
218        super(TurtleSerializer, self).__init__(store)
219        self.keywords: Dict[Node, str] = {RDF.type: "a"}
220        self.reset()
221        self.stream = None
222        self._spacious = _SPACIOUS_OUTPUT
223
224    # type error: Return type "str" of "addNamespace" incompatible with return type "None" in supertype "RecursiveSerializer"
225    def addNamespace(self, prefix: str, namespace: URIRef) -> str:  # type: ignore[override]
226        # Turtle does not support prefix that start with _
227        # if they occur in the graph, rewrite to p_blah
228        # this is more complicated since we need to make sure p_blah
229        # does not already exist. And we register namespaces as we go, i.e.
230        # we may first see a triple with prefix _9 - rewrite it to p_9
231        # and then later find a triple with a "real" p_9 prefix
232
233        # so we need to keep track of ns rewrites we made so far.
234
235        if (prefix > "" and prefix[0] == "_") or self.namespaces.get(
236            prefix, namespace
237        ) != namespace:
238            if prefix not in self._ns_rewrite:
239                p = "p" + prefix
240                while p in self.namespaces:
241                    p = "p" + p
242                self._ns_rewrite[prefix] = p
243
244            prefix = self._ns_rewrite.get(prefix, prefix)
245
246        super(TurtleSerializer, self).addNamespace(prefix, namespace)
247        return prefix
248
249    def reset(self) -> None:
250        super(TurtleSerializer, self).reset()
251        # typing as Dict[None, None] because nothing seems to be using it
252        self._shortNames: Dict[None, None] = {}
253        self._started = False
254        self._ns_rewrite = {}
255
256    def serialize(
257        self,
258        stream: IO[bytes],
259        base: Optional[str] = None,
260        encoding: Optional[str] = None,
261        spacious: Optional[bool] = None,
262        **kwargs: Any,
263    ) -> None:
264        self.reset()
265        self.stream = stream
266        # if base is given here, use that, if not and a base is set for the graph use that
267        if base is not None:
268            self.base = base
269        elif self.store.base is not None:
270            self.base = self.store.base
271
272        if spacious is not None:
273            self._spacious = spacious
274
275        self.preprocess()
276        subjects_list = self.orderSubjects()
277
278        self.startDocument()
279
280        firstTime = True
281        for subject in subjects_list:
282            if self.isDone(subject):
283                continue
284            if firstTime:
285                firstTime = False
286            if self.statement(subject) and not firstTime:
287                self.write("\n")
288
289        self.endDocument()
290        stream.write("\n".encode("latin-1"))
291
292        self.base = None
293
294    def preprocessTriple(self, triple: _TripleType) -> None:
295        super(TurtleSerializer, self).preprocessTriple(triple)
296        for i, node in enumerate(triple):
297            if i == VERB:
298                if node in self.keywords:
299                    # predicate is a keyword
300                    continue
301                if (
302                    self.base is not None
303                    and isinstance(node, URIRef)
304                    and node.startswith(self.base)
305                    and "#" not in node.replace(self.base, "")
306                    and "/" not in node.replace(self.base, "")
307                ):
308                    # predicate corresponds to base namespace
309                    continue
310            # Don't use generated prefixes for subjects and objects
311            self.get_pname(node, gen_prefix=(i == VERB))
312            if isinstance(node, Literal) and node.datatype:
313                self.get_pname(node.datatype, gen_prefix=_GEN_QNAME_FOR_DT)
314        p = triple[1]
315        if isinstance(p, BNode):  # hmm - when is P ever a bnode?
316            self._references[p] += 1
317
318    # Refer to Productions for terminals PNAME_NS and PNAME_LN https://www.w3.org/TR/turtle/#sec-grammar-grammar
319    def get_pname(self, uri: Node, gen_prefix: bool = True) -> Optional[str]:
320        if not isinstance(uri, URIRef):
321            return None
322
323        parts = None
324
325        try:
326            parts = self.store.compute_qname(uri, generate=gen_prefix)
327        except Exception:
328            # is the uri a namespace in itself?
329            pfx = self.store.store.prefix(uri)
330
331            if pfx is not None:
332                parts = (pfx, uri, "")
333            else:
334                # nothing worked
335                return None
336
337        prefix, namespace, local = parts
338
339        # To understand treatment of % character refer to Productions for terminal PLX at
340        # https://www.w3.org/TR/turtle/#grammar-production-PLX
341        # Only % NOT followed by two hex chars requires manual backslash escaping
342        local = local.replace(r"(", r"\(").replace(r")", r"\)")
343        local = self.LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX.sub(
344            "\\%", local
345        )
346
347        # PName cannot end with .
348        if local.endswith("."):
349            return None
350
351        prefix = self.addNamespace(prefix, namespace)
352
353        return "%s:%s" % (prefix, local)
354
355    def getQName(self, uri: Node, gen_prefix: bool = True) -> Optional[str]:
356        warnings.warn(
357            "TurtleSerializer.getQName is deprecated, use TurtleSerializer.get_pname instead.",
358            DeprecationWarning,
359            stacklevel=2,
360        )
361        return self.get_pname(uri, gen_prefix)
362
363    def startDocument(self) -> None:
364        self._started = True
365        ns_list = sorted(self.namespaces.items())
366
367        if self.base:
368            self.write(self.indent() + "@base <%s> .\n" % self.base)
369        for prefix, uri in ns_list:
370            self.write(self.indent() + "@prefix %s: <%s> .\n" % (prefix, uri))
371        if ns_list and self._spacious:
372            self.write("\n")
373
374    def endDocument(self) -> None:
375        if self._spacious:
376            self.write("\n")
377
378    def statement(self, subject: _SubjectType) -> bool:
379        self.subjectDone(subject)
380        return self.s_squared(subject) or self.s_default(subject)
381
382    def s_default(self, subject: _SubjectType) -> bool:
383        self.write("\n" + self.indent())
384        self.path(subject, SUBJECT)
385        self.predicateList(subject)
386        self.write(" .")
387        return True
388
389    def s_squared(self, subject: _SubjectType) -> bool:
390        if (self._references[subject] > 0) or not isinstance(subject, BNode):
391            return False
392        self.write("\n" + self.indent() + "[]")
393        self.predicateList(subject)
394        self.write(" .")
395        return True
396
397    def path(self, node: Node, position: int, newline: bool = False) -> None:
398        if not (
399            self.p_squared(node, position, newline)
400            or self.p_default(node, position, newline)
401        ):
402            raise Error("Cannot serialize node '%s'" % (node,))
403
404    def p_default(self, node: Node, position: int, newline: bool = False) -> bool:
405        if position != SUBJECT and not newline:
406            self.write(" ")
407        self.write(self.label(node, position))
408        return True
409
410    def label(self, node: Node, position: int) -> str:
411        if node == RDF.nil:
412            return "()"
413        if position is VERB and node in self.keywords:
414            return self.keywords[node]
415        if isinstance(node, Literal):
416            return node._literal_n3(
417                use_plain=True,
418                qname_callback=lambda dt: self.get_pname(dt, _GEN_QNAME_FOR_DT),
419            )
420        else:
421            node = self.relativize(node)  # type: ignore[type-var]
422
423            return self.get_pname(node, position == VERB) or node.n3()
424
425    def p_squared(self, node: Node, position: int, newline: bool = False) -> bool:
426        if (
427            not isinstance(node, BNode)
428            or node in self._serialized
429            or self._references[node] > 1
430            or position == SUBJECT
431        ):
432            return False
433
434        if not newline:
435            self.write(" ")
436
437        if self.isValidList(node):
438            # this is a list
439            self.write("(")
440            self.depth += 1  # 2
441            self.doList(node)
442            self.depth -= 1  # 2
443            self.write(" )")
444        else:
445            self.subjectDone(node)
446            self.depth += 2
447            # self.write('[\n' + self.indent())
448            self.write("[")
449            self.depth -= 1
450            # self.predicateList(node, newline=True)
451            self.predicateList(node, newline=False)
452            # self.write('\n' + self.indent() + ']')
453            self.write(" ]")
454            self.depth -= 1
455
456        return True
457
458    def isValidList(self, l_: Node) -> bool:
459        """
460        Checks if l is a valid RDF list, i.e. no nodes have other properties.
461        """
462        try:
463            if self.store.value(l_, RDF.first) is None:
464                return False
465        except Exception:
466            return False
467        while l_:
468            if l_ != RDF.nil and len(list(self.store.predicate_objects(l_))) != 2:
469                return False
470            # type error: Incompatible types in assignment (expression has type "Optional[Node]", variable has type "Node")
471            l_ = self.store.value(l_, RDF.rest)  # type: ignore[assignment]
472        return True
473
474    def doList(self, l_: Node) -> None:
475        while l_:
476            item = self.store.value(l_, RDF.first)
477            if item is not None:
478                self.path(item, OBJECT)
479                self.subjectDone(l_)
480            # type error: Incompatible types in assignment (expression has type "Optional[Node]", variable has type "Node")
481            l_ = self.store.value(l_, RDF.rest)  # type: ignore[assignment]
482
483    def predicateList(self, subject: Node, newline: bool = False) -> None:
484        properties = self.buildPredicateHash(subject)
485        propList = self.sortProperties(properties)
486        if len(propList) == 0:
487            return
488        self.verb(propList[0], newline=newline)
489        self.objectList(properties[propList[0]])
490        for predicate in propList[1:]:
491            self.write(" ;\n" + self.indent(1))
492            self.verb(predicate, newline=True)
493            self.objectList(properties[predicate])
494
495    def verb(self, node: Node, newline: bool = False) -> None:
496        self.path(node, VERB, newline)
497
498    def objectList(self, objects: Sequence[Node]) -> None:
499        count = len(objects)
500        if count == 0:
501            return
502        depthmod = (count == 1) and 0 or 1
503        self.depth += depthmod
504        self.path(objects[0], OBJECT)
505        for obj in objects[1:]:
506            self.write(",\n" + self.indent(1))
507            self.path(obj, OBJECT, newline=True)
508        self.depth -= depthmod