src/rdflib/plugins/serializers/turtle.py
1"""
2Turtle RDF graph serializer for RDFLib.
3See <http://www.w3.org/TeamSubmission/turtle/> for syntax specification.
4"""
5
6from __future__ import annotations
7
8import re
9import warnings
10from collections import defaultdict
11from typing import (
12 IO,
13 TYPE_CHECKING,
14 Any,
15 DefaultDict,
16 Dict,
17 List,
18 Mapping,
19 Optional,
20 Sequence,
21 Tuple,
22 TypeVar,
23 Union,
24)
25
26from rdflib.exceptions import Error
27from rdflib.graph import Graph
28from rdflib.namespace import RDF, RDFS
29from rdflib.serializer import Serializer
30from rdflib.term import BNode, Literal, Node, URIRef
31
32_StrT = TypeVar("_StrT", bound=str)
33
34if TYPE_CHECKING:
35 from rdflib.graph import _PredicateType, _SubjectType, _TripleType
36
37__all__ = ["RecursiveSerializer", "TurtleSerializer"]
38
39
40class RecursiveSerializer(Serializer):
41 """Base class for recursive serializers."""
42
43 topClasses = [RDFS.Class]
44 predicateOrder = [RDF.type, RDFS.label]
45 maxDepth = 10
46 indentString = " "
47 roundtrip_prefixes: Tuple[Any, ...] = ()
48 LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX = re.compile(
49 r"%(?![0-9A-Fa-f]{2})"
50 )
51
52 def __init__(self, store: Graph):
53 super(RecursiveSerializer, self).__init__(store)
54 self.stream: Optional[IO[bytes]] = None
55 self.reset()
56
57 def addNamespace(self, prefix: str, uri: URIRef) -> None:
58 if prefix in self.namespaces and self.namespaces[prefix] != uri:
59 raise Exception(
60 "Trying to override namespace prefix %s => %s, but it's already bound to %s"
61 % (prefix, uri, self.namespaces[prefix])
62 )
63 self.namespaces[prefix] = uri
64
65 def checkSubject(self, subject: _SubjectType) -> bool:
66 """Check to see if the subject should be serialized yet"""
67 if (
68 (self.isDone(subject))
69 or (subject not in self._subjects)
70 or ((subject in self._topLevels) and (self.depth > 1))
71 or (isinstance(subject, URIRef) and (self.depth >= self.maxDepth))
72 ):
73 return False
74 return True
75
76 def isDone(self, subject: _SubjectType) -> bool:
77 """Return true if subject is serialized"""
78 return subject in self._serialized
79
80 def orderSubjects(self) -> List[_SubjectType]:
81 seen: Dict[_SubjectType, bool] = {}
82 subjects: List[_SubjectType] = []
83
84 for classURI in self.topClasses:
85 members = list(self.store.subjects(RDF.type, classURI))
86 # type error: All overload variants of "sort" of "list" require at least one argument
87 members.sort() # type: ignore[call-overload]
88
89 subjects.extend(members)
90 for member in members:
91 self._topLevels[member] = True
92 seen[member] = True
93
94 recursable = [
95 (isinstance(subject, BNode), self._references[subject], subject)
96 for subject in self._subjects
97 if subject not in seen
98 ]
99
100 recursable.sort()
101 subjects.extend([subject for (isbnode, refs, subject) in recursable])
102
103 return subjects
104
105 def preprocess(self) -> None:
106 for triple in self.store.triples((None, None, None)):
107 self.preprocessTriple(triple)
108
109 def preprocessTriple(self, spo: _TripleType) -> None:
110 s, p, o = spo
111 self._references[o] += 1
112 self._subjects[s] = True
113
114 def reset(self) -> None:
115 self.depth = 0
116 # Typed none because nothing is using it ...
117 self.lists: Dict[None, None] = {}
118 self.namespaces: Dict[str, URIRef] = {}
119 self._references: DefaultDict[Node, int] = defaultdict(int)
120 self._serialized: Dict[_SubjectType, bool] = {}
121 self._subjects: Dict[_SubjectType, bool] = {}
122 self._topLevels: Dict[_SubjectType, bool] = {}
123
124 if self.roundtrip_prefixes:
125 if hasattr(self.roundtrip_prefixes, "__iter__"):
126 for prefix, ns in self.store.namespaces():
127 if prefix in self.roundtrip_prefixes:
128 self.addNamespace(prefix, ns)
129 else:
130 for prefix, ns in self.store.namespaces():
131 self.addNamespace(prefix, ns)
132
133 def buildPredicateHash(
134 self, subject: _SubjectType
135 ) -> Mapping[_PredicateType, List[Node]]:
136 """
137 Build a hash key by predicate to a list of objects for the given
138 subject
139 """
140 properties: Dict[_PredicateType, List[Node]] = {}
141 for s, p, o in self.store.triples((subject, None, None)):
142 oList = properties.get(p, [])
143 oList.append(o)
144 properties[p] = oList
145 return properties
146
147 def sortProperties(
148 self, properties: Mapping[_PredicateType, List[Node]]
149 ) -> List[_PredicateType]:
150 """Take a hash from predicate uris to lists of values.
151 Sort the lists of values. Return a sorted list of properties."""
152 # Sort object lists
153 for prop, objects in properties.items():
154 # type error: All overload variants of "sort" of "list" require at least one argument
155 objects.sort() # type: ignore[call-overload]
156
157 # Make sorted list of properties
158 propList: List[_PredicateType] = []
159 seen: Dict[_PredicateType, bool] = {}
160 for prop in self.predicateOrder:
161 if (prop in properties) and (prop not in seen):
162 propList.append(prop)
163 seen[prop] = True
164 props = list(properties.keys())
165 # type error: All overload variants of "sort" of "list" require at least one argument
166 props.sort() # type: ignore[call-overload]
167 for prop in props:
168 if prop not in seen:
169 propList.append(prop)
170 seen[prop] = True
171 return propList
172
173 def subjectDone(self, subject: _SubjectType) -> None:
174 """Mark a subject as done."""
175 self._serialized[subject] = True
176
177 def indent(self, modifier: int = 0) -> str:
178 """Returns indent string multiplied by the depth"""
179 return (self.depth + modifier) * self.indentString
180
181 def write(self, text: str) -> None:
182 """Write text in given encoding."""
183 # type error: Item "None" of "Optional[IO[bytes]]" has no attribute "write"
184 self.stream.write(text.encode(self.encoding, "replace")) # type: ignore[union-attr]
185
186 def relativize(self, uri: _StrT) -> Union[_StrT, URIRef]:
187 base = self.base
188 if (
189 base is not None
190 and uri.startswith(base)
191 and "#" not in uri.replace(base, "")
192 and "/" not in uri.replace(base, "")
193 ):
194 # type error: Incompatible types in assignment (expression has type "str", variable has type "Node")
195 uri = URIRef(uri.replace(base, "", 1)) # type: ignore[assignment]
196 return uri
197
198
199SUBJECT = 0
200VERB = 1
201OBJECT = 2
202
203_GEN_QNAME_FOR_DT = False
204_SPACIOUS_OUTPUT = False
205
206
207class TurtleSerializer(RecursiveSerializer):
208 """Turtle RDF graph serializer."""
209
210 short_name = "turtle"
211 indentString = " "
212 LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX = re.compile(
213 r"%(?![0-9A-Fa-f]{2})"
214 )
215
216 def __init__(self, store: Graph):
217 self._ns_rewrite: Dict[str, str] = {}
218 super(TurtleSerializer, self).__init__(store)
219 self.keywords: Dict[Node, str] = {RDF.type: "a"}
220 self.reset()
221 self.stream = None
222 self._spacious = _SPACIOUS_OUTPUT
223
224 # type error: Return type "str" of "addNamespace" incompatible with return type "None" in supertype "RecursiveSerializer"
225 def addNamespace(self, prefix: str, namespace: URIRef) -> str: # type: ignore[override]
226 # Turtle does not support prefix that start with _
227 # if they occur in the graph, rewrite to p_blah
228 # this is more complicated since we need to make sure p_blah
229 # does not already exist. And we register namespaces as we go, i.e.
230 # we may first see a triple with prefix _9 - rewrite it to p_9
231 # and then later find a triple with a "real" p_9 prefix
232
233 # so we need to keep track of ns rewrites we made so far.
234
235 if (prefix > "" and prefix[0] == "_") or self.namespaces.get(
236 prefix, namespace
237 ) != namespace:
238 if prefix not in self._ns_rewrite:
239 p = "p" + prefix
240 while p in self.namespaces:
241 p = "p" + p
242 self._ns_rewrite[prefix] = p
243
244 prefix = self._ns_rewrite.get(prefix, prefix)
245
246 super(TurtleSerializer, self).addNamespace(prefix, namespace)
247 return prefix
248
249 def reset(self) -> None:
250 super(TurtleSerializer, self).reset()
251 # typing as Dict[None, None] because nothing seems to be using it
252 self._shortNames: Dict[None, None] = {}
253 self._started = False
254 self._ns_rewrite = {}
255
256 def serialize(
257 self,
258 stream: IO[bytes],
259 base: Optional[str] = None,
260 encoding: Optional[str] = None,
261 spacious: Optional[bool] = None,
262 **kwargs: Any,
263 ) -> None:
264 self.reset()
265 self.stream = stream
266 # if base is given here, use that, if not and a base is set for the graph use that
267 if base is not None:
268 self.base = base
269 elif self.store.base is not None:
270 self.base = self.store.base
271
272 if spacious is not None:
273 self._spacious = spacious
274
275 self.preprocess()
276 subjects_list = self.orderSubjects()
277
278 self.startDocument()
279
280 firstTime = True
281 for subject in subjects_list:
282 if self.isDone(subject):
283 continue
284 if firstTime:
285 firstTime = False
286 if self.statement(subject) and not firstTime:
287 self.write("\n")
288
289 self.endDocument()
290 stream.write("\n".encode("latin-1"))
291
292 self.base = None
293
294 def preprocessTriple(self, triple: _TripleType) -> None:
295 super(TurtleSerializer, self).preprocessTriple(triple)
296 for i, node in enumerate(triple):
297 if i == VERB:
298 if node in self.keywords:
299 # predicate is a keyword
300 continue
301 if (
302 self.base is not None
303 and isinstance(node, URIRef)
304 and node.startswith(self.base)
305 and "#" not in node.replace(self.base, "")
306 and "/" not in node.replace(self.base, "")
307 ):
308 # predicate corresponds to base namespace
309 continue
310 # Don't use generated prefixes for subjects and objects
311 self.get_pname(node, gen_prefix=(i == VERB))
312 if isinstance(node, Literal) and node.datatype:
313 self.get_pname(node.datatype, gen_prefix=_GEN_QNAME_FOR_DT)
314 p = triple[1]
315 if isinstance(p, BNode): # hmm - when is P ever a bnode?
316 self._references[p] += 1
317
318 # Refer to Productions for terminals PNAME_NS and PNAME_LN https://www.w3.org/TR/turtle/#sec-grammar-grammar
319 def get_pname(self, uri: Node, gen_prefix: bool = True) -> Optional[str]:
320 if not isinstance(uri, URIRef):
321 return None
322
323 parts = None
324
325 try:
326 parts = self.store.compute_qname(uri, generate=gen_prefix)
327 except Exception:
328 # is the uri a namespace in itself?
329 pfx = self.store.store.prefix(uri)
330
331 if pfx is not None:
332 parts = (pfx, uri, "")
333 else:
334 # nothing worked
335 return None
336
337 prefix, namespace, local = parts
338
339 # To understand treatment of % character refer to Productions for terminal PLX at
340 # https://www.w3.org/TR/turtle/#grammar-production-PLX
341 # Only % NOT followed by two hex chars requires manual backslash escaping
342 local = local.replace(r"(", r"\(").replace(r")", r"\)")
343 local = self.LOCALNAME_PECRENT_CHARACTER_REQUIRING_ESCAPE_REGEX.sub(
344 "\\%", local
345 )
346
347 # PName cannot end with .
348 if local.endswith("."):
349 return None
350
351 prefix = self.addNamespace(prefix, namespace)
352
353 return "%s:%s" % (prefix, local)
354
355 def getQName(self, uri: Node, gen_prefix: bool = True) -> Optional[str]:
356 warnings.warn(
357 "TurtleSerializer.getQName is deprecated, use TurtleSerializer.get_pname instead.",
358 DeprecationWarning,
359 stacklevel=2,
360 )
361 return self.get_pname(uri, gen_prefix)
362
363 def startDocument(self) -> None:
364 self._started = True
365 ns_list = sorted(self.namespaces.items())
366
367 if self.base:
368 self.write(self.indent() + "@base <%s> .\n" % self.base)
369 for prefix, uri in ns_list:
370 self.write(self.indent() + "@prefix %s: <%s> .\n" % (prefix, uri))
371 if ns_list and self._spacious:
372 self.write("\n")
373
374 def endDocument(self) -> None:
375 if self._spacious:
376 self.write("\n")
377
378 def statement(self, subject: _SubjectType) -> bool:
379 self.subjectDone(subject)
380 return self.s_squared(subject) or self.s_default(subject)
381
382 def s_default(self, subject: _SubjectType) -> bool:
383 self.write("\n" + self.indent())
384 self.path(subject, SUBJECT)
385 self.predicateList(subject)
386 self.write(" .")
387 return True
388
389 def s_squared(self, subject: _SubjectType) -> bool:
390 if (self._references[subject] > 0) or not isinstance(subject, BNode):
391 return False
392 self.write("\n" + self.indent() + "[]")
393 self.predicateList(subject)
394 self.write(" .")
395 return True
396
397 def path(self, node: Node, position: int, newline: bool = False) -> None:
398 if not (
399 self.p_squared(node, position, newline)
400 or self.p_default(node, position, newline)
401 ):
402 raise Error("Cannot serialize node '%s'" % (node,))
403
404 def p_default(self, node: Node, position: int, newline: bool = False) -> bool:
405 if position != SUBJECT and not newline:
406 self.write(" ")
407 self.write(self.label(node, position))
408 return True
409
410 def label(self, node: Node, position: int) -> str:
411 if node == RDF.nil:
412 return "()"
413 if position is VERB and node in self.keywords:
414 return self.keywords[node]
415 if isinstance(node, Literal):
416 return node._literal_n3(
417 use_plain=True,
418 qname_callback=lambda dt: self.get_pname(dt, _GEN_QNAME_FOR_DT),
419 )
420 else:
421 node = self.relativize(node) # type: ignore[type-var]
422
423 return self.get_pname(node, position == VERB) or node.n3()
424
425 def p_squared(self, node: Node, position: int, newline: bool = False) -> bool:
426 if (
427 not isinstance(node, BNode)
428 or node in self._serialized
429 or self._references[node] > 1
430 or position == SUBJECT
431 ):
432 return False
433
434 if not newline:
435 self.write(" ")
436
437 if self.isValidList(node):
438 # this is a list
439 self.write("(")
440 self.depth += 1 # 2
441 self.doList(node)
442 self.depth -= 1 # 2
443 self.write(" )")
444 else:
445 self.subjectDone(node)
446 self.depth += 2
447 # self.write('[\n' + self.indent())
448 self.write("[")
449 self.depth -= 1
450 # self.predicateList(node, newline=True)
451 self.predicateList(node, newline=False)
452 # self.write('\n' + self.indent() + ']')
453 self.write(" ]")
454 self.depth -= 1
455
456 return True
457
458 def isValidList(self, l_: Node) -> bool:
459 """
460 Checks if l is a valid RDF list, i.e. no nodes have other properties.
461 """
462 try:
463 if self.store.value(l_, RDF.first) is None:
464 return False
465 except Exception:
466 return False
467 while l_:
468 if l_ != RDF.nil and len(list(self.store.predicate_objects(l_))) != 2:
469 return False
470 # type error: Incompatible types in assignment (expression has type "Optional[Node]", variable has type "Node")
471 l_ = self.store.value(l_, RDF.rest) # type: ignore[assignment]
472 return True
473
474 def doList(self, l_: Node) -> None:
475 while l_:
476 item = self.store.value(l_, RDF.first)
477 if item is not None:
478 self.path(item, OBJECT)
479 self.subjectDone(l_)
480 # type error: Incompatible types in assignment (expression has type "Optional[Node]", variable has type "Node")
481 l_ = self.store.value(l_, RDF.rest) # type: ignore[assignment]
482
483 def predicateList(self, subject: Node, newline: bool = False) -> None:
484 properties = self.buildPredicateHash(subject)
485 propList = self.sortProperties(properties)
486 if len(propList) == 0:
487 return
488 self.verb(propList[0], newline=newline)
489 self.objectList(properties[propList[0]])
490 for predicate in propList[1:]:
491 self.write(" ;\n" + self.indent(1))
492 self.verb(predicate, newline=True)
493 self.objectList(properties[predicate])
494
495 def verb(self, node: Node, newline: bool = False) -> None:
496 self.path(node, VERB, newline)
497
498 def objectList(self, objects: Sequence[Node]) -> None:
499 count = len(objects)
500 if count == 0:
501 return
502 depthmod = (count == 1) and 0 or 1
503 self.depth += depthmod
504 self.path(objects[0], OBJECT)
505 for obj in objects[1:]:
506 self.write(",\n" + self.indent(1))
507 self.path(obj, OBJECT, newline=True)
508 self.depth -= depthmod