src/rdflib/util.py
1"""
2Some utility functions.
3
4Miscellaneous utilities
5
6* list2set
7* first
8* uniq
9* more_than
10
11Term characterisation and generation
12
13* to_term
14* from_n3
15
16Date/time utilities
17
18* date_time
19* parse_date_time
20
21"""
22
23from __future__ import annotations
24
25from calendar import timegm
26from os.path import splitext
27
28# from time import daylight
29from time import altzone, gmtime, localtime, time, timezone
30from typing import (
31 TYPE_CHECKING,
32 Any,
33 Callable,
34 Dict,
35 Hashable,
36 Iterable,
37 Iterator,
38 List,
39 Optional,
40 Set,
41 Tuple,
42 TypeVar,
43 Union,
44 overload,
45)
46from urllib.parse import quote, urlsplit, urlunsplit
47
48import rdflib.graph # avoid circular dependency
49import rdflib.namespace
50import rdflib.term
51from rdflib.compat import sign
52
53if TYPE_CHECKING:
54 from rdflib.graph import Graph
55
56
57__all__ = [
58 "list2set",
59 "first",
60 "uniq",
61 "more_than",
62 "to_term",
63 "from_n3",
64 "date_time",
65 "parse_date_time",
66 "guess_format",
67 "find_roots",
68 "get_tree",
69 "_coalesce",
70 "_iri2uri",
71]
72
73_HashableT = TypeVar("_HashableT", bound=Hashable)
74_AnyT = TypeVar("_AnyT")
75
76
77SUFFIX_FORMAT_MAP = {
78 "xml": "xml",
79 "rdf": "xml",
80 "owl": "xml",
81 "n3": "n3",
82 "ttl": "turtle",
83 "nt": "nt",
84 "trix": "trix",
85 "xhtml": "rdfa",
86 "html": "rdfa",
87 "svg": "rdfa",
88 "nq": "nquads",
89 "nquads": "nquads",
90 "trig": "trig",
91 "json": "json-ld",
92 "jsonld": "json-ld",
93 "json-ld": "json-ld",
94}
95
96
97FORMAT_MIMETYPE_MAP = {
98 "xml": ["application/rdf+xml"],
99 "n3": ["text/n3"],
100 "turtle": ["text/turtle"],
101 "nt": ["application/n-triples"],
102 "trix": ["application/trix"],
103 "rdfa": ["text/html", "application/xhtml+xml"],
104 "nquads": ["application/n-quads"],
105 "trig": ["application/trig"],
106 "json-ld": ["application/ld+json"],
107}
108
109
110RESPONSE_TABLE_FORMAT_MIMETYPE_MAP = {
111 "xml": ["application/sparql-results+xml"],
112 "json": ["application/sparql-results+json"],
113 "csv": ["text/csv"],
114 "tsv": ["text/tab-separated-values"],
115}
116
117
118def list2set(seq: Iterable[_HashableT]) -> List[_HashableT]:
119 """
120 Return a new list without duplicates.
121 Preserves the order, unlike set(seq)
122 """
123 seen = set()
124 # type error: "add" of "set" does not return a value
125 return [x for x in seq if x not in seen and not seen.add(x)] # type: ignore[func-returns-value]
126
127
128def first(seq: Iterable[_AnyT]) -> Optional[_AnyT]:
129 """
130 return the first element in a python sequence
131 for graphs, use graph.value instead
132 """
133 for result in seq:
134 return result
135 return None
136
137
138def uniq(sequence: Iterable[str], strip: int = 0) -> Set[str]:
139 """removes duplicate strings from the sequence."""
140 if strip:
141 return set(s.strip() for s in sequence)
142 else:
143 return set(sequence)
144
145
146def more_than(sequence: Iterable[Any], number: int) -> int:
147 "Returns 1 if sequence has more items than number and 0 if not."
148 i = 0
149 for item in sequence:
150 i += 1
151 if i > number:
152 return 1
153 return 0
154
155
156def to_term(
157 s: Optional[str], default: Optional[rdflib.term.Identifier] = None
158) -> Optional[rdflib.term.Identifier]:
159 """
160 Creates and returns an Identifier of type corresponding
161 to the pattern of the given positional argument string `s`:
162
163 '' returns the `default` keyword argument value or `None`
164
165 '<s>' returns `URIRef(s)` (i.e. without angle brackets)
166
167 '"s"' returns `Literal(s)` (i.e. without doublequotes)
168
169 '_s' returns `BNode(s)` (i.e. without leading underscore)
170
171 """
172 if not s:
173 return default
174 elif s.startswith("<") and s.endswith(">"):
175 return rdflib.term.URIRef(s[1:-1])
176 elif s.startswith('"') and s.endswith('"'):
177 return rdflib.term.Literal(s[1:-1])
178 elif s.startswith("_"):
179 return rdflib.term.BNode(s)
180 else:
181 msg = "Unrecognised term syntax: '%s'" % s
182 raise Exception(msg)
183
184
185def from_n3(
186 s: str,
187 default: Optional[str] = None,
188 backend: Optional[str] = None,
189 nsm: Optional[rdflib.namespace.NamespaceManager] = None,
190) -> Optional[Union[rdflib.term.Node, str]]:
191 r'''Creates the Identifier corresponding to the given n3 string.
192
193 ```python
194 >>> from rdflib.term import URIRef, Literal
195 >>> from rdflib.namespace import NamespaceManager
196 >>> from_n3('<http://ex.com/foo>') == URIRef('http://ex.com/foo')
197 True
198 >>> from_n3('"foo"@de') == Literal('foo', lang='de')
199 True
200 >>> from_n3('"""multi\nline\nstring"""@en') == Literal(
201 ... 'multi\nline\nstring', lang='en')
202 True
203 >>> from_n3('42') == Literal(42)
204 True
205 >>> from_n3(Literal(42).n3()) == Literal(42)
206 True
207 >>> from_n3('"42"^^xsd:integer') == Literal(42)
208 True
209 >>> from rdflib import RDFS
210 >>> from_n3('rdfs:label') == RDFS['label']
211 True
212 >>> nsm = NamespaceManager(rdflib.graph.Graph())
213 >>> nsm.bind('dbpedia', 'http://dbpedia.org/resource/')
214 >>> berlin = URIRef('http://dbpedia.org/resource/Berlin')
215 >>> from_n3('dbpedia:Berlin', nsm=nsm) == berlin
216 True
217
218 ```
219 '''
220 if not s:
221 return default
222 if s.startswith("<"):
223 # Hack: this should correctly handle strings with either native unicode
224 # characters, or \u1234 unicode escapes.
225 return rdflib.term.URIRef(
226 s[1:-1].encode("raw-unicode-escape").decode("unicode-escape")
227 )
228 elif s.startswith('"'):
229 if s.startswith('"""'):
230 quotes = '"""'
231 else:
232 quotes = '"'
233 value, rest = s.rsplit(quotes, 1)
234 value = value[len(quotes) :] # strip leading quotes
235 datatype = None
236 language = None
237
238 # as a given datatype overrules lang-tag check for it first
239 dtoffset = rest.rfind("^^")
240 if dtoffset >= 0:
241 # found a datatype
242 # datatype has to come after lang-tag so ignore everything before
243 # see: http://www.w3.org/TR/2011/WD-turtle-20110809/
244 # #prod-turtle2-RDFLiteral
245 datatype = from_n3(rest[dtoffset + 2 :], default, backend, nsm)
246 else:
247 if rest.startswith("@"):
248 language = rest[1:] # strip leading at sign
249
250 value = value.replace(r"\"", '"')
251 # unicode-escape interprets \xhh as an escape sequence,
252 # but n3 does not define it as such.
253 value = value.replace(r"\x", r"\\x")
254 # Hack: this should correctly handle strings with either native unicode
255 # characters, or \u1234 unicode escapes.
256 value = value.encode("raw-unicode-escape").decode("unicode-escape")
257 # type error: Argument 3 to "Literal" has incompatible type "Union[Node, str, None]"; expected "Optional[str]"
258 return rdflib.term.Literal(value, language, datatype) # type: ignore[arg-type]
259 elif s == "true" or s == "false":
260 return rdflib.term.Literal(s == "true")
261 elif (
262 s.lower()
263 .replace(".", "", 1)
264 .replace("-", "", 1)
265 .replace("e", "", 1)
266 .isnumeric()
267 ):
268 if "e" in s.lower():
269 return rdflib.term.Literal(s, datatype=rdflib.namespace.XSD.double)
270 if "." in s:
271 return rdflib.term.Literal(float(s), datatype=rdflib.namespace.XSD.decimal)
272 return rdflib.term.Literal(int(s), datatype=rdflib.namespace.XSD.integer)
273
274 elif s.startswith("{"):
275 identifier = from_n3(s[1:-1])
276 # type error: Argument 1 to "QuotedGraph" has incompatible type "Optional[str]"; expected "Union[Store, str]"
277 # type error: Argument 2 to "QuotedGraph" has incompatible type "Union[Node, str, None]"; expected "Union[IdentifiedNode, str, None]"
278 return rdflib.graph.QuotedGraph(backend, identifier) # type: ignore[arg-type]
279 elif s.startswith("["):
280 identifier = from_n3(s[1:-1])
281 # type error: Argument 1 to "Graph" has incompatible type "Optional[str]"; expected "Union[Store, str]"
282 # type error: Argument 2 to "Graph" has incompatible type "Union[Node, str, None]"; expected "Union[IdentifiedNode, str, None]"
283 return rdflib.graph.Graph(backend, identifier) # type: ignore[arg-type]
284 elif s.startswith("_:"):
285 return rdflib.term.BNode(s[2:])
286 elif ":" in s:
287 if nsm is None:
288 # instantiate default NamespaceManager and rely on its defaults
289 nsm = rdflib.namespace.NamespaceManager(rdflib.graph.Graph())
290 prefix, last_part = s.split(":", 1)
291 ns = dict(nsm.namespaces())[prefix]
292 return rdflib.namespace.Namespace(ns)[last_part]
293 else:
294 return rdflib.term.BNode(s)
295
296
297def date_time(t=None, local_time_zone=False):
298 """http://www.w3.org/TR/NOTE-datetime ex: 1997-07-16T19:20:30Z
299
300 ```python
301 >>> date_time(1126482850)
302 '2005-09-11T23:54:10Z'
303
304 @@ this will change depending on where it is run
305 #>>> date_time(1126482850, local_time_zone=True)
306 #'2005-09-11T19:54:10-04:00'
307
308 >>> date_time(1)
309 '1970-01-01T00:00:01Z'
310
311 >>> date_time(0)
312 '1970-01-01T00:00:00Z'
313
314 ```
315 """
316 if t is None:
317 t = time()
318
319 if local_time_zone:
320 time_tuple = localtime(t)
321 if time_tuple[8]:
322 tz_mins = altzone // 60
323 else:
324 tz_mins = timezone // 60
325 tzd = "-%02d:%02d" % (tz_mins // 60, tz_mins % 60)
326 else:
327 time_tuple = gmtime(t)
328 tzd = "Z"
329
330 year, month, day, hh, mm, ss, wd, y, z = time_tuple
331 s = "%0004d-%02d-%02dT%02d:%02d:%02d%s" % (year, month, day, hh, mm, ss, tzd)
332 return s
333
334
335def parse_date_time(val: str) -> int:
336 """always returns seconds in UTC
337
338 ```python
339 # tests are written like this to make any errors easier to understand
340 >>> parse_date_time('2005-09-11T23:54:10Z') - 1126482850.0
341 0.0
342
343 >>> parse_date_time('2005-09-11T16:54:10-07:00') - 1126482850.0
344 0.0
345
346 >>> parse_date_time('1970-01-01T00:00:01Z') - 1.0
347 0.0
348
349 >>> parse_date_time('1970-01-01T00:00:00Z') - 0.0
350 0.0
351 >>> parse_date_time("2005-09-05T10:42:00") - 1125916920.0
352 0.0
353
354 ```
355 """
356
357 if "T" not in val:
358 val += "T00:00:00Z"
359
360 ymd, time = val.split("T")
361 hms, tz_str = time[0:8], time[8:]
362
363 if not tz_str or tz_str == "Z":
364 time = time[:-1]
365 tz_offset = 0
366 else:
367 signed_hrs = int(tz_str[:3])
368 mins = int(tz_str[4:6])
369 secs = (sign(signed_hrs) * mins + signed_hrs * 60) * 60
370 tz_offset = -secs
371
372 year, month, day = ymd.split("-")
373 hour, minute, second = hms.split(":")
374
375 t = timegm(
376 (int(year), int(month), int(day), int(hour), int(minute), int(second), 0, 0, 0)
377 )
378 t = t + tz_offset
379 return t
380
381
382def guess_format(fpath: str, fmap: Optional[Dict[str, str]] = None) -> Optional[str]:
383 """
384 Guess RDF serialization based on file suffix. Uses
385 `SUFFIX_FORMAT_MAP` unless `fmap` is provided.
386
387 Example:
388 ```python
389 >>> guess_format('path/to/file.rdf')
390 'xml'
391 >>> guess_format('path/to/file.owl')
392 'xml'
393 >>> guess_format('path/to/file.ttl')
394 'turtle'
395 >>> guess_format('path/to/file.json')
396 'json-ld'
397 >>> guess_format('path/to/file.xhtml')
398 'rdfa'
399 >>> guess_format('path/to/file.svg')
400 'rdfa'
401 >>> guess_format('path/to/file.xhtml', {'xhtml': 'grddl'})
402 'grddl'
403
404 ```
405
406 This also works with just the suffixes, with or without leading dot, and
407 regardless of letter case:
408
409 ```python
410 >>> guess_format('.rdf')
411 'xml'
412 >>> guess_format('rdf')
413 'xml'
414 >>> guess_format('RDF')
415 'xml'
416
417 ```
418 """
419 fmap = fmap or SUFFIX_FORMAT_MAP
420 return fmap.get(_get_ext(fpath)) or fmap.get(fpath.lower())
421
422
423def _get_ext(fpath: str, lower: bool = True) -> str:
424 """
425 Gets the file extension from a file(path); stripped of leading '.' and in
426 lower case.
427
428 Example:
429 ```python
430 >>> _get_ext("path/to/file.txt")
431 'txt'
432 >>> _get_ext("OTHER.PDF")
433 'pdf'
434 >>> _get_ext("noext")
435 ''
436 >>> _get_ext(".rdf")
437 'rdf'
438
439 ```
440 """
441 ext = splitext(fpath)[-1]
442 if ext == "" and fpath.startswith("."):
443 ext = fpath
444 if lower:
445 ext = ext.lower()
446 if ext.startswith("."):
447 ext = ext[1:]
448 return ext
449
450
451def find_roots(
452 graph: Graph,
453 prop: rdflib.term.URIRef,
454 roots: Optional[Set[rdflib.term.Node]] = None,
455) -> Set[rdflib.term.Node]:
456 """Find the roots in some sort of transitive hierarchy.
457
458 find_roots(graph, rdflib.RDFS.subClassOf)
459 will return a set of all roots of the sub-class hierarchy
460
461 Assumes triple of the form (child, prop, parent), i.e. the direction of
462 `RDFS.subClassOf` or `SKOS.broader`
463 """
464
465 non_roots: Set[rdflib.term.Node] = set()
466 if roots is None:
467 roots = set()
468 for x, y in graph.subject_objects(prop):
469 non_roots.add(x)
470 if x in roots:
471 roots.remove(x)
472 if y not in non_roots:
473 roots.add(y)
474 return roots
475
476
477def get_tree(
478 graph: Graph,
479 root: rdflib.term.Node,
480 prop: rdflib.term.URIRef,
481 mapper: Callable[[rdflib.term.Node], rdflib.term.Node] = lambda x: x,
482 sortkey: Optional[Callable[[Any], Any]] = None,
483 done: Optional[Set[rdflib.term.Node]] = None,
484 dir: str = "down",
485) -> Optional[Tuple[rdflib.term.Node, List[Any]]]:
486 """
487 Return a nested list/tuple structure representing the tree
488 built by the transitive property given, starting from the root given
489
490 i.e.
491
492 ```python
493 get_tree(
494 graph,
495 rdflib.URIRef("http://xmlns.com/foaf/0.1/Person"),
496 rdflib.RDFS.subClassOf,
497 )
498 ```
499
500 will return the structure for the subClassTree below person.
501
502 dir='down' assumes triple of the form (child, prop, parent),
503 i.e. the direction of RDFS.subClassOf or SKOS.broader
504 Any other dir traverses in the other direction
505 """
506
507 if done is None:
508 done = set()
509 if root in done:
510 # type error: Return value expected
511 return # type: ignore[return-value]
512 done.add(root)
513 tree = []
514
515 branches: Iterator[rdflib.term.Node]
516 if dir == "down":
517 branches = graph.subjects(prop, root)
518 else:
519 branches = graph.objects(root, prop)
520
521 for branch in branches:
522 t = get_tree(graph, branch, prop, mapper, sortkey, done, dir)
523 if t:
524 tree.append(t)
525
526 return (mapper(root), sorted(tree, key=sortkey))
527
528
529@overload
530def _coalesce(*args: Optional[_AnyT], default: _AnyT) -> _AnyT: ...
531
532
533@overload
534def _coalesce(
535 *args: Optional[_AnyT], default: Optional[_AnyT] = ...
536) -> Optional[_AnyT]: ...
537
538
539def _coalesce(
540 *args: Optional[_AnyT], default: Optional[_AnyT] = None
541) -> Optional[_AnyT]:
542 """
543 This is a null coalescing function, it will return the first non-`None`
544 argument passed to it, otherwise it will return `default` which is `None`
545 by default.
546
547 For more info regarding the rationale of this function see deferred
548 [PEP 505](https://peps.python.org/pep-0505/).
549
550 Args:
551 *args: Values to consider as candidates to return, the first arg that
552 is not `None` will be returned. If no argument is passed this function
553 will return None.
554 default: The default value to return if none of the args are not `None`.
555
556 Returns:
557 The first `args` that is not `None`, otherwise the value of
558 `default` if there are no `args` or if all `args` are `None`.
559 """
560 for arg in args:
561 if arg is not None:
562 return arg
563 return default
564
565
566_RFC3986_SUBDELIMS = "!$&'()*+,;="
567"""
568`sub-delims` production from
569[RFC 3986, section 2.2](https://www.rfc-editor.org/rfc/rfc3986.html#section-2.2).
570"""
571
572_RFC3986_PCHAR_NU = "%" + _RFC3986_SUBDELIMS + ":@"
573"""
574The non-unreserved characters in the `pchar` production from RFC 3986.
575"""
576
577_QUERY_SAFE_CHARS = _RFC3986_PCHAR_NU + "/?"
578"""
579The non-unreserved characters that are safe to use in in the query and fragment
580components.
581
582```
583pchar = unreserved / pct-encoded / sub-delims / ":" / "@" query
584= *( pchar / "/" / "?" ) fragment = *( pchar / "/" / "?" )
585```
586"""
587
588_USERNAME_SAFE_CHARS = _RFC3986_SUBDELIMS + "%"
589"""
590The non-unreserved characters that are safe to use in the username and password
591components.
592
593```
594userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
595```
596
597":" is excluded as this is only used for the username and password components,
598and they are treated separately.
599"""
600
601_PATH_SAFE_CHARS = _RFC3986_PCHAR_NU + "/"
602"""
603The non-unreserved characters that are safe to use in the path component.
604
605This is based on various path-related productions from RFC 3986.
606"""
607
608
609def _iri2uri(iri: str) -> str:
610 """
611 Prior art:
612
613 - [iri_to_uri from Werkzeug](https://github.com/pallets/werkzeug/blob/92c6380248c7272ee668e1f8bbd80447027ccce2/src/werkzeug/urls.py#L926-L931)
614
615 ```python
616 >>> _iri2uri("https://dbpedia.org/resource/Almería")
617 'https://dbpedia.org/resource/Almer%C3%ADa'
618
619 ```
620 """
621 # https://datatracker.ietf.org/doc/html/rfc3986
622 # https://datatracker.ietf.org/doc/html/rfc3305
623
624 parts = urlsplit(iri)
625 (scheme, netloc, path, query, fragment) = parts
626
627 # Just support http/https, otherwise return the iri unaltered
628 if scheme not in ["http", "https"]:
629 return iri
630
631 path = quote(path, safe=_PATH_SAFE_CHARS)
632 query = quote(query, safe=_QUERY_SAFE_CHARS)
633 fragment = quote(fragment, safe=_QUERY_SAFE_CHARS)
634
635 if parts.hostname:
636 netloc = parts.hostname.encode("idna").decode("ascii")
637 else:
638 netloc = ""
639
640 if ":" in netloc:
641 # Quote IPv6 addresses
642 netloc = f"[{netloc}]"
643
644 if parts.port:
645 netloc = f"{netloc}:{parts.port}"
646
647 if parts.username:
648 auth = quote(parts.username, safe=_USERNAME_SAFE_CHARS)
649 if parts.password:
650 pass_quoted = quote(parts.password, safe=_USERNAME_SAFE_CHARS)
651 auth = f"{auth}:{pass_quoted}"
652 netloc = f"{auth}@{netloc}"
653
654 uri = urlunsplit((scheme, netloc, path, query, fragment))
655
656 if iri.endswith("#") and not uri.endswith("#"):
657 uri += "#"
658
659 return uri