Source code

Download source archive ↓

src/rdflib/util.py

1"""
2Some utility functions.
3
4Miscellaneous utilities
5
6* list2set
7* first
8* uniq
9* more_than
10
11Term characterisation and generation
12
13* to_term
14* from_n3
15
16Date/time utilities
17
18* date_time
19* parse_date_time
20
21"""
22
23from __future__ import annotations
24
25from calendar import timegm
26from os.path import splitext
27
28# from time import daylight
29from time import altzone, gmtime, localtime, time, timezone
30from typing import (
31    TYPE_CHECKING,
32    Any,
33    Callable,
34    Dict,
35    Hashable,
36    Iterable,
37    Iterator,
38    List,
39    Optional,
40    Set,
41    Tuple,
42    TypeVar,
43    Union,
44    overload,
45)
46from urllib.parse import quote, urlsplit, urlunsplit
47
48import rdflib.graph  # avoid circular dependency
49import rdflib.namespace
50import rdflib.term
51from rdflib.compat import sign
52
53if TYPE_CHECKING:
54    from rdflib.graph import Graph
55
56
57__all__ = [
58    "list2set",
59    "first",
60    "uniq",
61    "more_than",
62    "to_term",
63    "from_n3",
64    "date_time",
65    "parse_date_time",
66    "guess_format",
67    "find_roots",
68    "get_tree",
69    "_coalesce",
70    "_iri2uri",
71]
72
73_HashableT = TypeVar("_HashableT", bound=Hashable)
74_AnyT = TypeVar("_AnyT")
75
76
77SUFFIX_FORMAT_MAP = {
78    "xml": "xml",
79    "rdf": "xml",
80    "owl": "xml",
81    "n3": "n3",
82    "ttl": "turtle",
83    "nt": "nt",
84    "trix": "trix",
85    "xhtml": "rdfa",
86    "html": "rdfa",
87    "svg": "rdfa",
88    "nq": "nquads",
89    "nquads": "nquads",
90    "trig": "trig",
91    "json": "json-ld",
92    "jsonld": "json-ld",
93    "json-ld": "json-ld",
94}
95
96
97FORMAT_MIMETYPE_MAP = {
98    "xml": ["application/rdf+xml"],
99    "n3": ["text/n3"],
100    "turtle": ["text/turtle"],
101    "nt": ["application/n-triples"],
102    "trix": ["application/trix"],
103    "rdfa": ["text/html", "application/xhtml+xml"],
104    "nquads": ["application/n-quads"],
105    "trig": ["application/trig"],
106    "json-ld": ["application/ld+json"],
107}
108
109
110RESPONSE_TABLE_FORMAT_MIMETYPE_MAP = {
111    "xml": ["application/sparql-results+xml"],
112    "json": ["application/sparql-results+json"],
113    "csv": ["text/csv"],
114    "tsv": ["text/tab-separated-values"],
115}
116
117
118def list2set(seq: Iterable[_HashableT]) -> List[_HashableT]:
119    """
120    Return a new list without duplicates.
121    Preserves the order, unlike set(seq)
122    """
123    seen = set()
124    # type error: "add" of "set" does not return a value
125    return [x for x in seq if x not in seen and not seen.add(x)]  # type: ignore[func-returns-value]
126
127
128def first(seq: Iterable[_AnyT]) -> Optional[_AnyT]:
129    """
130    return the first element in a python sequence
131    for graphs, use graph.value instead
132    """
133    for result in seq:
134        return result
135    return None
136
137
138def uniq(sequence: Iterable[str], strip: int = 0) -> Set[str]:
139    """removes duplicate strings from the sequence."""
140    if strip:
141        return set(s.strip() for s in sequence)
142    else:
143        return set(sequence)
144
145
146def more_than(sequence: Iterable[Any], number: int) -> int:
147    "Returns 1 if sequence has more items than number and 0 if not."
148    i = 0
149    for item in sequence:
150        i += 1
151        if i > number:
152            return 1
153    return 0
154
155
156def to_term(
157    s: Optional[str], default: Optional[rdflib.term.Identifier] = None
158) -> Optional[rdflib.term.Identifier]:
159    """
160    Creates and returns an Identifier of type corresponding
161    to the pattern of the given positional argument string `s`:
162
163    '' returns the `default` keyword argument value or `None`
164
165    '<s>' returns `URIRef(s)` (i.e. without angle brackets)
166
167    '"s"' returns `Literal(s)` (i.e. without doublequotes)
168
169    '_s' returns `BNode(s)` (i.e. without leading underscore)
170
171    """
172    if not s:
173        return default
174    elif s.startswith("<") and s.endswith(">"):
175        return rdflib.term.URIRef(s[1:-1])
176    elif s.startswith('"') and s.endswith('"'):
177        return rdflib.term.Literal(s[1:-1])
178    elif s.startswith("_"):
179        return rdflib.term.BNode(s)
180    else:
181        msg = "Unrecognised term syntax: '%s'" % s
182        raise Exception(msg)
183
184
185def from_n3(
186    s: str,
187    default: Optional[str] = None,
188    backend: Optional[str] = None,
189    nsm: Optional[rdflib.namespace.NamespaceManager] = None,
190) -> Optional[Union[rdflib.term.Node, str]]:
191    r'''Creates the Identifier corresponding to the given n3 string.
192
193    ```python
194    >>> from rdflib.term import URIRef, Literal
195    >>> from rdflib.namespace import NamespaceManager
196    >>> from_n3('<http://ex.com/foo>') == URIRef('http://ex.com/foo')
197    True
198    >>> from_n3('"foo"@de') == Literal('foo', lang='de')
199    True
200    >>> from_n3('"""multi\nline\nstring"""@en') == Literal(
201    ...     'multi\nline\nstring', lang='en')
202    True
203    >>> from_n3('42') == Literal(42)
204    True
205    >>> from_n3(Literal(42).n3()) == Literal(42)
206    True
207    >>> from_n3('"42"^^xsd:integer') == Literal(42)
208    True
209    >>> from rdflib import RDFS
210    >>> from_n3('rdfs:label') == RDFS['label']
211    True
212    >>> nsm = NamespaceManager(rdflib.graph.Graph())
213    >>> nsm.bind('dbpedia', 'http://dbpedia.org/resource/')
214    >>> berlin = URIRef('http://dbpedia.org/resource/Berlin')
215    >>> from_n3('dbpedia:Berlin', nsm=nsm) == berlin
216    True
217
218    ```
219    '''
220    if not s:
221        return default
222    if s.startswith("<"):
223        # Hack: this should correctly handle strings with either native unicode
224        # characters, or \u1234 unicode escapes.
225        return rdflib.term.URIRef(
226            s[1:-1].encode("raw-unicode-escape").decode("unicode-escape")
227        )
228    elif s.startswith('"'):
229        if s.startswith('"""'):
230            quotes = '"""'
231        else:
232            quotes = '"'
233        value, rest = s.rsplit(quotes, 1)
234        value = value[len(quotes) :]  # strip leading quotes
235        datatype = None
236        language = None
237
238        # as a given datatype overrules lang-tag check for it first
239        dtoffset = rest.rfind("^^")
240        if dtoffset >= 0:
241            # found a datatype
242            # datatype has to come after lang-tag so ignore everything before
243            # see: http://www.w3.org/TR/2011/WD-turtle-20110809/
244            # #prod-turtle2-RDFLiteral
245            datatype = from_n3(rest[dtoffset + 2 :], default, backend, nsm)
246        else:
247            if rest.startswith("@"):
248                language = rest[1:]  # strip leading at sign
249
250        value = value.replace(r"\"", '"')
251        # unicode-escape interprets \xhh as an escape sequence,
252        # but n3 does not define it as such.
253        value = value.replace(r"\x", r"\\x")
254        # Hack: this should correctly handle strings with either native unicode
255        # characters, or \u1234 unicode escapes.
256        value = value.encode("raw-unicode-escape").decode("unicode-escape")
257        # type error: Argument 3 to "Literal" has incompatible type "Union[Node, str, None]"; expected "Optional[str]"
258        return rdflib.term.Literal(value, language, datatype)  # type: ignore[arg-type]
259    elif s == "true" or s == "false":
260        return rdflib.term.Literal(s == "true")
261    elif (
262        s.lower()
263        .replace(".", "", 1)
264        .replace("-", "", 1)
265        .replace("e", "", 1)
266        .isnumeric()
267    ):
268        if "e" in s.lower():
269            return rdflib.term.Literal(s, datatype=rdflib.namespace.XSD.double)
270        if "." in s:
271            return rdflib.term.Literal(float(s), datatype=rdflib.namespace.XSD.decimal)
272        return rdflib.term.Literal(int(s), datatype=rdflib.namespace.XSD.integer)
273
274    elif s.startswith("{"):
275        identifier = from_n3(s[1:-1])
276        # type error: Argument 1 to "QuotedGraph" has incompatible type "Optional[str]"; expected "Union[Store, str]"
277        # type error: Argument 2 to "QuotedGraph" has incompatible type "Union[Node, str, None]"; expected "Union[IdentifiedNode, str, None]"
278        return rdflib.graph.QuotedGraph(backend, identifier)  # type: ignore[arg-type]
279    elif s.startswith("["):
280        identifier = from_n3(s[1:-1])
281        # type error: Argument 1 to "Graph" has incompatible type "Optional[str]"; expected "Union[Store, str]"
282        # type error: Argument 2 to "Graph" has incompatible type "Union[Node, str, None]"; expected "Union[IdentifiedNode, str, None]"
283        return rdflib.graph.Graph(backend, identifier)  # type: ignore[arg-type]
284    elif s.startswith("_:"):
285        return rdflib.term.BNode(s[2:])
286    elif ":" in s:
287        if nsm is None:
288            # instantiate default NamespaceManager and rely on its defaults
289            nsm = rdflib.namespace.NamespaceManager(rdflib.graph.Graph())
290        prefix, last_part = s.split(":", 1)
291        ns = dict(nsm.namespaces())[prefix]
292        return rdflib.namespace.Namespace(ns)[last_part]
293    else:
294        return rdflib.term.BNode(s)
295
296
297def date_time(t=None, local_time_zone=False):
298    """http://www.w3.org/TR/NOTE-datetime ex: 1997-07-16T19:20:30Z
299
300    ```python
301    >>> date_time(1126482850)
302    '2005-09-11T23:54:10Z'
303
304    @@ this will change depending on where it is run
305    #>>> date_time(1126482850, local_time_zone=True)
306    #'2005-09-11T19:54:10-04:00'
307
308    >>> date_time(1)
309    '1970-01-01T00:00:01Z'
310
311    >>> date_time(0)
312    '1970-01-01T00:00:00Z'
313
314    ```
315    """
316    if t is None:
317        t = time()
318
319    if local_time_zone:
320        time_tuple = localtime(t)
321        if time_tuple[8]:
322            tz_mins = altzone // 60
323        else:
324            tz_mins = timezone // 60
325        tzd = "-%02d:%02d" % (tz_mins // 60, tz_mins % 60)
326    else:
327        time_tuple = gmtime(t)
328        tzd = "Z"
329
330    year, month, day, hh, mm, ss, wd, y, z = time_tuple
331    s = "%0004d-%02d-%02dT%02d:%02d:%02d%s" % (year, month, day, hh, mm, ss, tzd)
332    return s
333
334
335def parse_date_time(val: str) -> int:
336    """always returns seconds in UTC
337
338    ```python
339    # tests are written like this to make any errors easier to understand
340    >>> parse_date_time('2005-09-11T23:54:10Z') - 1126482850.0
341    0.0
342
343    >>> parse_date_time('2005-09-11T16:54:10-07:00') - 1126482850.0
344    0.0
345
346    >>> parse_date_time('1970-01-01T00:00:01Z') - 1.0
347    0.0
348
349    >>> parse_date_time('1970-01-01T00:00:00Z') - 0.0
350    0.0
351    >>> parse_date_time("2005-09-05T10:42:00") - 1125916920.0
352    0.0
353
354    ```
355    """
356
357    if "T" not in val:
358        val += "T00:00:00Z"
359
360    ymd, time = val.split("T")
361    hms, tz_str = time[0:8], time[8:]
362
363    if not tz_str or tz_str == "Z":
364        time = time[:-1]
365        tz_offset = 0
366    else:
367        signed_hrs = int(tz_str[:3])
368        mins = int(tz_str[4:6])
369        secs = (sign(signed_hrs) * mins + signed_hrs * 60) * 60
370        tz_offset = -secs
371
372    year, month, day = ymd.split("-")
373    hour, minute, second = hms.split(":")
374
375    t = timegm(
376        (int(year), int(month), int(day), int(hour), int(minute), int(second), 0, 0, 0)
377    )
378    t = t + tz_offset
379    return t
380
381
382def guess_format(fpath: str, fmap: Optional[Dict[str, str]] = None) -> Optional[str]:
383    """
384    Guess RDF serialization based on file suffix. Uses
385    `SUFFIX_FORMAT_MAP` unless `fmap` is provided.
386
387    Example:
388        ```python
389        >>> guess_format('path/to/file.rdf')
390        'xml'
391        >>> guess_format('path/to/file.owl')
392        'xml'
393        >>> guess_format('path/to/file.ttl')
394        'turtle'
395        >>> guess_format('path/to/file.json')
396        'json-ld'
397        >>> guess_format('path/to/file.xhtml')
398        'rdfa'
399        >>> guess_format('path/to/file.svg')
400        'rdfa'
401        >>> guess_format('path/to/file.xhtml', {'xhtml': 'grddl'})
402        'grddl'
403
404        ```
405
406        This also works with just the suffixes, with or without leading dot, and
407        regardless of letter case:
408
409        ```python
410        >>> guess_format('.rdf')
411        'xml'
412        >>> guess_format('rdf')
413        'xml'
414        >>> guess_format('RDF')
415        'xml'
416
417        ```
418    """
419    fmap = fmap or SUFFIX_FORMAT_MAP
420    return fmap.get(_get_ext(fpath)) or fmap.get(fpath.lower())
421
422
423def _get_ext(fpath: str, lower: bool = True) -> str:
424    """
425    Gets the file extension from a file(path); stripped of leading '.' and in
426    lower case.
427
428    Example:
429        ```python
430        >>> _get_ext("path/to/file.txt")
431        'txt'
432        >>> _get_ext("OTHER.PDF")
433        'pdf'
434        >>> _get_ext("noext")
435        ''
436        >>> _get_ext(".rdf")
437        'rdf'
438
439        ```
440    """
441    ext = splitext(fpath)[-1]
442    if ext == "" and fpath.startswith("."):
443        ext = fpath
444    if lower:
445        ext = ext.lower()
446    if ext.startswith("."):
447        ext = ext[1:]
448    return ext
449
450
451def find_roots(
452    graph: Graph,
453    prop: rdflib.term.URIRef,
454    roots: Optional[Set[rdflib.term.Node]] = None,
455) -> Set[rdflib.term.Node]:
456    """Find the roots in some sort of transitive hierarchy.
457
458    find_roots(graph, rdflib.RDFS.subClassOf)
459    will return a set of all roots of the sub-class hierarchy
460
461    Assumes triple of the form (child, prop, parent), i.e. the direction of
462    `RDFS.subClassOf` or `SKOS.broader`
463    """
464
465    non_roots: Set[rdflib.term.Node] = set()
466    if roots is None:
467        roots = set()
468    for x, y in graph.subject_objects(prop):
469        non_roots.add(x)
470        if x in roots:
471            roots.remove(x)
472        if y not in non_roots:
473            roots.add(y)
474    return roots
475
476
477def get_tree(
478    graph: Graph,
479    root: rdflib.term.Node,
480    prop: rdflib.term.URIRef,
481    mapper: Callable[[rdflib.term.Node], rdflib.term.Node] = lambda x: x,
482    sortkey: Optional[Callable[[Any], Any]] = None,
483    done: Optional[Set[rdflib.term.Node]] = None,
484    dir: str = "down",
485) -> Optional[Tuple[rdflib.term.Node, List[Any]]]:
486    """
487    Return a nested list/tuple structure representing the tree
488    built by the transitive property given, starting from the root given
489
490    i.e.
491
492    ```python
493    get_tree(
494        graph,
495        rdflib.URIRef("http://xmlns.com/foaf/0.1/Person"),
496        rdflib.RDFS.subClassOf,
497    )
498    ```
499
500    will return the structure for the subClassTree below person.
501
502    dir='down' assumes triple of the form (child, prop, parent),
503    i.e. the direction of RDFS.subClassOf or SKOS.broader
504    Any other dir traverses in the other direction
505    """
506
507    if done is None:
508        done = set()
509    if root in done:
510        # type error: Return value expected
511        return  # type: ignore[return-value]
512    done.add(root)
513    tree = []
514
515    branches: Iterator[rdflib.term.Node]
516    if dir == "down":
517        branches = graph.subjects(prop, root)
518    else:
519        branches = graph.objects(root, prop)
520
521    for branch in branches:
522        t = get_tree(graph, branch, prop, mapper, sortkey, done, dir)
523        if t:
524            tree.append(t)
525
526    return (mapper(root), sorted(tree, key=sortkey))
527
528
529@overload
530def _coalesce(*args: Optional[_AnyT], default: _AnyT) -> _AnyT: ...
531
532
533@overload
534def _coalesce(
535    *args: Optional[_AnyT], default: Optional[_AnyT] = ...
536) -> Optional[_AnyT]: ...
537
538
539def _coalesce(
540    *args: Optional[_AnyT], default: Optional[_AnyT] = None
541) -> Optional[_AnyT]:
542    """
543    This is a null coalescing function, it will return the first non-`None`
544    argument passed to it, otherwise it will return `default` which is `None`
545    by default.
546
547    For more info regarding the rationale of this function see deferred
548    [PEP 505](https://peps.python.org/pep-0505/).
549
550    Args:
551        *args: Values to consider as candidates to return, the first arg that
552            is not `None` will be returned. If no argument is passed this function
553            will return None.
554        default: The default value to return if none of the args are not `None`.
555
556    Returns:
557        The first `args` that is not `None`, otherwise the value of
558            `default` if there are no `args` or if all `args` are `None`.
559    """
560    for arg in args:
561        if arg is not None:
562            return arg
563    return default
564
565
566_RFC3986_SUBDELIMS = "!$&'()*+,;="
567"""
568`sub-delims` production from
569[RFC 3986, section 2.2](https://www.rfc-editor.org/rfc/rfc3986.html#section-2.2).
570"""
571
572_RFC3986_PCHAR_NU = "%" + _RFC3986_SUBDELIMS + ":@"
573"""
574The non-unreserved characters in the `pchar` production from RFC 3986.
575"""
576
577_QUERY_SAFE_CHARS = _RFC3986_PCHAR_NU + "/?"
578"""
579The non-unreserved characters that are safe to use in in the query and fragment
580components.
581
582```
583pchar         = unreserved / pct-encoded / sub-delims / ":" / "@" query
584= *( pchar / "/" / "?" ) fragment      = *( pchar / "/" / "?" )
585```
586"""
587
588_USERNAME_SAFE_CHARS = _RFC3986_SUBDELIMS + "%"
589"""
590The non-unreserved characters that are safe to use in the username and password
591components.
592
593```
594userinfo      = *( unreserved / pct-encoded / sub-delims / ":" )
595```
596
597":" is excluded as this is only used for the username and password components,
598and they are treated separately.
599"""
600
601_PATH_SAFE_CHARS = _RFC3986_PCHAR_NU + "/"
602"""
603The non-unreserved characters that are safe to use in the path component.
604
605This is based on various path-related productions from RFC 3986.
606"""
607
608
609def _iri2uri(iri: str) -> str:
610    """
611    Prior art:
612
613    - [iri_to_uri from Werkzeug](https://github.com/pallets/werkzeug/blob/92c6380248c7272ee668e1f8bbd80447027ccce2/src/werkzeug/urls.py#L926-L931)
614
615    ```python
616    >>> _iri2uri("https://dbpedia.org/resource/Almería")
617    'https://dbpedia.org/resource/Almer%C3%ADa'
618
619    ```
620    """
621    # https://datatracker.ietf.org/doc/html/rfc3986
622    # https://datatracker.ietf.org/doc/html/rfc3305
623
624    parts = urlsplit(iri)
625    (scheme, netloc, path, query, fragment) = parts
626
627    # Just support http/https, otherwise return the iri unaltered
628    if scheme not in ["http", "https"]:
629        return iri
630
631    path = quote(path, safe=_PATH_SAFE_CHARS)
632    query = quote(query, safe=_QUERY_SAFE_CHARS)
633    fragment = quote(fragment, safe=_QUERY_SAFE_CHARS)
634
635    if parts.hostname:
636        netloc = parts.hostname.encode("idna").decode("ascii")
637    else:
638        netloc = ""
639
640    if ":" in netloc:
641        # Quote IPv6 addresses
642        netloc = f"[{netloc}]"
643
644    if parts.port:
645        netloc = f"{netloc}:{parts.port}"
646
647    if parts.username:
648        auth = quote(parts.username, safe=_USERNAME_SAFE_CHARS)
649        if parts.password:
650            pass_quoted = quote(parts.password, safe=_USERNAME_SAFE_CHARS)
651            auth = f"{auth}:{pass_quoted}"
652        netloc = f"{auth}@{netloc}"
653
654    uri = urlunsplit((scheme, netloc, path, query, fragment))
655
656    if iri.endswith("#") and not uri.endswith("#"):
657        uri += "#"
658
659    return uri