Source code

Download source archive ↓

src/rdflib/tools/sparqlquery.py

1#!/usr/bin/env python
2"""
3A commandline tool for querying with SPARQL on local files and remote sparql endpoints with custom serialization.
4
5example usage:
6```bash
7    sq path/to/data.ttl -q "SELECT ?x WHERE {?x a foaf:Person. }"
8    rdfpipe test.ttl | sparqlquery - -q "SELECT ?x WHERE {?x a foaf:Person. }" --format json
9    sq data.ttl -q "ASK {:john a foaf:Person}" --format xml | grep true
10    sq path/to/data.ttl --query-file query.rq
11    sq data1.ttl data2.ttl -q "DESCRIBE <http://example.com/john>" --format turtle:+spacious
12    sq http://example.com/sparqlendpoint --query-file query.rq
13    sq http://example.com/sparqlendpoint --query-file query.rq --username user --password secret
14    sq /pyth/to/berkeley.db -q "SELECT ?x WHERE {?x a foaf:Person. }" --remote-storetype BerkeleyDB
15```
16
17Tip: You can check the truth value for an ASK query, by regex in stdout for 'true'
18or 'false'.
19"""
20from __future__ import annotations
21
22import argparse
23import inspect
24import logging
25import os
26import sys
27from inspect import Parameter
28from typing import Any, Dict, List, Optional, Tuple, Type
29from urllib.parse import urlparse
30
31try:
32    # Pyparsing >=3.0.0
33    from pyparsing.exceptions import ParseException
34except ImportError:
35    # Pyparsing 2
36    from pyparsing import ParseException
37
38from rdflib.graph import Dataset, Graph
39from rdflib.plugin import PluginException
40from rdflib.plugin import get as get_plugin
41from rdflib.plugin import plugins as get_plugins
42from rdflib.query import Result, ResultSerializer
43from rdflib.serializer import Serializer
44from rdflib.store import Store
45
46from .rdfpipe import _format_and_kws
47
48__all__ = ["sparqlquery"]
49
50
51class _ArgumentError(Exception):
52    pass
53
54
55class _PrintHelpError(Exception):
56    pass
57
58
59class InvalidQueryError(Exception):
60    pass
61
62
63def sparqlquery(
64    endpoints: List[str],
65    query: str,
66    result_format: Optional[str] = None,
67    result_keywords: Dict[str, str] = {},
68    auth: Optional[Tuple[str, str]] = None,
69    use_stdin: bool = False,
70    remote_storetype: Optional[str] = None,
71):
72    if use_stdin:
73        g = Graph().parse(sys.stdin)
74    else:
75        g = _get_graph(endpoints, auth, remote_storetype)
76    try:
77        results: Result = g.query(query)
78    except ParseException as err:
79        raise InvalidQueryError(query) from err
80
81    if result_format is not None:
82        ret_bytes = results.serialize(format=result_format, **result_keywords)
83    else:
84        ret_bytes = results.serialize(**result_keywords)
85    if ret_bytes is not None:
86        print(ret_bytes.decode())
87
88
89def _dest_is_local(dest: str):
90    if os.path.isabs(dest):
91        return True
92
93    q = urlparse(dest)
94    # Handle Windows drive letters (single letter followed by colon)
95    if len(q.scheme) == 1 and q.scheme.isalpha():
96        return True
97
98    return q.scheme in ["", "file"]
99
100
101def _dest_is_internet_addr(dest: str):
102    q = urlparse(dest)
103    return q.scheme in ["http", "https"]
104
105
106def _get_graph(
107    endpoints, auth: Optional[Tuple[str, str]], remote_storetype: Optional[str]
108) -> Graph:
109    graph: Graph
110    if remote_storetype is not None:
111        storeplugin = get_plugin(remote_storetype, Store)
112        if auth:
113            store = storeplugin(endpoints[0], auth=auth)  # type: ignore[call-arg]
114        else:
115            store = storeplugin(endpoints[0])
116        graph = Dataset(store)
117    else:
118        graph = Graph()
119        for x in endpoints:
120            graph.parse(location=x)
121    return graph
122
123
124def _extract_query_and_format(parser) -> Tuple[Dict[str, Any], Optional[str]]:
125    opts: Dict[str, Any] = {}
126    tmp_args, rest_args = parser.parse_known_args()
127    if tmp_args.query and tmp_args.queryfile is None:
128        query = tmp_args.query
129    elif tmp_args.queryfile and tmp_args.query is None:
130        with open(tmp_args.queryfile, "r") as f:
131            query = f.read()
132    else:
133        query = None
134
135    if query is None:
136        construct = False
137    elif "DESCRIBE" in query or "CONSTRUCT" in query:
138        construct = True
139    else:
140        construct = False
141
142    if tmp_args.format is not None:
143        format_, format_keywords = _format_and_kws(tmp_args.format)
144    elif construct:
145        format_keywords = {}
146        format_ = "turtle"
147        construct = True
148    else:
149        format_keywords = {}
150        format_ = "json"
151    epilog = _create_epilog_from_format(format_, construct)
152    opts = {
153        "query": query,
154        "result_format": format_,
155        "result_keywords": format_keywords,
156    }
157    return opts, epilog
158
159
160def parse_args():
161    extra_kwargs: Dict[str, Any] = {}
162    if sys.version_info > (3, 9):
163        extra_kwargs["exit_on_error"] = False
164    parser = argparse.ArgumentParser(
165        prog="sparqlquery",
166        description=__doc__,
167        add_help=False,  # add dynamic epilog before help is added
168        formatter_class=argparse.RawDescriptionHelpFormatter,
169        # else __doc__ wont be printed on error:
170        **extra_kwargs,
171    )
172    parser.add_argument(
173        "-q",
174        "--query",
175        type=str,
176        help="Sparql query. Cannot be set together with -qf/--queryfile.",
177    )
178    parser.add_argument(
179        "-qf",
180        "--queryfile",
181        type=str,
182        help="File from where the sparql query is read. "
183        "Cannot be set together with -q/--query",
184    )
185    parser.add_argument(
186        "-f",
187        "--format",
188        type=str,
189        help="Print sparql result in given format. "
190        "Defaults to 'json' on SELECT, to 'xml' on ASK "
191        "and to 'turtle' on DESCRIBE and CONSTRUCT. "
192        "Keywords as described in epilog can be given "
193        "after format like: "
194        "FORMAT:(+)KW1,-KW2,KW3=VALUE.",
195    )
196    opts: Dict[str, Any]
197    opts, parser.epilog = _extract_query_and_format(parser)
198
199    parser.add_argument(
200        "endpoint",
201        nargs="+",
202        type=str,
203        help="Endpoints for sparql query. "
204        "Can be set to multiple files. "
205        "Reads from stdin if '-' is given. ",
206    )
207    parser.add_argument(
208        "-w",
209        "--warn",
210        action="store_true",
211        default=False,
212        help="Output warnings to stderr " "(by default only critical errors).",
213    )
214    parser.add_argument(
215        "-h",
216        "--help",
217        # action="store_true",
218        # default=False,
219        action="help",
220        help="show help message and exit. "
221        "Also prints information about given format.",
222    )
223    parser.add_argument(
224        "-u", "--username", type=str, help="Username used during authentication."
225    )
226    parser.add_argument(
227        "-p", "--password", type=str, help="Password used during authentication."
228    )
229    parser.add_argument(
230        "-rs",
231        "--remote-storetype",
232        type=str,
233        help="You can specify which storetype should be used. "
234        "Can only be set, when using a single endpoint and not stdin. "
235        "Will default to 'SparqlStore' when endpoint is an internetaddress.",
236    )
237
238    try:  # catch error because exit_on_error=False
239        args = parser.parse_args()
240    except argparse.ArgumentError as err:
241        parser.print_help()
242        raise err
243
244    forbidden_format_keywords = [
245        x
246        for x in opts.get("result_keywords", dict())
247        if x in {"self", "stream", "encoding", "format"}
248    ]
249    if forbidden_format_keywords:
250        raise _ArgumentError(
251            "'self', 'stream', 'encoding' and 'format' "
252            "mustnt be used as keywords for format."
253        )
254
255    if opts.get("query") is None:
256        parser.print_help()
257        raise _ArgumentError("Either -q/--query or -qf/--queryfile must be provided")
258
259    remote_storetype = args.remote_storetype
260
261    if len(args.endpoint) == 1:
262        if args.endpoint[0] == "-":
263            if remote_storetype is not None:
264                raise _ArgumentError(
265                    "Cant us remote graphtype, when endpoint is stdin(-)"
266                )
267            endpoints = []
268            opts["use_stdin"] = True
269        elif _dest_is_internet_addr(args.endpoint[0]):
270            endpoints = args.endpoint
271            if remote_storetype is None:
272                remote_storetype = "SPARQLStore"
273        else:
274            endpoints = args.endpoint
275    else:
276        if remote_storetype is not None:
277            raise _ArgumentError(
278                "If remote graphtype is set, only a single endpoint is valid."
279            )
280        endpoints = list(args.endpoint)
281        if any(not (_dest_is_local(x)) for x in args.endpoint):
282            raise NotImplementedError(
283                "If multiple endpoints are given, all must be local files."
284            )
285
286    if args.username is not None and args.password is not None:
287        if remote_storetype not in ["SPARQLStore"]:
288            raise _ArgumentError(
289                "Can use password and username only, "
290                "when remote-storetype is 'SPARQLStore'."
291            )
292        opts["auth"] = (args.username, args.password)
293    elif args.username is None and args.password is None:
294        pass
295    else:
296        parser.print_help()
297        raise _ArgumentError("User only provided one of password and username")
298
299    return endpoints, remote_storetype, args.warn, opts
300
301
302def _create_epilog_from_format(format_, construct) -> Optional[str]:
303    serializer_plugin_type: Type[ResultSerializer | Serializer]
304    if construct:
305        serializer_plugin_type = Serializer
306    else:
307        serializer_plugin_type = ResultSerializer
308    try:
309        plugin = get_plugin(format_, serializer_plugin_type)
310    except PluginException:
311        available_plugins = [x.name for x in get_plugins(None, ResultSerializer)]
312        return (
313            f"No plugin registered for sparql result in format '{format_}'. "
314            f"available plugins: {available_plugins}"
315        )
316    serialize_method = plugin.serialize  # type: ignore[attr-defined]
317    module = inspect.getmodule(serialize_method)
318    if module is None:
319        return None
320    pydoc_target = ".".join([module.__name__, serialize_method.__qualname__])
321    sig = inspect.signature(serialize_method)
322    available_keywords = [
323        x
324        for x, y in sig.parameters.items()
325        if y.kind in [Parameter.KEYWORD_ONLY, Parameter.POSITIONAL_OR_KEYWORD]
326    ]
327    available_keywords.pop(0)  # pop self
328    available_keywords.pop(0)  # pop stream
329    if serializer_plugin_type == Serializer:
330        available_keywords.pop(1)  # pop encoding
331    else:
332        available_keywords.pop(0)  # pop encoding
333
334    epilog = (
335        f"For more customization for format '{format_}' "
336        f"use `pydoc {pydoc_target}`. "
337    )
338    if len(available_keywords) > 0:
339        epilog += f"Known keywords are {available_keywords}."
340    # there is always **kwargs in .serialize
341    epilog += " Further keywords might be valid."
342    return epilog
343
344
345def main():
346    try:
347        (
348            endpoints,
349            remote_storetype,
350            warn,
351            opts,
352        ) = parse_args()
353    except _PrintHelpError:
354        exit()
355    except (_ArgumentError, argparse.ArgumentError) as err:
356        print(err, file=sys.stderr)
357        exit(2)
358
359    if warn:
360        loglevel = logging.WARNING
361    else:
362        loglevel = logging.CRITICAL
363    logging.basicConfig(level=loglevel)
364
365    sparqlquery(
366        endpoints,
367        remote_storetype=remote_storetype,
368        **opts,
369    )
370
371
372if __name__ == "__main__":
373    main()