src/rdflib/tools/sparqlquery.py
1#!/usr/bin/env python
2"""
3A commandline tool for querying with SPARQL on local files and remote sparql endpoints with custom serialization.
4
5example usage:
6```bash
7 sq path/to/data.ttl -q "SELECT ?x WHERE {?x a foaf:Person. }"
8 rdfpipe test.ttl | sparqlquery - -q "SELECT ?x WHERE {?x a foaf:Person. }" --format json
9 sq data.ttl -q "ASK {:john a foaf:Person}" --format xml | grep true
10 sq path/to/data.ttl --query-file query.rq
11 sq data1.ttl data2.ttl -q "DESCRIBE <http://example.com/john>" --format turtle:+spacious
12 sq http://example.com/sparqlendpoint --query-file query.rq
13 sq http://example.com/sparqlendpoint --query-file query.rq --username user --password secret
14 sq /pyth/to/berkeley.db -q "SELECT ?x WHERE {?x a foaf:Person. }" --remote-storetype BerkeleyDB
15```
16
17Tip: You can check the truth value for an ASK query, by regex in stdout for 'true'
18or 'false'.
19"""
20from __future__ import annotations
21
22import argparse
23import inspect
24import logging
25import os
26import sys
27from inspect import Parameter
28from typing import Any, Dict, List, Optional, Tuple, Type
29from urllib.parse import urlparse
30
31try:
32 # Pyparsing >=3.0.0
33 from pyparsing.exceptions import ParseException
34except ImportError:
35 # Pyparsing 2
36 from pyparsing import ParseException
37
38from rdflib.graph import Dataset, Graph
39from rdflib.plugin import PluginException
40from rdflib.plugin import get as get_plugin
41from rdflib.plugin import plugins as get_plugins
42from rdflib.query import Result, ResultSerializer
43from rdflib.serializer import Serializer
44from rdflib.store import Store
45
46from .rdfpipe import _format_and_kws
47
48__all__ = ["sparqlquery"]
49
50
51class _ArgumentError(Exception):
52 pass
53
54
55class _PrintHelpError(Exception):
56 pass
57
58
59class InvalidQueryError(Exception):
60 pass
61
62
63def sparqlquery(
64 endpoints: List[str],
65 query: str,
66 result_format: Optional[str] = None,
67 result_keywords: Dict[str, str] = {},
68 auth: Optional[Tuple[str, str]] = None,
69 use_stdin: bool = False,
70 remote_storetype: Optional[str] = None,
71):
72 if use_stdin:
73 g = Graph().parse(sys.stdin)
74 else:
75 g = _get_graph(endpoints, auth, remote_storetype)
76 try:
77 results: Result = g.query(query)
78 except ParseException as err:
79 raise InvalidQueryError(query) from err
80
81 if result_format is not None:
82 ret_bytes = results.serialize(format=result_format, **result_keywords)
83 else:
84 ret_bytes = results.serialize(**result_keywords)
85 if ret_bytes is not None:
86 print(ret_bytes.decode())
87
88
89def _dest_is_local(dest: str):
90 if os.path.isabs(dest):
91 return True
92
93 q = urlparse(dest)
94 # Handle Windows drive letters (single letter followed by colon)
95 if len(q.scheme) == 1 and q.scheme.isalpha():
96 return True
97
98 return q.scheme in ["", "file"]
99
100
101def _dest_is_internet_addr(dest: str):
102 q = urlparse(dest)
103 return q.scheme in ["http", "https"]
104
105
106def _get_graph(
107 endpoints, auth: Optional[Tuple[str, str]], remote_storetype: Optional[str]
108) -> Graph:
109 graph: Graph
110 if remote_storetype is not None:
111 storeplugin = get_plugin(remote_storetype, Store)
112 if auth:
113 store = storeplugin(endpoints[0], auth=auth) # type: ignore[call-arg]
114 else:
115 store = storeplugin(endpoints[0])
116 graph = Dataset(store)
117 else:
118 graph = Graph()
119 for x in endpoints:
120 graph.parse(location=x)
121 return graph
122
123
124def _extract_query_and_format(parser) -> Tuple[Dict[str, Any], Optional[str]]:
125 opts: Dict[str, Any] = {}
126 tmp_args, rest_args = parser.parse_known_args()
127 if tmp_args.query and tmp_args.queryfile is None:
128 query = tmp_args.query
129 elif tmp_args.queryfile and tmp_args.query is None:
130 with open(tmp_args.queryfile, "r") as f:
131 query = f.read()
132 else:
133 query = None
134
135 if query is None:
136 construct = False
137 elif "DESCRIBE" in query or "CONSTRUCT" in query:
138 construct = True
139 else:
140 construct = False
141
142 if tmp_args.format is not None:
143 format_, format_keywords = _format_and_kws(tmp_args.format)
144 elif construct:
145 format_keywords = {}
146 format_ = "turtle"
147 construct = True
148 else:
149 format_keywords = {}
150 format_ = "json"
151 epilog = _create_epilog_from_format(format_, construct)
152 opts = {
153 "query": query,
154 "result_format": format_,
155 "result_keywords": format_keywords,
156 }
157 return opts, epilog
158
159
160def parse_args():
161 extra_kwargs: Dict[str, Any] = {}
162 if sys.version_info > (3, 9):
163 extra_kwargs["exit_on_error"] = False
164 parser = argparse.ArgumentParser(
165 prog="sparqlquery",
166 description=__doc__,
167 add_help=False, # add dynamic epilog before help is added
168 formatter_class=argparse.RawDescriptionHelpFormatter,
169 # else __doc__ wont be printed on error:
170 **extra_kwargs,
171 )
172 parser.add_argument(
173 "-q",
174 "--query",
175 type=str,
176 help="Sparql query. Cannot be set together with -qf/--queryfile.",
177 )
178 parser.add_argument(
179 "-qf",
180 "--queryfile",
181 type=str,
182 help="File from where the sparql query is read. "
183 "Cannot be set together with -q/--query",
184 )
185 parser.add_argument(
186 "-f",
187 "--format",
188 type=str,
189 help="Print sparql result in given format. "
190 "Defaults to 'json' on SELECT, to 'xml' on ASK "
191 "and to 'turtle' on DESCRIBE and CONSTRUCT. "
192 "Keywords as described in epilog can be given "
193 "after format like: "
194 "FORMAT:(+)KW1,-KW2,KW3=VALUE.",
195 )
196 opts: Dict[str, Any]
197 opts, parser.epilog = _extract_query_and_format(parser)
198
199 parser.add_argument(
200 "endpoint",
201 nargs="+",
202 type=str,
203 help="Endpoints for sparql query. "
204 "Can be set to multiple files. "
205 "Reads from stdin if '-' is given. ",
206 )
207 parser.add_argument(
208 "-w",
209 "--warn",
210 action="store_true",
211 default=False,
212 help="Output warnings to stderr " "(by default only critical errors).",
213 )
214 parser.add_argument(
215 "-h",
216 "--help",
217 # action="store_true",
218 # default=False,
219 action="help",
220 help="show help message and exit. "
221 "Also prints information about given format.",
222 )
223 parser.add_argument(
224 "-u", "--username", type=str, help="Username used during authentication."
225 )
226 parser.add_argument(
227 "-p", "--password", type=str, help="Password used during authentication."
228 )
229 parser.add_argument(
230 "-rs",
231 "--remote-storetype",
232 type=str,
233 help="You can specify which storetype should be used. "
234 "Can only be set, when using a single endpoint and not stdin. "
235 "Will default to 'SparqlStore' when endpoint is an internetaddress.",
236 )
237
238 try: # catch error because exit_on_error=False
239 args = parser.parse_args()
240 except argparse.ArgumentError as err:
241 parser.print_help()
242 raise err
243
244 forbidden_format_keywords = [
245 x
246 for x in opts.get("result_keywords", dict())
247 if x in {"self", "stream", "encoding", "format"}
248 ]
249 if forbidden_format_keywords:
250 raise _ArgumentError(
251 "'self', 'stream', 'encoding' and 'format' "
252 "mustnt be used as keywords for format."
253 )
254
255 if opts.get("query") is None:
256 parser.print_help()
257 raise _ArgumentError("Either -q/--query or -qf/--queryfile must be provided")
258
259 remote_storetype = args.remote_storetype
260
261 if len(args.endpoint) == 1:
262 if args.endpoint[0] == "-":
263 if remote_storetype is not None:
264 raise _ArgumentError(
265 "Cant us remote graphtype, when endpoint is stdin(-)"
266 )
267 endpoints = []
268 opts["use_stdin"] = True
269 elif _dest_is_internet_addr(args.endpoint[0]):
270 endpoints = args.endpoint
271 if remote_storetype is None:
272 remote_storetype = "SPARQLStore"
273 else:
274 endpoints = args.endpoint
275 else:
276 if remote_storetype is not None:
277 raise _ArgumentError(
278 "If remote graphtype is set, only a single endpoint is valid."
279 )
280 endpoints = list(args.endpoint)
281 if any(not (_dest_is_local(x)) for x in args.endpoint):
282 raise NotImplementedError(
283 "If multiple endpoints are given, all must be local files."
284 )
285
286 if args.username is not None and args.password is not None:
287 if remote_storetype not in ["SPARQLStore"]:
288 raise _ArgumentError(
289 "Can use password and username only, "
290 "when remote-storetype is 'SPARQLStore'."
291 )
292 opts["auth"] = (args.username, args.password)
293 elif args.username is None and args.password is None:
294 pass
295 else:
296 parser.print_help()
297 raise _ArgumentError("User only provided one of password and username")
298
299 return endpoints, remote_storetype, args.warn, opts
300
301
302def _create_epilog_from_format(format_, construct) -> Optional[str]:
303 serializer_plugin_type: Type[ResultSerializer | Serializer]
304 if construct:
305 serializer_plugin_type = Serializer
306 else:
307 serializer_plugin_type = ResultSerializer
308 try:
309 plugin = get_plugin(format_, serializer_plugin_type)
310 except PluginException:
311 available_plugins = [x.name for x in get_plugins(None, ResultSerializer)]
312 return (
313 f"No plugin registered for sparql result in format '{format_}'. "
314 f"available plugins: {available_plugins}"
315 )
316 serialize_method = plugin.serialize # type: ignore[attr-defined]
317 module = inspect.getmodule(serialize_method)
318 if module is None:
319 return None
320 pydoc_target = ".".join([module.__name__, serialize_method.__qualname__])
321 sig = inspect.signature(serialize_method)
322 available_keywords = [
323 x
324 for x, y in sig.parameters.items()
325 if y.kind in [Parameter.KEYWORD_ONLY, Parameter.POSITIONAL_OR_KEYWORD]
326 ]
327 available_keywords.pop(0) # pop self
328 available_keywords.pop(0) # pop stream
329 if serializer_plugin_type == Serializer:
330 available_keywords.pop(1) # pop encoding
331 else:
332 available_keywords.pop(0) # pop encoding
333
334 epilog = (
335 f"For more customization for format '{format_}' "
336 f"use `pydoc {pydoc_target}`. "
337 )
338 if len(available_keywords) > 0:
339 epilog += f"Known keywords are {available_keywords}."
340 # there is always **kwargs in .serialize
341 epilog += " Further keywords might be valid."
342 return epilog
343
344
345def main():
346 try:
347 (
348 endpoints,
349 remote_storetype,
350 warn,
351 opts,
352 ) = parse_args()
353 except _PrintHelpError:
354 exit()
355 except (_ArgumentError, argparse.ArgumentError) as err:
356 print(err, file=sys.stderr)
357 exit(2)
358
359 if warn:
360 loglevel = logging.WARNING
361 else:
362 loglevel = logging.CRITICAL
363 logging.basicConfig(level=loglevel)
364
365 sparqlquery(
366 endpoints,
367 remote_storetype=remote_storetype,
368 **opts,
369 )
370
371
372if __name__ == "__main__":
373 main()