src/rdflib/plugins/stores/berkeleydb.py
1from __future__ import annotations
2
3import logging
4from os import mkdir
5from os.path import abspath, exists
6from threading import Thread
7from typing import (
8 TYPE_CHECKING,
9 Any,
10 Callable,
11 Dict,
12 Generator,
13 List,
14 Optional,
15 Tuple,
16 Union,
17)
18from urllib.request import pathname2url
19
20from rdflib.store import NO_STORE, VALID_STORE, Store
21from rdflib.term import Identifier, Node, URIRef
22
23if TYPE_CHECKING:
24 from rdflib.graph import Graph, _ContextType, _TriplePatternType, _TripleType
25
26
27def bb(u: str) -> bytes:
28 return u.encode("utf-8")
29
30
31try:
32 from berkeleydb import db
33
34 has_bsddb = True
35except ImportError:
36 has_bsddb = False
37
38
39if has_bsddb:
40 # These are passed to bsddb when creating DBs
41
42 # passed to db.DBEnv.set_flags
43 ENVSETFLAGS = db.DB_CDB_ALLDB
44 # passed to db.DBEnv.open
45 ENVFLAGS = db.DB_INIT_MPOOL | db.DB_INIT_CDB | db.DB_THREAD
46 CACHESIZE = 1024 * 1024 * 50
47
48 # passed to db.DB.Open()
49 DBOPENFLAGS = db.DB_THREAD
50
51logger = logging.getLogger(__name__)
52
53__all__ = [
54 "BerkeleyDB",
55 "_ToKeyFunc",
56 "_FromKeyFunc",
57 "_GetPrefixFunc",
58 "_ResultsFromKeyFunc",
59]
60
61
62_ToKeyFunc = Callable[[Tuple[bytes, bytes, bytes], bytes], bytes]
63_FromKeyFunc = Callable[[bytes], Tuple[bytes, bytes, bytes, bytes]]
64_GetPrefixFunc = Callable[
65 [Tuple[str, str, str], Optional[str]], Generator[str, None, None]
66]
67_ResultsFromKeyFunc = Callable[
68 [bytes, Optional[Node], Optional[Node], Optional[Node], bytes],
69 Tuple[Tuple[Node, Node, Node], Generator[Node, None, None]],
70]
71
72
73class BerkeleyDB(Store):
74 """A store that allows for on-disk persistent using BerkeleyDB, a fast key/value DB.
75
76 This store implementation used to be known, previous to rdflib 6.0.0
77 as 'Sleepycat' due to that being the then name of the Python wrapper
78 for BerkeleyDB.
79
80 This store allows for quads as well as triples. See examples of use
81 in both the `examples.berkeleydb_example` and `test/test_store/test_store_berkeleydb.py`
82 files.
83
84 **NOTE on installation**:
85
86 To use this store, you must have BerkeleyDB installed on your system
87 separately to Python (`brew install berkeley-db` on a Mac) and also have
88 the BerkeleyDB Python wrapper installed (`pip install berkeleydb`).
89 You may need to install BerkeleyDB Python wrapper like this:
90 `YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1 pip install berkeleydb`
91 """
92
93 context_aware = True
94 formula_aware = True
95 transaction_aware = False
96 graph_aware = True
97 db_env: db.DBEnv = None
98
99 def __init__(
100 self,
101 configuration: Optional[str] = None,
102 identifier: Optional[Identifier] = None,
103 ):
104 if not has_bsddb:
105 raise ImportError("Unable to import berkeleydb, store is unusable.")
106 self.__open = False
107 self.__identifier = identifier
108 super(BerkeleyDB, self).__init__(configuration)
109 self._loads = self.node_pickler.loads
110 self._dumps = self.node_pickler.dumps
111 self.__indicies_info: List[Tuple[Any, _ToKeyFunc, _FromKeyFunc]]
112
113 def __get_identifier(self) -> Optional[Identifier]:
114 return self.__identifier
115
116 identifier = property(__get_identifier)
117
118 def _init_db_environment(
119 self, homeDir: str, create: bool = True # noqa: N803
120 ) -> db.DBEnv:
121 if not exists(homeDir):
122 if create is True:
123 mkdir(homeDir)
124 # TODO: implement create method and refactor this to it
125 self.create(homeDir)
126 else:
127 return NO_STORE
128 db_env = db.DBEnv()
129 db_env.set_cachesize(0, CACHESIZE) # TODO
130 # db_env.set_lg_max(1024*1024)
131 db_env.set_flags(ENVSETFLAGS, 1)
132 db_env.open(homeDir, ENVFLAGS | db.DB_CREATE)
133 return db_env
134
135 def is_open(self) -> bool:
136 return self.__open
137
138 def open(
139 self, configuration: Union[str, tuple[str, str]], create: bool = True
140 ) -> Optional[int]:
141 if not has_bsddb:
142 return NO_STORE
143
144 if type(configuration) is str:
145 homeDir = configuration # noqa: N806
146 else:
147 raise Exception("Invalid configuration provided")
148
149 if self.__identifier is None:
150 self.__identifier = URIRef(pathname2url(abspath(homeDir)))
151
152 db_env = self._init_db_environment(homeDir, create)
153 if db_env == NO_STORE:
154 return NO_STORE
155 self.db_env = db_env
156 self.__open = True
157
158 dbname = None
159 dbtype = db.DB_BTREE
160 # auto-commit ensures that the open-call commits when transactions
161 # are enabled
162
163 dbopenflags = DBOPENFLAGS
164 if self.transaction_aware is True:
165 dbopenflags |= db.DB_AUTO_COMMIT
166
167 if create:
168 dbopenflags |= db.DB_CREATE
169
170 dbmode = 0o660
171 dbsetflags = 0
172
173 # create and open the DBs
174 self.__indicies: List[db.DB] = [
175 None,
176 ] * 3
177 # NOTE on type ingore: this is because type checker does not like this
178 # way of initializing, using a temporary variable will solve it.
179 # type error: error: List item 0 has incompatible type "None"; expected "Tuple[Any, Callable[[Tuple[bytes, bytes, bytes], bytes], bytes], Callable[[bytes], Tuple[bytes, bytes, bytes, bytes]]]"
180 self.__indicies_info = [
181 None, # type: ignore[list-item]
182 ] * 3
183 for i in range(0, 3):
184 index_name = to_key_func(i)(
185 ("s".encode("latin-1"), "p".encode("latin-1"), "o".encode("latin-1")),
186 "c".encode("latin-1"),
187 ).decode()
188 index = db.DB(db_env)
189 index.set_flags(dbsetflags)
190 index.open(index_name, dbname, dbtype, dbopenflags, dbmode)
191 self.__indicies[i] = index
192 self.__indicies_info[i] = (index, to_key_func(i), from_key_func(i))
193
194 lookup: Dict[
195 int, Tuple[db.DB, _GetPrefixFunc, _FromKeyFunc, _ResultsFromKeyFunc]
196 ] = {}
197 for i in range(0, 8):
198 results: List[Tuple[Tuple[int, int], int, int]] = []
199 for start in range(0, 3):
200 score = 1
201 len = 0
202 for j in range(start, start + 3):
203 if i & (1 << (j % 3)):
204 score = score << 1
205 len += 1
206 else:
207 break
208 tie_break = 2 - start
209 results.append(((score, tie_break), start, len))
210
211 results.sort()
212 # NOTE on type error: this is because the variable `score` is
213 # reused with different type
214 # type error: Incompatible types in assignment (expression has type "Tuple[int, int]", variable has type "int")
215 score, start, len = results[-1] # type: ignore[assignment]
216
217 def get_prefix_func(start: int, end: int) -> _GetPrefixFunc:
218 def get_prefix(
219 triple: Tuple[str, str, str], context: Optional[str]
220 ) -> Generator[str, None, None]:
221 if context is None:
222 yield ""
223 else:
224 yield context
225 i = start
226 while i < end:
227 yield triple[i % 3]
228 i += 1
229 yield ""
230
231 return get_prefix
232
233 lookup[i] = (
234 self.__indicies[start],
235 get_prefix_func(start, start + len),
236 from_key_func(start),
237 results_from_key_func(start, self._from_string),
238 )
239
240 self.__lookup_dict = lookup
241
242 self.__contexts = db.DB(db_env)
243 self.__contexts.set_flags(dbsetflags)
244 self.__contexts.open("contexts", dbname, dbtype, dbopenflags, dbmode)
245
246 self.__namespace = db.DB(db_env)
247 self.__namespace.set_flags(dbsetflags)
248 self.__namespace.open("namespace", dbname, dbtype, dbopenflags, dbmode)
249
250 self.__prefix = db.DB(db_env)
251 self.__prefix.set_flags(dbsetflags)
252 self.__prefix.open("prefix", dbname, dbtype, dbopenflags, dbmode)
253
254 self.__k2i = db.DB(db_env)
255 self.__k2i.set_flags(dbsetflags)
256 self.__k2i.open("k2i", dbname, db.DB_HASH, dbopenflags, dbmode)
257
258 self.__i2k = db.DB(db_env)
259 self.__i2k.set_flags(dbsetflags)
260 self.__i2k.open("i2k", dbname, db.DB_RECNO, dbopenflags, dbmode)
261
262 self.__needs_sync = False
263 t = Thread(target=self.__sync_run)
264 t.setDaemon(True)
265 t.start()
266 self.__sync_thread = t
267 return VALID_STORE
268
269 def __sync_run(self) -> None:
270 from time import sleep, time
271
272 try:
273 min_seconds, max_seconds = 10, 300
274 while self.__open:
275 if self.__needs_sync:
276 t0 = t1 = time()
277 self.__needs_sync = False
278 while self.__open:
279 sleep(0.1)
280 if self.__needs_sync:
281 t1 = time()
282 self.__needs_sync = False
283 if time() - t1 > min_seconds or time() - t0 > max_seconds:
284 self.__needs_sync = False
285 logger.debug("sync")
286 self.sync()
287 break
288 else:
289 sleep(1)
290 except Exception as e:
291 logger.exception(e)
292
293 def sync(self) -> None:
294 if self.__open:
295 for i in self.__indicies:
296 i.sync()
297 self.__contexts.sync()
298 self.__namespace.sync()
299 self.__prefix.sync()
300 self.__i2k.sync()
301 self.__k2i.sync()
302
303 def close(self, commit_pending_transaction: bool = False) -> None:
304 self.__open = False
305 self.__sync_thread.join()
306 for i in self.__indicies:
307 i.close()
308 self.__contexts.close()
309 self.__namespace.close()
310 self.__prefix.close()
311 self.__i2k.close()
312 self.__k2i.close()
313 self.db_env.close()
314
315 def add(
316 self,
317 triple: _TripleType,
318 context: _ContextType,
319 quoted: bool = False,
320 txn: Optional[Any] = None,
321 ) -> None:
322 """\
323 Add a triple to the store of triples.
324 """
325 (subject, predicate, object) = triple
326 assert self.__open, "The Store must be open."
327 assert context != self, "Can not add triple directly to store"
328 Store.add(self, (subject, predicate, object), context, quoted)
329
330 _to_string = self._to_string
331
332 s = _to_string(subject, txn=txn)
333 p = _to_string(predicate, txn=txn)
334 o = _to_string(object, txn=txn)
335 c = _to_string(context, txn=txn)
336
337 cspo, cpos, cosp = self.__indicies
338
339 value = cspo.get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
340 if value is None:
341 self.__contexts.put(bb(c), b"", txn=txn)
342
343 contexts_value = cspo.get(
344 bb("%s^%s^%s^%s^" % ("", s, p, o)), txn=txn
345 ) or "".encode("latin-1")
346 contexts = set(contexts_value.split("^".encode("latin-1")))
347 contexts.add(bb(c))
348 contexts_value = "^".encode("latin-1").join(contexts)
349 assert contexts_value is not None
350
351 cspo.put(bb("%s^%s^%s^%s^" % (c, s, p, o)), b"", txn=txn)
352 cpos.put(bb("%s^%s^%s^%s^" % (c, p, o, s)), b"", txn=txn)
353 cosp.put(bb("%s^%s^%s^%s^" % (c, o, s, p)), b"", txn=txn)
354 if not quoted:
355 cspo.put(bb("%s^%s^%s^%s^" % ("", s, p, o)), contexts_value, txn=txn)
356 cpos.put(bb("%s^%s^%s^%s^" % ("", p, o, s)), contexts_value, txn=txn)
357 cosp.put(bb("%s^%s^%s^%s^" % ("", o, s, p)), contexts_value, txn=txn)
358
359 self.__needs_sync = True
360
361 def __remove(
362 self,
363 spo: Tuple[bytes, bytes, bytes],
364 c: bytes,
365 quoted: bool = False,
366 txn: Optional[Any] = None,
367 ) -> None:
368 s, p, o = spo
369 cspo, cpos, cosp = self.__indicies
370 contexts_value = cspo.get(
371 "^".encode("latin-1").join(
372 ["".encode("latin-1"), s, p, o, "".encode("latin-1")]
373 ),
374 txn=txn,
375 ) or "".encode("latin-1")
376 contexts = set(contexts_value.split("^".encode("latin-1")))
377 contexts.discard(c)
378 contexts_value = "^".encode("latin-1").join(contexts)
379 for i, _to_key, _from_key in self.__indicies_info:
380 i.delete(_to_key((s, p, o), c), txn=txn)
381 if not quoted:
382 if contexts_value:
383 for i, _to_key, _from_key in self.__indicies_info:
384 i.put(
385 _to_key((s, p, o), "".encode("latin-1")),
386 contexts_value,
387 txn=txn,
388 )
389 else:
390 for i, _to_key, _from_key in self.__indicies_info:
391 try:
392 i.delete(_to_key((s, p, o), "".encode("latin-1")), txn=txn)
393 except db.DBNotFoundError:
394 pass # TODO: is it okay to ignore these?
395
396 # type error: Signature of "remove" incompatible with supertype "Store"
397 def remove( # type: ignore[override]
398 self,
399 spo: _TriplePatternType,
400 context: Optional[_ContextType],
401 txn: Optional[Any] = None,
402 ) -> None:
403 subject, predicate, object = spo
404 assert self.__open, "The Store must be open."
405 Store.remove(self, (subject, predicate, object), context)
406 _to_string = self._to_string
407
408 if context is not None:
409 if context == self:
410 context = None
411
412 if (
413 subject is not None
414 and predicate is not None
415 and object is not None
416 and context is not None
417 ):
418 s = _to_string(subject, txn=txn)
419 p = _to_string(predicate, txn=txn)
420 o = _to_string(object, txn=txn)
421 c = _to_string(context, txn=txn)
422 value = self.__indicies[0].get(bb("%s^%s^%s^%s^" % (c, s, p, o)), txn=txn)
423 if value is not None:
424 self.__remove((bb(s), bb(p), bb(o)), bb(c), txn=txn)
425 self.__needs_sync = True
426 else:
427 cspo, cpos, cosp = self.__indicies
428 index, prefix, from_key, results_from_key = self.__lookup(
429 (subject, predicate, object), context, txn=txn
430 )
431
432 cursor = index.cursor(txn=txn)
433 try:
434 current = cursor.set_range(prefix)
435 needs_sync = True
436 except db.DBNotFoundError:
437 current = None
438 needs_sync = False
439 cursor.close()
440 while current:
441 key, value = current
442 cursor = index.cursor(txn=txn)
443 try:
444 cursor.set_range(key)
445 # Hack to stop 2to3 converting this to next(cursor)
446 current = getattr(cursor, "next")()
447 except db.DBNotFoundError:
448 current = None
449 cursor.close()
450 if key.startswith(prefix):
451 # NOTE on type error: variables are being reused with a
452 # different type
453 # type error: Incompatible types in assignment (expression has type "bytes", variable has type "str")
454 c, s, p, o = from_key(key) # type: ignore[assignment]
455 if context is None:
456 contexts_value = index.get(key, txn=txn) or "".encode("latin-1")
457 # remove triple from all non quoted contexts
458 contexts = set(contexts_value.split("^".encode("latin-1")))
459 # and from the conjunctive index
460 contexts.add("".encode("latin-1"))
461 for c in contexts:
462 for i, _to_key, _ in self.__indicies_info:
463 # NOTE on type error: variables are being
464 # reused with a different type
465 # type error: Argument 1 has incompatible type "Tuple[str, str, str]"; expected "Tuple[bytes, bytes, bytes]"
466 # type error: Argument 2 has incompatible type "str"; expected "bytes"
467 i.delete(_to_key((s, p, o), c), txn=txn) # type: ignore[arg-type]
468 else:
469 # type error: Argument 1 to "__remove" of "BerkeleyDB" has incompatible type "Tuple[str, str, str]"; expected "Tuple[bytes, bytes, bytes]"
470 # type error: Argument 2 to "__remove" of "BerkeleyDB" has incompatible type "str"; expected "bytes"
471 self.__remove((s, p, o), c, txn=txn) # type: ignore[arg-type]
472 else:
473 break
474
475 if context is not None:
476 if subject is None and predicate is None and object is None:
477 # TODO: also if context becomes empty and not just on
478 # remove((None, None, None), c)
479 try:
480 self.__contexts.delete(
481 bb(_to_string(context, txn=txn)), txn=txn
482 )
483 except db.DBNotFoundError:
484 pass
485
486 self.__needs_sync = needs_sync
487
488 def triples(
489 self,
490 spo: _TriplePatternType,
491 context: Optional[_ContextType] = None,
492 txn: Optional[Any] = None,
493 ) -> Generator[
494 Tuple[_TripleType, Generator[Optional[_ContextType], None, None]],
495 None,
496 None,
497 ]:
498 """A generator over all the triples matching"""
499 assert self.__open, "The Store must be open."
500
501 subject, predicate, object = spo
502
503 if context is not None:
504 if context == self:
505 context = None
506
507 # _from_string = self._from_string ## UNUSED
508 index, prefix, from_key, results_from_key = self.__lookup(
509 (subject, predicate, object), context, txn=txn
510 )
511
512 cursor = index.cursor(txn=txn)
513 try:
514 current = cursor.set_range(prefix)
515 except db.DBNotFoundError:
516 current = None
517 cursor.close()
518 while current:
519 key, value = current
520 cursor = index.cursor(txn=txn)
521 try:
522 cursor.set_range(key)
523 # Cheap hack so 2to3 doesn't convert to next(cursor)
524 current = getattr(cursor, "next")()
525 except db.DBNotFoundError:
526 current = None
527 cursor.close()
528 if key and key.startswith(prefix):
529 contexts_value = index.get(key, txn=txn)
530 # type error: Incompatible types in "yield" (actual type "Tuple[Tuple[Node, Node, Node], Generator[Node, None, None]]", expected type "Tuple[Tuple[IdentifiedNode, URIRef, Identifier], Iterator[Optional[Graph]]]")
531 # NOTE on type ignore: this is needed because some context is
532 # lost in the process of extracting triples from the database.
533 yield results_from_key(key, subject, predicate, object, contexts_value) # type: ignore[misc]
534 else:
535 break
536
537 def __len__(self, context: Optional[_ContextType] = None) -> int:
538 assert self.__open, "The Store must be open."
539 if context is not None:
540 if context == self:
541 context = None
542
543 if context is None:
544 prefix = "^".encode("latin-1")
545 else:
546 prefix = bb("%s^" % self._to_string(context))
547
548 index = self.__indicies[0]
549 cursor = index.cursor()
550 current = cursor.set_range(prefix)
551 count = 0
552 while current:
553 key, value = current
554 if key.startswith(prefix):
555 count += 1
556 # Hack to stop 2to3 converting this to next(cursor)
557 current = getattr(cursor, "next")()
558 else:
559 break
560 cursor.close()
561 return count
562
563 def bind(self, prefix: str, namespace: URIRef, override: bool = True) -> None:
564 # NOTE on type error: this is because the variables are reused with
565 # another type.
566 # type error: Incompatible types in assignment (expression has type "bytes", variable has type "str")
567 prefix = prefix.encode("utf-8") # type: ignore[assignment]
568 # type error: Incompatible types in assignment (expression has type "bytes", variable has type "URIRef")
569 namespace = namespace.encode("utf-8") # type: ignore[assignment]
570 bound_prefix = self.__prefix.get(namespace)
571 bound_namespace = self.__namespace.get(prefix)
572 if override:
573 if bound_prefix:
574 self.__namespace.delete(bound_prefix)
575 if bound_namespace:
576 self.__prefix.delete(bound_namespace)
577 self.__prefix[namespace] = prefix
578 self.__namespace[prefix] = namespace
579 else:
580 self.__prefix[bound_namespace or namespace] = bound_prefix or prefix
581 self.__namespace[bound_prefix or prefix] = bound_namespace or namespace
582
583 def namespace(self, prefix: str) -> Optional[URIRef]:
584 # NOTE on type error: this is because the variable is reused with
585 # another type.
586 # type error: Incompatible types in assignment (expression has type "bytes", variable has type "str")
587 prefix = prefix.encode("utf-8") # type: ignore[assignment]
588 ns = self.__namespace.get(prefix, None)
589 if ns is not None:
590 return URIRef(ns.decode("utf-8"))
591 return None
592
593 def prefix(self, namespace: URIRef) -> Optional[str]:
594 # NOTE on type error: this is because the variable is reused with
595 # another type.
596 # type error: Incompatible types in assignment (expression has type "bytes", variable has type "URIRef")
597 namespace = namespace.encode("utf-8") # type: ignore[assignment]
598 prefix = self.__prefix.get(namespace, None)
599 if prefix is not None:
600 return prefix.decode("utf-8")
601 return None
602
603 def namespaces(self) -> Generator[Tuple[str, URIRef], None, None]:
604 cursor = self.__namespace.cursor()
605 results = []
606 current = cursor.first()
607 while current:
608 prefix, namespace = current
609 results.append((prefix.decode("utf-8"), namespace.decode("utf-8")))
610 # Hack to stop 2to3 converting this to next(cursor)
611 current = getattr(cursor, "next")()
612 cursor.close()
613 for prefix, namespace in results:
614 yield prefix, URIRef(namespace)
615
616 def contexts(
617 self, triple: Optional[_TripleType] = None
618 ) -> Generator[_ContextType, None, None]:
619 _from_string = self._from_string
620 _to_string = self._to_string
621 # NOTE on type errors: context is lost because of how data is loaded
622 # from the DB.
623 if triple:
624 s: str
625 p: str
626 o: str
627 # type error: Incompatible types in assignment (expression has type "Node", variable has type "str")
628 s, p, o = triple # type: ignore[assignment]
629 # type error: Argument 1 has incompatible type "str"; expected "Node"
630 s = _to_string(s) # type: ignore[arg-type]
631 # type error: Argument 1 has incompatible type "str"; expected "Node"
632 p = _to_string(p) # type: ignore[arg-type]
633 # type error: Argument 1 has incompatible type "str"; expected "Node"
634 o = _to_string(o) # type: ignore[arg-type]
635 contexts = self.__indicies[0].get(bb("%s^%s^%s^%s^" % ("", s, p, o)))
636 if contexts:
637 for c in contexts.split("^".encode("latin-1")):
638 if c:
639 # type error: Incompatible types in "yield" (actual type "Node", expected type "Graph")
640 yield _from_string(c) # type: ignore[misc]
641 else:
642 index = self.__contexts
643 cursor = index.cursor()
644 current = cursor.first()
645 cursor.close()
646 while current:
647 key, value = current
648 context = _from_string(key)
649 # type error: Incompatible types in "yield" (actual type "Node", expected type "Graph")
650 yield context # type: ignore[misc]
651 cursor = index.cursor()
652 try:
653 cursor.set_range(key)
654 # Hack to stop 2to3 converting this to next(cursor)
655 current = getattr(cursor, "next")()
656 except db.DBNotFoundError:
657 current = None
658 cursor.close()
659
660 def add_graph(self, graph: Graph) -> None:
661 self.__contexts.put(bb(self._to_string(graph)), b"")
662
663 def remove_graph(self, graph: Graph):
664 self.remove((None, None, None), graph)
665
666 def _from_string(self, i: bytes) -> Node:
667 k = self.__i2k.get(int(i))
668 return self._loads(k)
669
670 def _to_string(self, term: Node, txn: Optional[Any] = None) -> str:
671 k = self._dumps(term)
672 i = self.__k2i.get(k, txn=txn)
673 if i is None:
674 # weird behaviour from bsddb not taking a txn as a keyword argument
675 # for append
676 if self.transaction_aware:
677 i = "%s" % self.__i2k.append(k, txn)
678 else:
679 i = "%s" % self.__i2k.append(k)
680
681 self.__k2i.put(k, i.encode(), txn=txn)
682 else:
683 i = i.decode()
684 return i
685
686 def __lookup(
687 self,
688 spo: _TriplePatternType,
689 context: Optional[_ContextType],
690 txn: Optional[Any] = None,
691 ) -> Tuple[db.DB, bytes, _FromKeyFunc, _ResultsFromKeyFunc]:
692 subject, predicate, object = spo
693 _to_string = self._to_string
694 # NOTE on type errors: this is because the same variable is used with different types.
695 if context is not None:
696 # type error: Incompatible types in assignment (expression has type "str", variable has type "Optional[Graph]")
697 context = _to_string(context, txn=txn) # type: ignore[assignment]
698 i = 0
699 if subject is not None:
700 i += 1
701 # type error: Incompatible types in assignment (expression has type "str", variable has type "Node")
702 subject = _to_string(subject, txn=txn) # type: ignore[assignment]
703 if predicate is not None:
704 i += 2
705 # type error: Incompatible types in assignment (expression has type "str", variable has type "Node")
706 predicate = _to_string(predicate, txn=txn) # type: ignore[assignment]
707 if object is not None:
708 i += 4
709 # type error: Incompatible types in assignment (expression has type "str", variable has type "Node")
710 object = _to_string(object, txn=txn) # type: ignore[assignment]
711 index, prefix_func, from_key, results_from_key = self.__lookup_dict[i]
712 # print (subject, predicate, object), context, prefix_func, index
713 # #DEBUG
714 # type error: Argument 1 has incompatible type "Tuple[Node, Node, Node]"; expected "Tuple[str, str, str]"
715 # type error: Argument 2 has incompatible type "Optional[Graph]"; expected "Optional[str]"
716 prefix = bb("^".join(prefix_func((subject, predicate, object), context))) # type: ignore[arg-type]
717 return index, prefix, from_key, results_from_key
718
719
720def to_key_func(i: int) -> _ToKeyFunc:
721 def to_key(triple: Tuple[bytes, bytes, bytes], context: bytes) -> bytes:
722 "Takes a string; returns key"
723 return "^".encode("latin-1").join(
724 (
725 context,
726 triple[i % 3],
727 triple[(i + 1) % 3],
728 triple[(i + 2) % 3],
729 "".encode("latin-1"),
730 )
731 ) # "" to tac on the trailing ^
732
733 return to_key
734
735
736def from_key_func(i: int) -> _FromKeyFunc:
737 def from_key(key: bytes) -> Tuple[bytes, bytes, bytes, bytes]:
738 "Takes a key; returns string"
739 parts = key.split("^".encode("latin-1"))
740 return (
741 parts[0],
742 parts[(3 - i + 0) % 3 + 1],
743 parts[(3 - i + 1) % 3 + 1],
744 parts[(3 - i + 2) % 3 + 1],
745 )
746
747 return from_key
748
749
750def results_from_key_func(
751 i: int, from_string: Callable[[bytes], Node]
752) -> _ResultsFromKeyFunc:
753 def from_key(
754 key: bytes,
755 subject: Optional[Node],
756 predicate: Optional[Node],
757 object: Optional[Node],
758 contexts_value: bytes,
759 ) -> Tuple[Tuple[Node, Node, Node], Generator[Node, None, None]]:
760 "Takes a key and subject, predicate, object; returns tuple for yield"
761 parts = key.split("^".encode("latin-1"))
762 if subject is None:
763 # TODO: i & 1: # dis assemble and/or measure to see which is faster
764 # subject is None or i & 1
765 s = from_string(parts[(3 - i + 0) % 3 + 1])
766 else:
767 s = subject
768 if predicate is None: # i & 2:
769 p = from_string(parts[(3 - i + 1) % 3 + 1])
770 else:
771 p = predicate
772 if object is None: # i & 4:
773 o = from_string(parts[(3 - i + 2) % 3 + 1])
774 else:
775 o = object
776 return (
777 (s, p, o),
778 (from_string(c) for c in contexts_value.split("^".encode("latin-1")) if c),
779 )
780
781 return from_key
782
783
784# TODO: Remove unused
785def readable_index(i: int) -> str:
786 # type error: Unpacking a string is disallowed
787 s, p, o = "?" * 3 # type: ignore[misc]
788 if i & 1:
789 s = "s"
790 if i & 2:
791 p = "p"
792 if i & 4:
793 o = "o"
794 return "%s,%s,%s" % (s, p, o)