src/rdflib/tools/chunk_serializer.py
1"""
2This file provides a single function `serialize_in_chunks()` which can serialize a
3Graph into a number of NT files with a maximum number of triples or maximum file size.
4
5There is an option to preserve any prefixes declared for the original graph in the first
6file, which will be a Turtle file.
7"""
8
9from __future__ import annotations
10
11from contextlib import ExitStack, contextmanager
12from pathlib import Path
13from typing import TYPE_CHECKING, BinaryIO, Generator, Optional, Tuple
14
15from rdflib.graph import Graph
16from rdflib.plugins.serializers.nt import _nt_row
17
18# from rdflib.term import Literal
19
20# if TYPE_CHECKING:
21# from rdflib.graph import _TriplePatternType
22
23__all__ = ["serialize_in_chunks"]
24
25
26def serialize_in_chunks(
27 g: Graph,
28 max_triples: int = 10000,
29 max_file_size_kb: Optional[int] = None,
30 file_name_stem: str = "chunk",
31 output_dir: Optional[Path] = None,
32 write_prefixes: bool = False,
33) -> None:
34 """Serializes a given Graph into a series of n-triples with a given length.
35
36 Args:
37 g: The graph to serialize.
38 max_file_size_kb: Maximum size per NT file in kB (1,000 bytes)
39 Equivalent to ~6,000 triples, depending on Literal sizes.
40 max_triples: Maximum size per NT file in triples
41 Equivalent to lines in file.
42 If both this parameter and max_file_size_kb are set, max_file_size_kb will be used.
43 file_name_stem: Prefix of each file name.
44 e.g. "chunk" = chunk_000001.nt, chunk_000002.nt...
45 output_dir: The directory you want the files to be written to.
46 write_prefixes: The first file created is a Turtle file containing original graph prefixes.
47
48 See `../test/test_tools/test_chunk_serializer.py` for examples of this in use.
49 """
50
51 if output_dir is None:
52 output_dir = Path.cwd()
53
54 if not output_dir.is_dir():
55 raise ValueError(
56 "If you specify an output_dir, it must actually be a directory!"
57 )
58
59 @contextmanager
60 def _start_new_file(file_no: int) -> Generator[Tuple[Path, BinaryIO], None, None]:
61 if TYPE_CHECKING:
62 # this is here because mypy gets a bit confused
63 assert output_dir is not None
64 fp = Path(output_dir) / f"{file_name_stem}_{str(file_no).zfill(6)}.nt"
65 with open(fp, "wb") as fh:
66 yield fp, fh
67
68 def _serialize_prefixes(g: Graph) -> str:
69 pres = []
70 for k, v in g.namespace_manager.namespaces():
71 pres.append(f"PREFIX {k}: <{v}>")
72
73 return "\n".join(sorted(pres)) + "\n"
74
75 if write_prefixes:
76 with open(
77 Path(output_dir) / f"{file_name_stem}_000000.ttl", "w", encoding="utf-8"
78 ) as fh:
79 fh.write(_serialize_prefixes(g))
80
81 bytes_written = 0
82 with ExitStack() as xstack:
83 if max_file_size_kb is not None:
84 max_file_size = max_file_size_kb * 1000
85 file_no = 1 if write_prefixes else 0
86 for i, t in enumerate(g.triples((None, None, None))):
87 row_bytes = _nt_row(t).encode("utf-8")
88 if len(row_bytes) > max_file_size:
89 raise ValueError(
90 # type error: Unsupported operand types for / ("bytes" and "int")
91 f"cannot write triple {t!r} as it's serialized size of {row_bytes / 1000} exceeds max_file_size_kb = {max_file_size_kb}" # type: ignore[operator]
92 )
93 if i == 0:
94 fp, fhb = xstack.enter_context(_start_new_file(file_no))
95 bytes_written = 0
96 elif (bytes_written + len(row_bytes)) >= max_file_size:
97 file_no += 1
98 fp, fhb = xstack.enter_context(_start_new_file(file_no))
99 bytes_written = 0
100
101 bytes_written += fhb.write(row_bytes)
102
103 else:
104 # count the triples in the graph
105 graph_length = len(g)
106
107 if graph_length <= max_triples:
108 # the graph is less than max so just NT serialize the whole thing
109 g.serialize(
110 destination=Path(output_dir) / f"{file_name_stem}_all.nt",
111 format="nt",
112 )
113 else:
114 # graph_length is > max_lines, make enough files for all graph
115 # no_files = math.ceil(graph_length / max_triples)
116 file_no = 1 if write_prefixes else 0
117 for i, t in enumerate(g.triples((None, None, None))):
118 if i % max_triples == 0:
119 fp, fhb = xstack.enter_context(_start_new_file(file_no))
120 file_no += 1
121 fhb.write(_nt_row(t).encode("utf-8"))
122 return