Source code

Download source archive ↓

src/rdflib/tools/chunk_serializer.py

1"""
2This file provides a single function `serialize_in_chunks()` which can serialize a
3Graph into a number of NT files with a maximum number of triples or maximum file size.
4
5There is an option to preserve any prefixes declared for the original graph in the first
6file, which will be a Turtle file.
7"""
8
9from __future__ import annotations
10
11from contextlib import ExitStack, contextmanager
12from pathlib import Path
13from typing import TYPE_CHECKING, BinaryIO, Generator, Optional, Tuple
14
15from rdflib.graph import Graph
16from rdflib.plugins.serializers.nt import _nt_row
17
18# from rdflib.term import Literal
19
20# if TYPE_CHECKING:
21#     from rdflib.graph import _TriplePatternType
22
23__all__ = ["serialize_in_chunks"]
24
25
26def serialize_in_chunks(
27    g: Graph,
28    max_triples: int = 10000,
29    max_file_size_kb: Optional[int] = None,
30    file_name_stem: str = "chunk",
31    output_dir: Optional[Path] = None,
32    write_prefixes: bool = False,
33) -> None:
34    """Serializes a given Graph into a series of n-triples with a given length.
35
36    Args:
37        g: The graph to serialize.
38        max_file_size_kb: Maximum size per NT file in kB (1,000 bytes)
39            Equivalent to ~6,000 triples, depending on Literal sizes.
40        max_triples: Maximum size per NT file in triples
41            Equivalent to lines in file.
42            If both this parameter and max_file_size_kb are set, max_file_size_kb will be used.
43        file_name_stem: Prefix of each file name.
44            e.g. "chunk" = chunk_000001.nt, chunk_000002.nt...
45        output_dir: The directory you want the files to be written to.
46        write_prefixes: The first file created is a Turtle file containing original graph prefixes.
47
48    See `../test/test_tools/test_chunk_serializer.py` for examples of this in use.
49    """
50
51    if output_dir is None:
52        output_dir = Path.cwd()
53
54    if not output_dir.is_dir():
55        raise ValueError(
56            "If you specify an output_dir, it must actually be a directory!"
57        )
58
59    @contextmanager
60    def _start_new_file(file_no: int) -> Generator[Tuple[Path, BinaryIO], None, None]:
61        if TYPE_CHECKING:
62            # this is here because mypy gets a bit confused
63            assert output_dir is not None
64        fp = Path(output_dir) / f"{file_name_stem}_{str(file_no).zfill(6)}.nt"
65        with open(fp, "wb") as fh:
66            yield fp, fh
67
68    def _serialize_prefixes(g: Graph) -> str:
69        pres = []
70        for k, v in g.namespace_manager.namespaces():
71            pres.append(f"PREFIX {k}: <{v}>")
72
73        return "\n".join(sorted(pres)) + "\n"
74
75    if write_prefixes:
76        with open(
77            Path(output_dir) / f"{file_name_stem}_000000.ttl", "w", encoding="utf-8"
78        ) as fh:
79            fh.write(_serialize_prefixes(g))
80
81    bytes_written = 0
82    with ExitStack() as xstack:
83        if max_file_size_kb is not None:
84            max_file_size = max_file_size_kb * 1000
85            file_no = 1 if write_prefixes else 0
86            for i, t in enumerate(g.triples((None, None, None))):
87                row_bytes = _nt_row(t).encode("utf-8")
88                if len(row_bytes) > max_file_size:
89                    raise ValueError(
90                        # type error: Unsupported operand types for / ("bytes" and "int")
91                        f"cannot write triple {t!r} as it's serialized size of {row_bytes / 1000} exceeds max_file_size_kb = {max_file_size_kb}"  # type: ignore[operator]
92                    )
93                if i == 0:
94                    fp, fhb = xstack.enter_context(_start_new_file(file_no))
95                    bytes_written = 0
96                elif (bytes_written + len(row_bytes)) >= max_file_size:
97                    file_no += 1
98                    fp, fhb = xstack.enter_context(_start_new_file(file_no))
99                    bytes_written = 0
100
101                bytes_written += fhb.write(row_bytes)
102
103        else:
104            # count the triples in the graph
105            graph_length = len(g)
106
107            if graph_length <= max_triples:
108                # the graph is less than max so just NT serialize the whole thing
109                g.serialize(
110                    destination=Path(output_dir) / f"{file_name_stem}_all.nt",
111                    format="nt",
112                )
113            else:
114                # graph_length is > max_lines, make enough files for all graph
115                # no_files = math.ceil(graph_length / max_triples)
116                file_no = 1 if write_prefixes else 0
117                for i, t in enumerate(g.triples((None, None, None))):
118                    if i % max_triples == 0:
119                        fp, fhb = xstack.enter_context(_start_new_file(file_no))
120                        file_no += 1
121                    fhb.write(_nt_row(t).encode("utf-8"))
122            return