Source code

Download source archive ↓

src/rdflib/plugins/parsers/ntriples.py

1"""
2N-Triples Parser
3License: GPL 2, W3C, BSD, or MIT
4Author: Sean B. Palmer, inamidst.com
5"""
6
7from __future__ import annotations
8
9import codecs
10import re
11from io import BytesIO, StringIO, TextIOBase
12from typing import (
13    IO,
14    TYPE_CHECKING,
15    Any,
16    Match,
17    MutableMapping,
18    Optional,
19    Pattern,
20    TextIO,
21    Union,
22)
23
24from rdflib.compat import _string_escape_map, decodeUnicodeEscape
25from rdflib.exceptions import ParserError as ParseError
26from rdflib.parser import InputSource, Parser
27from rdflib.term import BNode as bNode
28from rdflib.term import Literal, URIRef
29from rdflib.term import URIRef as URI  # noqa: N814
30
31if TYPE_CHECKING:
32    import typing_extensions as te
33
34    from rdflib.graph import Graph, _ObjectType, _PredicateType, _SubjectType
35
36__all__ = [
37    "unquote",
38    "uriquote",
39    "W3CNTriplesParser",
40    "NTGraphSink",
41    "NTParser",
42    "DummySink",
43]
44
45uriref = r'<([^:]+:[^\s"<>]*)>'
46literal = r'"([^"\\]*(?:\\.[^"\\]*)*)"'
47litinfo = r"(?:@([a-zA-Z]+(?:-[a-zA-Z0-9]+)*)|\^\^" + uriref + r")?"
48
49r_line = re.compile(r"([^\r\n]*)(?:\r\n|\r|\n)")
50r_wspace = re.compile(r"[ \t]*")
51r_wspaces = re.compile(r"[ \t]+")
52r_tail = re.compile(r"[ \t]*\.[ \t]*(#.*)?")
53r_uriref = re.compile(uriref)
54r_nodeid = re.compile(r"_:([A-Za-z0-9_:]([-A-Za-z0-9_:\.]*[-A-Za-z0-9_:])?)")
55r_literal = re.compile(literal + litinfo)
56
57bufsiz = 2048
58validate = False
59
60
61class DummySink:
62    def __init__(self):
63        self.length = 0
64
65    def triple(self, s, p, o):
66        self.length += 1
67        print(s, p, o)
68
69
70r_safe = re.compile(r"([\x20\x21\x23-\x5B\x5D-\x7E]+)")
71r_quot = re.compile(r"""\\([tbnrf"'\\])""")
72r_uniquot = re.compile(r"\\u([0-9A-Fa-f]{4})|\\U([0-9A-Fa-f]{8})")
73
74
75def unquote(s: str) -> str:
76    """Unquote an N-Triples string."""
77    if not validate:
78        if isinstance(s, str):  # nquads
79            s = decodeUnicodeEscape(s)
80        else:
81            s = s.decode("unicode-escape")  # type: ignore[unreachable]
82
83        return s
84    else:
85        result = []
86        while s:
87            m = r_safe.match(s)
88            if m:
89                s = s[m.end() :]
90                result.append(m.group(1))
91                continue
92
93            m = r_quot.match(s)
94            if m:
95                s = s[2:]
96                result.append(_string_escape_map[m.group(1)])
97                continue
98
99            m = r_uniquot.match(s)
100            if m:
101                s = s[m.end() :]
102                u, U = m.groups()  # noqa: N806
103                codepoint = int(u or U, 16)
104                if codepoint > 0x10FFFF:
105                    raise ParseError("Disallowed codepoint: %08X" % codepoint)
106                result.append(chr(codepoint))
107            elif s.startswith("\\"):
108                raise ParseError("Illegal escape at: %s..." % s[:10])
109            else:
110                raise ParseError("Illegal literal character: %r" % s[0])
111        return "".join(result)
112
113
114r_hibyte = re.compile(r"([\x80-\xFF])")
115
116
117def uriquote(uri: str) -> str:
118    if not validate:
119        return uri
120    else:
121        return r_hibyte.sub(lambda m: "%%%02X" % ord(m.group(1)), uri)
122
123
124_BNodeContextType = MutableMapping[str, bNode]
125
126
127class W3CNTriplesParser:
128    """An N-Triples Parser.
129
130    This is a legacy-style Triples parser for NTriples provided by W3C
131
132    Example:
133        ```python
134        p = W3CNTriplesParser(sink=MySink())
135        sink = p.parse(f) # file; use parsestring for a string
136        ```
137
138    To define a context in which blank node identifiers refer to the same blank node
139    across instances of NTriplesParser, pass the same dict as `bnode_context` to each
140    instance. By default, a new blank node context is created for each instance of
141    `W3CNTriplesParser`.
142    """
143
144    __slots__ = ("_bnode_ids", "sink", "buffer", "file", "line", "skolemize")
145
146    def __init__(
147        self,
148        sink: Optional[Union[DummySink, NTGraphSink]] = None,
149        bnode_context: Optional[_BNodeContextType] = None,
150    ):
151        self.skolemize = False
152
153        if bnode_context is not None:
154            self._bnode_ids = bnode_context
155        else:
156            self._bnode_ids = {}
157
158        self.sink: Union[DummySink, NTGraphSink]
159        if sink is not None:
160            self.sink = sink
161        else:
162            self.sink = DummySink()
163
164        self.buffer: Optional[str] = None
165        self.file: Optional[Union[TextIO, codecs.StreamReader]] = None
166        self.line: Optional[str] = ""
167
168    def parse(
169        self,
170        f: Union[TextIO, IO[bytes], codecs.StreamReader],
171        bnode_context: Optional[_BNodeContextType] = None,
172        skolemize: bool = False,
173    ) -> Union[DummySink, NTGraphSink]:
174        """Parse f as an N-Triples file.
175
176        Args:
177            f: The N-Triples source
178            bnode_context: A dict mapping blank node identifiers (e.g., `a` in `_:a`)
179                to [`BNode`][rdflib.term.BNode] instances. An empty dict can be
180                passed in to define a distinct context for a given call to
181                `parse`.
182            skolemize: Whether to skolemize blank nodes
183
184        Returns:
185            The sink containing the parsed triples
186        """
187
188        if not hasattr(f, "read"):
189            raise ParseError("Item to parse must be a file-like object.")
190
191        if not hasattr(f, "encoding") and not hasattr(f, "charbuffer"):
192            # someone still using a bytestream here?
193            f = codecs.getreader("utf-8")(f)
194
195        self.skolemize = skolemize
196        self.file = f  # type: ignore[assignment]
197        self.buffer = ""
198        while True:
199            self.line = self.readline()
200            if self.line is None:
201                break
202            try:
203                self.parseline(bnode_context=bnode_context)
204            except ParseError:
205                raise ParseError("Invalid line: {}".format(self.line))
206        return self.sink
207
208    def parsestring(self, s: Union[bytes, bytearray, str], **kwargs) -> None:
209        """Parse s as an N-Triples string."""
210        if not isinstance(s, (str, bytes, bytearray)):
211            raise ParseError("Item to parse must be a string instance.")
212        f: Union[codecs.StreamReader, StringIO]
213        if isinstance(s, (bytes, bytearray)):
214            f = codecs.getreader("utf-8")(BytesIO(s))
215        else:
216            f = StringIO(s)
217        self.parse(f, **kwargs)
218
219    def readline(self) -> Optional[str]:
220        """Read an N-Triples line from buffered input."""
221        # N-Triples lines end in either CRLF, CR, or LF
222        # Therefore, we can't just use f.readline()
223        if not self.buffer:
224            # type error: Item "None" of "Union[TextIO, StreamReader, None]" has no attribute "read"
225            buffer = self.file.read(bufsiz)  # type: ignore[union-attr]
226            if not buffer:
227                return None
228            self.buffer = buffer
229
230        while True:
231            m = r_line.match(self.buffer)
232            if m:  # the more likely prospect
233                self.buffer = self.buffer[m.end() :]
234                return m.group(1)
235            else:
236                # type error: Item "None" of "Union[TextIO, StreamReader, None]" has no attribute "read"
237                buffer = self.file.read(bufsiz)  # type: ignore[union-attr]
238                if not buffer and not self.buffer.isspace():
239                    # Last line does not need to be terminated with a newline
240                    buffer += "\n"
241                elif not buffer:
242                    return None
243                self.buffer += buffer
244
245    def parseline(self, bnode_context: Optional[_BNodeContextType] = None) -> None:
246        self.eat(r_wspace)
247        if (not self.line) or self.line.startswith("#"):
248            return  # The line is empty or a comment
249
250        subject = self.subject(bnode_context)
251        self.eat(r_wspaces)
252
253        predicate = self.predicate()
254        self.eat(r_wspaces)
255
256        object_ = self.object(bnode_context)
257        self.eat(r_tail)
258
259        if self.line:
260            raise ParseError("Trailing garbage: {}".format(self.line))
261        self.sink.triple(subject, predicate, object_)
262
263    def peek(self, token: str) -> bool:
264        return self.line.startswith(token)  # type: ignore[union-attr]
265
266    def eat(self, pattern: Pattern[str]) -> Match[str]:
267        m = pattern.match(self.line)  # type: ignore[arg-type]
268        if not m:  # @@ Why can't we get the original pattern?
269            # print(dir(pattern))
270            # print repr(self.line), type(self.line)
271            raise ParseError("Failed to eat %s at %s" % (pattern.pattern, self.line))
272        self.line = self.line[m.end() :]  # type: ignore[index]
273        return m
274
275    def subject(self, bnode_context=None) -> Union[bNode, URIRef]:
276        # @@ Consider using dictionary cases
277        subj = self.uriref() or self.nodeid(bnode_context)
278        if not subj:
279            raise ParseError("Subject must be uriref or nodeID")
280        return subj
281
282    def predicate(self) -> Union[bNode, URIRef]:
283        pred = self.uriref()
284        if not pred:
285            raise ParseError("Predicate must be uriref")
286        return pred
287
288    def object(
289        self, bnode_context: Optional[_BNodeContextType] = None
290    ) -> Union[URI, bNode, Literal]:
291        objt = self.uriref() or self.nodeid(bnode_context) or self.literal()
292        if objt is False:
293            raise ParseError("Unrecognised object type")
294        return objt
295
296    def uriref(self) -> Union[te.Literal[False], URI]:
297        if self.peek("<"):
298            uri = self.eat(r_uriref).group(1)
299            uri = unquote(uri)
300            uri = uriquote(uri)
301            return URI(uri)
302        return False
303
304    def nodeid(
305        self, bnode_context: Optional[_BNodeContextType] = None
306    ) -> Union[te.Literal[False], bNode, URI]:
307        if self.peek("_"):
308            if self.skolemize:
309                bnode_id = self.eat(r_nodeid).group(1)
310                return bNode(bnode_id).skolemize()
311
312            else:
313                # Fix for https://github.com/RDFLib/rdflib/issues/204
314                if bnode_context is None:
315                    bnode_context = self._bnode_ids
316                bnode_id = self.eat(r_nodeid).group(1)
317                new_id = bnode_context.get(bnode_id, None)
318                if new_id is not None:
319                    # Re-map to id specific to this doc
320                    return bNode(new_id)
321                else:
322                    # Replace with freshly-generated document-specific BNode id
323                    bnode = bNode()
324                    # Store the mapping
325                    bnode_context[bnode_id] = bnode
326                    return bnode
327        return False
328
329    def literal(self) -> Union[te.Literal[False], Literal]:
330        if self.peek('"'):
331            lit, lang, dtype = self.eat(r_literal).groups()
332            if lang:
333                lang = lang
334            else:
335                lang = None
336            if dtype:
337                dtype = unquote(dtype)
338                dtype = uriquote(dtype)
339                dtype = URI(dtype)
340            else:
341                dtype = None
342            if lang and dtype:
343                raise ParseError("Can't have both a language and a datatype")
344            lit = unquote(lit)
345            return Literal(lit, lang, dtype)
346        return False
347
348
349class NTGraphSink:
350    __slots__ = ("g",)
351
352    def __init__(self, graph: Graph):
353        self.g = graph
354
355    def triple(self, s: _SubjectType, p: _PredicateType, o: _ObjectType) -> None:
356        self.g.add((s, p, o))
357
358
359class NTParser(Parser):
360    """Parser for the N-Triples format, often stored with the .nt extension.
361
362    See http://www.w3.org/TR/rdf-testcases/#ntriples
363    """
364
365    __slots__ = ()
366
367    @classmethod
368    def parse(cls, source: InputSource, sink: Graph, **kwargs: Any) -> None:
369        """Parse the NT format.
370
371        Args:
372            source: The source of NT-formatted data
373            sink: Where to send parsed triples
374            **kwargs: Additional arguments to pass to `W3CNTriplesParser.parse`
375        """
376        f: Union[TextIO, IO[bytes], codecs.StreamReader]
377        f = source.getCharacterStream()
378        if not f:
379            b = source.getByteStream()
380            # TextIOBase includes: StringIO and TextIOWrapper
381            if isinstance(b, TextIOBase):
382                # f is not really a ByteStream, but a CharacterStream
383                f = b  # type: ignore[assignment]
384            else:
385                # since N-Triples 1.1 files can and should be utf-8 encoded
386                f = codecs.getreader("utf-8")(b)
387        parser = W3CNTriplesParser(NTGraphSink(sink))
388        parser.parse(f, **kwargs)
389        f.close()