Source code

Download source archive ↓

src/rdflib/tools/csv2rdf.py

1"""
2A commandline tool for semi-automatically converting CSV to RDF.
3
4See also https://github.com/RDFLib/pyTARQL in the RDFlib family of tools
5
6try: `csv2rdf --help`
7"""
8
9from __future__ import annotations
10
11import codecs
12import configparser
13import csv
14import datetime
15import fileinput
16import getopt
17import re
18import sys
19import time
20import warnings
21from typing import Any, Dict, List, Optional, Tuple, Union
22from urllib.parse import quote
23
24import rdflib
25from rdflib.namespace import RDF, RDFS, split_uri
26from rdflib.term import URIRef
27
28__all__ = ["CSV2RDF"]
29
30HELP = """
31csv2rdf.py \
32    -b <instance-base> \
33    -p <property-base> \
34    [-D <default>] \
35    [-c <classname>] \
36    [-i <identity column(s)>] \
37    [-l <label columns>] \
38    [-s <N>] [-o <output>] \
39    [-f configfile] \
40    [--col<N> <colspec>] \
41    [--prop<N> <property>] \
42    <[-d <delim>] \
43    [-C] [files...]"
44
45Reads csv files from stdin or given files
46if -d is given, use this delimiter
47if -s is given, skips N lines at the start
48Creates a URI from the columns given to -i, or automatically by numbering if
49none is given
50Outputs RDFS labels from the columns given to -l
51if -c is given adds a type triple with the given classname
52if -C is given, the class is defined as rdfs:Class
53Outputs one RDF triple per column in each row.
54Output is in n3 format.
55Output is stdout, unless -o is specified
56
57Long options also supported: \
58    --base, \
59    --propbase, \
60    --ident, \
61    --class, \
62    --label, \
63    --out, \
64    --defineclass
65
66Long options --col0, --col1, ...
67can be used to specify conversion for columns.
68Conversions can be:
69    ignore, float(), int(), split(sep, [more]), uri(base, [class]), date(format)
70
71Long options --prop0, --prop1, ...
72can be used to use specific properties, rather than ones auto-generated
73from the headers
74
75-D sets the default conversion for columns not listed
76
77-f says to read config from a .ini/config file - the file must contain one
78section called csv2rdf, with keys like the long options, i.e.:
79
80[csv2rdf]
81out=output.n3
82base=http://example.org/
83col0=split(";")
84col1=split(";", uri("http://example.org/things/",
85                    "http://xmlns.com/foaf/0.1/Person"))
86col2=float()
87col3=int()
88col4=date("%Y-%b-%d %H:%M:%S")
89
90"""
91
92# bah - ugly global
93uris: Dict[Any, Tuple[URIRef, Optional[URIRef]]] = {}
94
95
96def toProperty(label: str):  # noqa: N802
97    """
98    CamelCase + lowercase initial a string
99
100
101    FIRST_NM => firstNm
102
103    firstNm => firstNm
104
105    """
106    label = re.sub(r"[^\w]", " ", label)
107    label = re.sub("([a-z])([A-Z])", "\\1 \\2", label)
108    # type error: Incompatible types in assignment (expression has type "None", variable has type "BinaryIO")
109    label = label.split(" ")  # type: ignore[assignment]
110    return "".join([label[0].lower()] + [x.capitalize() for x in label[1:]])
111
112
113def toPropertyLabel(label):  # noqa: N802
114    if not label[1:2].isupper():
115        return label[0:1].lower() + label[1:]
116    return label
117
118
119def index(l_: List[int], i: Tuple[int, ...]) -> Tuple[int, ...]:
120    """return a set of indexes from a list
121    >>> index([1,2,3],(0,2))
122    (1, 3)
123    """
124    return tuple([l_[x] for x in i])
125
126
127def csv_reader(csv_data, dialect=csv.excel, **kwargs):
128    csv_reader = csv.reader(csv_data, dialect=dialect, **kwargs)
129    for row in csv_reader:
130        yield row
131
132
133def prefixuri(x, prefix, class_: Optional[URIRef] = None):
134    if prefix:
135        r = rdflib.URIRef(prefix + quote(x.encode("utf8").replace(" ", "_"), safe=""))
136    else:
137        r = rdflib.URIRef(x)
138    uris[x] = (r, class_)
139    return r
140
141
142# meta-language for config
143
144
145class NodeMaker:
146    def range(self):
147        return rdflib.RDFS.Literal
148
149    def __call__(self, x: Any):
150        return rdflib.Literal(x)
151
152
153class NodeUri(NodeMaker):
154    def __init__(self, prefix, class_):
155        self.class_: Optional[URIRef] = None
156        self.prefix = prefix
157        if class_:
158            self.class_ = rdflib.URIRef(class_)
159        else:
160            self.class_ = None
161
162    def __call__(self, x):
163        return prefixuri(x, self.prefix, self.class_)
164
165    def range(self):
166        return self.class_ or rdflib.RDF.Resource
167
168
169class NodeLiteral(NodeMaker):
170    def __init__(self, f=None):
171        self.f = f
172
173
174class NodeFloat(NodeLiteral):
175    def __call__(self, x):
176        if not self.f:
177            return rdflib.Literal(float(x))
178        if callable(self.f):
179            return rdflib.Literal(float(self.f(x)))
180        raise Exception("Function passed to float is not callable")
181
182    def range(self):
183        return rdflib.XSD.double
184
185
186class NodeInt(NodeLiteral):
187    def __call__(self, x):
188        if not self.f:
189            return rdflib.Literal(int(x))
190        if callable(self.f):
191            return rdflib.Literal(int(self.f(x)))
192        raise Exception("Function passed to int is not callable")
193
194    def range(self):
195        return rdflib.XSD.int
196
197
198class NodeBool(NodeLiteral):
199    def __call__(self, x):
200        if not self.f:
201            return rdflib.Literal(bool(x))
202        if callable(self.f):
203            return rdflib.Literal(bool(self.f(x)))
204        raise Exception("Function passed to bool is not callable")
205
206    def range(self):
207        return rdflib.XSD.bool
208
209
210class NodeReplace(NodeMaker):
211    def __init__(self, a, b):
212        self.a = a
213        self.b = b
214
215    def __call__(self, x):
216        return x.replace(self.a, self.b)
217
218
219class NodeDate(NodeLiteral):
220    def __call__(self, x):
221        return rdflib.Literal(datetime.datetime.strptime(x, self.f))
222
223    def range(self):
224        return rdflib.XSD.dateTime
225
226
227class NodeSplit(NodeMaker):
228    def __init__(self, sep, f):
229        self.sep = sep
230        self.f = f
231
232    def __call__(self, x):
233        if not self.f:
234            self.f = rdflib.Literal
235        if not callable(self.f):
236            raise Exception("Function passed to split is not callable!")
237        return [self.f(y.strip()) for y in x.split(self.sep) if y.strip() != ""]
238
239    def range(self):
240        if self.f and isinstance(self.f, NodeMaker):
241            return self.f.range()
242        return NodeMaker.range(self)
243
244
245default_node_make = NodeMaker()
246
247
248def _config_ignore(*args, **kwargs):
249    return "ignore"
250
251
252def _config_uri(prefix=None, class_=None):
253    return NodeUri(prefix, class_)
254
255
256def _config_literal():
257    return NodeLiteral()
258
259
260def _config_float(f=None):
261    return NodeFloat(f)
262
263
264def _config_replace(a, b):
265    return NodeReplace(a, b)
266
267
268def _config_int(f=None):
269    return NodeInt(f)
270
271
272def _config_bool(f=None):
273    return NodeBool(f)
274
275
276def _config_date(format_):
277    return NodeDate(format_)
278
279
280def _config_split(sep=None, f=None):
281    return NodeSplit(sep, f)
282
283
284config_functions = {
285    "ignore": _config_ignore,
286    "uri": _config_uri,
287    "literal": _config_literal,
288    "float": _config_float,
289    "int": _config_int,
290    "date": _config_date,
291    "split": _config_split,
292    "replace": _config_replace,
293    "bool": _config_bool,
294}
295
296
297def column(v):
298    """Return a function for column mapping"""
299
300    return eval(v, config_functions)
301
302
303class CSV2RDF:
304    def __init__(self):
305        self.CLASS = None
306        self.BASE = None
307        self.PROPBASE = None
308        self.IDENT: Union[Tuple[str, ...], str] = "auto"
309        self.LABEL = None
310        self.DEFINECLASS = False
311        self.SKIP = 0
312        self.DELIM = ","
313        self.DEFAULT = None
314
315        self.COLUMNS = {}
316        self.PROPS = {}
317
318        self.OUT = sys.stdout
319
320        self.triples = 0
321
322    def triple(self, s, p, o):
323        self.OUT.write("%s %s %s .\n" % (s.n3(), p.n3(), o.n3()))
324        self.triples += 1
325
326    def convert(self, csvreader):
327        start = time.time()
328
329        if self.OUT:
330            sys.stderr.write("Output to %s\n" % self.OUT.name)
331
332        if self.IDENT != "auto" and not isinstance(self.IDENT, tuple):
333            self.IDENT = (self.IDENT,)
334
335        if not self.BASE:
336            warnings.warn("No base given, using http://example.org/instances/")
337            self.BASE = rdflib.Namespace("http://example.org/instances/")
338
339        if not self.PROPBASE:
340            warnings.warn("No property base given, using http://example.org/property/")
341            self.PROPBASE = rdflib.Namespace("http://example.org/props/")
342
343        # skip lines at the start
344        for x in range(self.SKIP):
345            next(csvreader)
346
347        # read header line
348        header_labels = list(next(csvreader))
349        headers = dict(enumerate([self.PROPBASE[toProperty(x)] for x in header_labels]))
350        # override header properties if some are given
351        for k, v in self.PROPS.items():
352            headers[k] = v
353            header_labels[k] = split_uri(v)[1]
354
355        if self.DEFINECLASS:
356            # output class/property definitions
357            self.triple(self.CLASS, RDF.type, RDFS.Class)
358            for i in range(len(headers)):
359                h, l_ = headers[i], header_labels[i]
360                if h == "" or l_ == "":
361                    continue
362                if self.COLUMNS.get(i, self.DEFAULT) == "ignore":
363                    continue
364                self.triple(h, RDF.type, RDF.Property)
365                self.triple(h, RDFS.label, rdflib.Literal(toPropertyLabel(l_)))
366                self.triple(h, RDFS.domain, self.CLASS)
367                self.triple(
368                    h, RDFS.range, self.COLUMNS.get(i, default_node_make).range()
369                )
370
371        rows = 0
372        for l_ in csvreader:
373            try:
374                if self.IDENT == "auto":
375                    uri = self.BASE["%d" % rows]
376                else:
377                    uri = self.BASE[
378                        "_".join(
379                            [
380                                # type error: "int" has no attribute "encode"
381                                quote(x.encode("utf8").replace(" ", "_"), safe="")  # type: ignore[attr-defined]
382                                # type error: Argument 2 to "index" has incompatible type "Union[Tuple[str, ...], str]"; expected "Tuple[int, ...]"
383                                for x in index(l_, self.IDENT)  # type: ignore[arg-type]
384                            ]
385                        )
386                    ]
387
388                if self.LABEL:
389                    self.triple(
390                        # type error: Argument 1 to "join" of "str" has incompatible type "Tuple[int, ...]"; expected "Iterable[str]"
391                        uri,
392                        RDFS.label,
393                        rdflib.Literal(" ".join(index(l_, self.LABEL))),  # type: ignore[arg-type]
394                    )
395
396                if self.CLASS:
397                    # type triple
398                    self.triple(uri, RDF.type, self.CLASS)
399
400                for i, x in enumerate(l_):
401                    # type error: "int" has no attribute "strip"
402                    x = x.strip()  # type: ignore[attr-defined]
403                    if x != "":
404                        if self.COLUMNS.get(i, self.DEFAULT) == "ignore":
405                            continue
406                        try:
407                            o = self.COLUMNS.get(i, rdflib.Literal)(x)
408                            if isinstance(o, list):
409                                for _o in o:
410                                    self.triple(uri, headers[i], _o)
411                            else:
412                                self.triple(uri, headers[i], o)
413
414                        except Exception as e:
415                            warnings.warn(
416                                "Could not process value for column "
417                                + "%d:%s in row %d, ignoring: %s "
418                                # type error: "Exception" has no attribute "message"
419                                % (i, headers[i], rows, e.message)  # type: ignore[attr-defined]
420                            )
421
422                rows += 1
423                if rows % 100000 == 0:
424                    sys.stderr.write(
425                        "%d rows, %d triples, elapsed %.2fs.\n"
426                        % (rows, self.triples, time.time() - start)
427                    )
428            except Exception:
429                sys.stderr.write("Error processing line: %d\n" % rows)
430                raise
431
432        # output types/labels for generated URIs
433        classes = set()
434        # type error: Incompatible types in assignment (expression has type "Tuple[URIRef, Optional[URIRef]]", variable has type "int")
435        for l_, x in uris.items():  # type: ignore[assignment]
436            # type error: "int" object is not iterable
437            u, c = x  # type: ignore[misc]
438            # type error: Cannot determine type of "u"
439            self.triple(u, RDFS.label, rdflib.Literal(l_))  # type: ignore[has-type]
440            # type error: Cannot determine type of "c"
441            if c:  # type: ignore[has-type]
442                # type error: Cannot determine type of "c"
443                c = rdflib.URIRef(c)  # type: ignore[has-type]
444                classes.add(c)
445                # type error: Cannot determine type of "u"
446                self.triple(u, RDF.type, c)  # type: ignore[has-type]
447
448        for c in classes:
449            self.triple(c, RDF.type, RDFS.Class)
450
451        self.OUT.close()
452        sys.stderr.write("Converted %d rows into %d triples.\n" % (rows, self.triples))
453        sys.stderr.write("Took %.2f seconds.\n" % (time.time() - start))
454
455
456def main():
457    csv2rdf = CSV2RDF()
458
459    opts: Union[Dict[str, str], List[Tuple[str, str]]]
460    opts, files = getopt.getopt(
461        sys.argv[1:],
462        "hc:b:p:i:o:Cf:l:s:d:D:",
463        [
464            "out=",
465            "base=",
466            "delim=",
467            "propbase=",
468            "class=",
469            "default=" "ident=",
470            "label=",
471            "skip=",
472            "defineclass",
473            "help",
474        ],
475    )
476    opts = dict(opts)
477
478    if "-h" in opts or "--help" in opts:
479        print(HELP)
480        sys.exit(-1)
481
482    if "-f" in opts:
483        config = configparser.ConfigParser()
484        config.read_file(open(opts["-f"]))
485        for k, v in config.items("csv2rdf"):
486            if k == "out":
487                csv2rdf.OUT = codecs.open(v, "w", "utf-8")
488            elif k == "base":
489                csv2rdf.BASE = rdflib.Namespace(v)
490            elif k == "propbase":
491                csv2rdf.PROPBASE = rdflib.Namespace(v)
492            elif k == "class":
493                csv2rdf.CLASS = rdflib.URIRef(v)
494            elif k == "defineclass":
495                csv2rdf.DEFINECLASS = bool(v)
496            elif k == "ident":
497                csv2rdf.IDENT = eval(v)
498            elif k == "label":
499                csv2rdf.LABEL = eval(v)
500            elif k == "delim":
501                csv2rdf.DELIM = v
502            elif k == "skip":
503                csv2rdf.SKIP = int(v)
504            elif k == "default":
505                csv2rdf.DEFAULT = column(v)
506            elif k.startswith("col"):
507                csv2rdf.COLUMNS[int(k[3:])] = column(v)
508            elif k.startswith("prop"):
509                csv2rdf.PROPS[int(k[4:])] = rdflib.URIRef(v)
510
511    if "-o" in opts:
512        csv2rdf.OUT = codecs.open(opts["-o"], "w", "utf-8")
513    if "--out" in opts:
514        csv2rdf.OUT = codecs.open(opts["--out"], "w", "utf-8")
515
516    if "-b" in opts:
517        csv2rdf.BASE = rdflib.Namespace(opts["-b"])
518    if "--base" in opts:
519        csv2rdf.BASE = rdflib.Namespace(opts["--base"])
520
521    if "-d" in opts:
522        csv2rdf.DELIM = opts["-d"]
523    if "--delim" in opts:
524        csv2rdf.DELIM = opts["--delim"]
525
526    if "-D" in opts:
527        csv2rdf.DEFAULT = column(opts["-D"])
528    if "--default" in opts:
529        csv2rdf.DEFAULT = column(opts["--default"])
530
531    if "-p" in opts:
532        csv2rdf.PROPBASE = rdflib.Namespace(opts["-p"])
533    if "--propbase" in opts:
534        csv2rdf.PROPBASE = rdflib.Namespace(opts["--propbase"])
535
536    if "-l" in opts:
537        csv2rdf.LABEL = eval(opts["-l"])
538    if "--label" in opts:
539        csv2rdf.LABEL = eval(opts["--label"])
540
541    if "-i" in opts:
542        csv2rdf.IDENT = eval(opts["-i"])
543    if "--ident" in opts:
544        csv2rdf.IDENT = eval(opts["--ident"])
545
546    if "-s" in opts:
547        csv2rdf.SKIP = int(opts["-s"])
548    if "--skip" in opts:
549        csv2rdf.SKIP = int(opts["--skip"])
550
551    if "-c" in opts:
552        csv2rdf.CLASS = rdflib.URIRef(opts["-c"])
553    if "--class" in opts:
554        csv2rdf.CLASS = rdflib.URIRef(opts["--class"])
555
556    for k, v in opts.items():
557        if k.startswith("--col"):
558            csv2rdf.COLUMNS[int(k[5:])] = column(v)
559        elif k.startswith("--prop"):
560            csv2rdf.PROPS[int(k[6:])] = rdflib.URIRef(v)
561
562    if csv2rdf.CLASS and ("-C" in opts or "--defineclass" in opts):
563        csv2rdf.DEFINECLASS = True
564
565    csv2rdf.convert(csv_reader(fileinput.input(files), delimiter=csv2rdf.DELIM))
566
567
568if __name__ == "__main__":
569    main()