src/rdflib/tools/csv2rdf.py
1"""
2A commandline tool for semi-automatically converting CSV to RDF.
3
4See also https://github.com/RDFLib/pyTARQL in the RDFlib family of tools
5
6try: `csv2rdf --help`
7"""
8
9from __future__ import annotations
10
11import codecs
12import configparser
13import csv
14import datetime
15import fileinput
16import getopt
17import re
18import sys
19import time
20import warnings
21from typing import Any, Dict, List, Optional, Tuple, Union
22from urllib.parse import quote
23
24import rdflib
25from rdflib.namespace import RDF, RDFS, split_uri
26from rdflib.term import URIRef
27
28__all__ = ["CSV2RDF"]
29
30HELP = """
31csv2rdf.py \
32 -b <instance-base> \
33 -p <property-base> \
34 [-D <default>] \
35 [-c <classname>] \
36 [-i <identity column(s)>] \
37 [-l <label columns>] \
38 [-s <N>] [-o <output>] \
39 [-f configfile] \
40 [--col<N> <colspec>] \
41 [--prop<N> <property>] \
42 <[-d <delim>] \
43 [-C] [files...]"
44
45Reads csv files from stdin or given files
46if -d is given, use this delimiter
47if -s is given, skips N lines at the start
48Creates a URI from the columns given to -i, or automatically by numbering if
49none is given
50Outputs RDFS labels from the columns given to -l
51if -c is given adds a type triple with the given classname
52if -C is given, the class is defined as rdfs:Class
53Outputs one RDF triple per column in each row.
54Output is in n3 format.
55Output is stdout, unless -o is specified
56
57Long options also supported: \
58 --base, \
59 --propbase, \
60 --ident, \
61 --class, \
62 --label, \
63 --out, \
64 --defineclass
65
66Long options --col0, --col1, ...
67can be used to specify conversion for columns.
68Conversions can be:
69 ignore, float(), int(), split(sep, [more]), uri(base, [class]), date(format)
70
71Long options --prop0, --prop1, ...
72can be used to use specific properties, rather than ones auto-generated
73from the headers
74
75-D sets the default conversion for columns not listed
76
77-f says to read config from a .ini/config file - the file must contain one
78section called csv2rdf, with keys like the long options, i.e.:
79
80[csv2rdf]
81out=output.n3
82base=http://example.org/
83col0=split(";")
84col1=split(";", uri("http://example.org/things/",
85 "http://xmlns.com/foaf/0.1/Person"))
86col2=float()
87col3=int()
88col4=date("%Y-%b-%d %H:%M:%S")
89
90"""
91
92# bah - ugly global
93uris: Dict[Any, Tuple[URIRef, Optional[URIRef]]] = {}
94
95
96def toProperty(label: str): # noqa: N802
97 """
98 CamelCase + lowercase initial a string
99
100
101 FIRST_NM => firstNm
102
103 firstNm => firstNm
104
105 """
106 label = re.sub(r"[^\w]", " ", label)
107 label = re.sub("([a-z])([A-Z])", "\\1 \\2", label)
108 # type error: Incompatible types in assignment (expression has type "None", variable has type "BinaryIO")
109 label = label.split(" ") # type: ignore[assignment]
110 return "".join([label[0].lower()] + [x.capitalize() for x in label[1:]])
111
112
113def toPropertyLabel(label): # noqa: N802
114 if not label[1:2].isupper():
115 return label[0:1].lower() + label[1:]
116 return label
117
118
119def index(l_: List[int], i: Tuple[int, ...]) -> Tuple[int, ...]:
120 """return a set of indexes from a list
121 >>> index([1,2,3],(0,2))
122 (1, 3)
123 """
124 return tuple([l_[x] for x in i])
125
126
127def csv_reader(csv_data, dialect=csv.excel, **kwargs):
128 csv_reader = csv.reader(csv_data, dialect=dialect, **kwargs)
129 for row in csv_reader:
130 yield row
131
132
133def prefixuri(x, prefix, class_: Optional[URIRef] = None):
134 if prefix:
135 r = rdflib.URIRef(prefix + quote(x.encode("utf8").replace(" ", "_"), safe=""))
136 else:
137 r = rdflib.URIRef(x)
138 uris[x] = (r, class_)
139 return r
140
141
142# meta-language for config
143
144
145class NodeMaker:
146 def range(self):
147 return rdflib.RDFS.Literal
148
149 def __call__(self, x: Any):
150 return rdflib.Literal(x)
151
152
153class NodeUri(NodeMaker):
154 def __init__(self, prefix, class_):
155 self.class_: Optional[URIRef] = None
156 self.prefix = prefix
157 if class_:
158 self.class_ = rdflib.URIRef(class_)
159 else:
160 self.class_ = None
161
162 def __call__(self, x):
163 return prefixuri(x, self.prefix, self.class_)
164
165 def range(self):
166 return self.class_ or rdflib.RDF.Resource
167
168
169class NodeLiteral(NodeMaker):
170 def __init__(self, f=None):
171 self.f = f
172
173
174class NodeFloat(NodeLiteral):
175 def __call__(self, x):
176 if not self.f:
177 return rdflib.Literal(float(x))
178 if callable(self.f):
179 return rdflib.Literal(float(self.f(x)))
180 raise Exception("Function passed to float is not callable")
181
182 def range(self):
183 return rdflib.XSD.double
184
185
186class NodeInt(NodeLiteral):
187 def __call__(self, x):
188 if not self.f:
189 return rdflib.Literal(int(x))
190 if callable(self.f):
191 return rdflib.Literal(int(self.f(x)))
192 raise Exception("Function passed to int is not callable")
193
194 def range(self):
195 return rdflib.XSD.int
196
197
198class NodeBool(NodeLiteral):
199 def __call__(self, x):
200 if not self.f:
201 return rdflib.Literal(bool(x))
202 if callable(self.f):
203 return rdflib.Literal(bool(self.f(x)))
204 raise Exception("Function passed to bool is not callable")
205
206 def range(self):
207 return rdflib.XSD.bool
208
209
210class NodeReplace(NodeMaker):
211 def __init__(self, a, b):
212 self.a = a
213 self.b = b
214
215 def __call__(self, x):
216 return x.replace(self.a, self.b)
217
218
219class NodeDate(NodeLiteral):
220 def __call__(self, x):
221 return rdflib.Literal(datetime.datetime.strptime(x, self.f))
222
223 def range(self):
224 return rdflib.XSD.dateTime
225
226
227class NodeSplit(NodeMaker):
228 def __init__(self, sep, f):
229 self.sep = sep
230 self.f = f
231
232 def __call__(self, x):
233 if not self.f:
234 self.f = rdflib.Literal
235 if not callable(self.f):
236 raise Exception("Function passed to split is not callable!")
237 return [self.f(y.strip()) for y in x.split(self.sep) if y.strip() != ""]
238
239 def range(self):
240 if self.f and isinstance(self.f, NodeMaker):
241 return self.f.range()
242 return NodeMaker.range(self)
243
244
245default_node_make = NodeMaker()
246
247
248def _config_ignore(*args, **kwargs):
249 return "ignore"
250
251
252def _config_uri(prefix=None, class_=None):
253 return NodeUri(prefix, class_)
254
255
256def _config_literal():
257 return NodeLiteral()
258
259
260def _config_float(f=None):
261 return NodeFloat(f)
262
263
264def _config_replace(a, b):
265 return NodeReplace(a, b)
266
267
268def _config_int(f=None):
269 return NodeInt(f)
270
271
272def _config_bool(f=None):
273 return NodeBool(f)
274
275
276def _config_date(format_):
277 return NodeDate(format_)
278
279
280def _config_split(sep=None, f=None):
281 return NodeSplit(sep, f)
282
283
284config_functions = {
285 "ignore": _config_ignore,
286 "uri": _config_uri,
287 "literal": _config_literal,
288 "float": _config_float,
289 "int": _config_int,
290 "date": _config_date,
291 "split": _config_split,
292 "replace": _config_replace,
293 "bool": _config_bool,
294}
295
296
297def column(v):
298 """Return a function for column mapping"""
299
300 return eval(v, config_functions)
301
302
303class CSV2RDF:
304 def __init__(self):
305 self.CLASS = None
306 self.BASE = None
307 self.PROPBASE = None
308 self.IDENT: Union[Tuple[str, ...], str] = "auto"
309 self.LABEL = None
310 self.DEFINECLASS = False
311 self.SKIP = 0
312 self.DELIM = ","
313 self.DEFAULT = None
314
315 self.COLUMNS = {}
316 self.PROPS = {}
317
318 self.OUT = sys.stdout
319
320 self.triples = 0
321
322 def triple(self, s, p, o):
323 self.OUT.write("%s %s %s .\n" % (s.n3(), p.n3(), o.n3()))
324 self.triples += 1
325
326 def convert(self, csvreader):
327 start = time.time()
328
329 if self.OUT:
330 sys.stderr.write("Output to %s\n" % self.OUT.name)
331
332 if self.IDENT != "auto" and not isinstance(self.IDENT, tuple):
333 self.IDENT = (self.IDENT,)
334
335 if not self.BASE:
336 warnings.warn("No base given, using http://example.org/instances/")
337 self.BASE = rdflib.Namespace("http://example.org/instances/")
338
339 if not self.PROPBASE:
340 warnings.warn("No property base given, using http://example.org/property/")
341 self.PROPBASE = rdflib.Namespace("http://example.org/props/")
342
343 # skip lines at the start
344 for x in range(self.SKIP):
345 next(csvreader)
346
347 # read header line
348 header_labels = list(next(csvreader))
349 headers = dict(enumerate([self.PROPBASE[toProperty(x)] for x in header_labels]))
350 # override header properties if some are given
351 for k, v in self.PROPS.items():
352 headers[k] = v
353 header_labels[k] = split_uri(v)[1]
354
355 if self.DEFINECLASS:
356 # output class/property definitions
357 self.triple(self.CLASS, RDF.type, RDFS.Class)
358 for i in range(len(headers)):
359 h, l_ = headers[i], header_labels[i]
360 if h == "" or l_ == "":
361 continue
362 if self.COLUMNS.get(i, self.DEFAULT) == "ignore":
363 continue
364 self.triple(h, RDF.type, RDF.Property)
365 self.triple(h, RDFS.label, rdflib.Literal(toPropertyLabel(l_)))
366 self.triple(h, RDFS.domain, self.CLASS)
367 self.triple(
368 h, RDFS.range, self.COLUMNS.get(i, default_node_make).range()
369 )
370
371 rows = 0
372 for l_ in csvreader:
373 try:
374 if self.IDENT == "auto":
375 uri = self.BASE["%d" % rows]
376 else:
377 uri = self.BASE[
378 "_".join(
379 [
380 # type error: "int" has no attribute "encode"
381 quote(x.encode("utf8").replace(" ", "_"), safe="") # type: ignore[attr-defined]
382 # type error: Argument 2 to "index" has incompatible type "Union[Tuple[str, ...], str]"; expected "Tuple[int, ...]"
383 for x in index(l_, self.IDENT) # type: ignore[arg-type]
384 ]
385 )
386 ]
387
388 if self.LABEL:
389 self.triple(
390 # type error: Argument 1 to "join" of "str" has incompatible type "Tuple[int, ...]"; expected "Iterable[str]"
391 uri,
392 RDFS.label,
393 rdflib.Literal(" ".join(index(l_, self.LABEL))), # type: ignore[arg-type]
394 )
395
396 if self.CLASS:
397 # type triple
398 self.triple(uri, RDF.type, self.CLASS)
399
400 for i, x in enumerate(l_):
401 # type error: "int" has no attribute "strip"
402 x = x.strip() # type: ignore[attr-defined]
403 if x != "":
404 if self.COLUMNS.get(i, self.DEFAULT) == "ignore":
405 continue
406 try:
407 o = self.COLUMNS.get(i, rdflib.Literal)(x)
408 if isinstance(o, list):
409 for _o in o:
410 self.triple(uri, headers[i], _o)
411 else:
412 self.triple(uri, headers[i], o)
413
414 except Exception as e:
415 warnings.warn(
416 "Could not process value for column "
417 + "%d:%s in row %d, ignoring: %s "
418 # type error: "Exception" has no attribute "message"
419 % (i, headers[i], rows, e.message) # type: ignore[attr-defined]
420 )
421
422 rows += 1
423 if rows % 100000 == 0:
424 sys.stderr.write(
425 "%d rows, %d triples, elapsed %.2fs.\n"
426 % (rows, self.triples, time.time() - start)
427 )
428 except Exception:
429 sys.stderr.write("Error processing line: %d\n" % rows)
430 raise
431
432 # output types/labels for generated URIs
433 classes = set()
434 # type error: Incompatible types in assignment (expression has type "Tuple[URIRef, Optional[URIRef]]", variable has type "int")
435 for l_, x in uris.items(): # type: ignore[assignment]
436 # type error: "int" object is not iterable
437 u, c = x # type: ignore[misc]
438 # type error: Cannot determine type of "u"
439 self.triple(u, RDFS.label, rdflib.Literal(l_)) # type: ignore[has-type]
440 # type error: Cannot determine type of "c"
441 if c: # type: ignore[has-type]
442 # type error: Cannot determine type of "c"
443 c = rdflib.URIRef(c) # type: ignore[has-type]
444 classes.add(c)
445 # type error: Cannot determine type of "u"
446 self.triple(u, RDF.type, c) # type: ignore[has-type]
447
448 for c in classes:
449 self.triple(c, RDF.type, RDFS.Class)
450
451 self.OUT.close()
452 sys.stderr.write("Converted %d rows into %d triples.\n" % (rows, self.triples))
453 sys.stderr.write("Took %.2f seconds.\n" % (time.time() - start))
454
455
456def main():
457 csv2rdf = CSV2RDF()
458
459 opts: Union[Dict[str, str], List[Tuple[str, str]]]
460 opts, files = getopt.getopt(
461 sys.argv[1:],
462 "hc:b:p:i:o:Cf:l:s:d:D:",
463 [
464 "out=",
465 "base=",
466 "delim=",
467 "propbase=",
468 "class=",
469 "default=" "ident=",
470 "label=",
471 "skip=",
472 "defineclass",
473 "help",
474 ],
475 )
476 opts = dict(opts)
477
478 if "-h" in opts or "--help" in opts:
479 print(HELP)
480 sys.exit(-1)
481
482 if "-f" in opts:
483 config = configparser.ConfigParser()
484 config.read_file(open(opts["-f"]))
485 for k, v in config.items("csv2rdf"):
486 if k == "out":
487 csv2rdf.OUT = codecs.open(v, "w", "utf-8")
488 elif k == "base":
489 csv2rdf.BASE = rdflib.Namespace(v)
490 elif k == "propbase":
491 csv2rdf.PROPBASE = rdflib.Namespace(v)
492 elif k == "class":
493 csv2rdf.CLASS = rdflib.URIRef(v)
494 elif k == "defineclass":
495 csv2rdf.DEFINECLASS = bool(v)
496 elif k == "ident":
497 csv2rdf.IDENT = eval(v)
498 elif k == "label":
499 csv2rdf.LABEL = eval(v)
500 elif k == "delim":
501 csv2rdf.DELIM = v
502 elif k == "skip":
503 csv2rdf.SKIP = int(v)
504 elif k == "default":
505 csv2rdf.DEFAULT = column(v)
506 elif k.startswith("col"):
507 csv2rdf.COLUMNS[int(k[3:])] = column(v)
508 elif k.startswith("prop"):
509 csv2rdf.PROPS[int(k[4:])] = rdflib.URIRef(v)
510
511 if "-o" in opts:
512 csv2rdf.OUT = codecs.open(opts["-o"], "w", "utf-8")
513 if "--out" in opts:
514 csv2rdf.OUT = codecs.open(opts["--out"], "w", "utf-8")
515
516 if "-b" in opts:
517 csv2rdf.BASE = rdflib.Namespace(opts["-b"])
518 if "--base" in opts:
519 csv2rdf.BASE = rdflib.Namespace(opts["--base"])
520
521 if "-d" in opts:
522 csv2rdf.DELIM = opts["-d"]
523 if "--delim" in opts:
524 csv2rdf.DELIM = opts["--delim"]
525
526 if "-D" in opts:
527 csv2rdf.DEFAULT = column(opts["-D"])
528 if "--default" in opts:
529 csv2rdf.DEFAULT = column(opts["--default"])
530
531 if "-p" in opts:
532 csv2rdf.PROPBASE = rdflib.Namespace(opts["-p"])
533 if "--propbase" in opts:
534 csv2rdf.PROPBASE = rdflib.Namespace(opts["--propbase"])
535
536 if "-l" in opts:
537 csv2rdf.LABEL = eval(opts["-l"])
538 if "--label" in opts:
539 csv2rdf.LABEL = eval(opts["--label"])
540
541 if "-i" in opts:
542 csv2rdf.IDENT = eval(opts["-i"])
543 if "--ident" in opts:
544 csv2rdf.IDENT = eval(opts["--ident"])
545
546 if "-s" in opts:
547 csv2rdf.SKIP = int(opts["-s"])
548 if "--skip" in opts:
549 csv2rdf.SKIP = int(opts["--skip"])
550
551 if "-c" in opts:
552 csv2rdf.CLASS = rdflib.URIRef(opts["-c"])
553 if "--class" in opts:
554 csv2rdf.CLASS = rdflib.URIRef(opts["--class"])
555
556 for k, v in opts.items():
557 if k.startswith("--col"):
558 csv2rdf.COLUMNS[int(k[5:])] = column(v)
559 elif k.startswith("--prop"):
560 csv2rdf.PROPS[int(k[6:])] = rdflib.URIRef(v)
561
562 if csv2rdf.CLASS and ("-C" in opts or "--defineclass" in opts):
563 csv2rdf.DEFINECLASS = True
564
565 csv2rdf.convert(csv_reader(fileinput.input(files), delimiter=csv2rdf.DELIM))
566
567
568if __name__ == "__main__":
569 main()