src/tables.py
1"""Bounded text, JSON and table adapters for selected Galaxy programs."""
2
3from collections import defaultdict
4import csv
5import io
6import json
7import math
8from xml.etree import ElementTree as ET
9from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
10
11from contract import lines, text
12
13
14def finite(value):
15 n = float(value)
16 if not math.isfinite(n):
17 raise ValueError("nonfinite JSON number")
18 return n
19
20
21def invalid_constant(value):
22 raise ValueError(f"invalid JSON constant: {value}")
23
24
25def parse_json(value):
26 try:
27 return json.loads(value, parse_float=finite, parse_constant=invalid_constant)
28 except RecursionError as error:
29 raise ValueError("JSON nesting too deep") from error
30
31
32def compact(value):
33 return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
34
35
36def json_lines(values):
37 return "".join(compact(v) + "\n" for v in values).encode()
38
39
40def json_to_tabular(rows):
41 output = []
42 for row in rows:
43 if not isinstance(row, list) or any(isinstance(v, (list, dict)) for v in row):
44 raise ValueError("TSV conversion requires a JSON array of scalar values per line")
45 fields = ["" if v is None else v if isinstance(v, str) else compact(v) for v in row]
46 escape = lambda value: value.replace("\\", "\\\\").replace("\t", "\\t").replace("\r", "\\r").replace("\n", "\\n")
47 output.append("\t".join(map(escape, fields)) + "\n")
48 return "".join(output).encode()
49
50
51def jq(op, inputs):
52 rows = lines(inputs["source"])
53 if op == "raw_lines":
54 return json_lines(rows)
55 if op == "merge_objects":
56 originals = [parse_json(text(inputs[port])) for port in ("source", "reference")]
57 if any(not isinstance(obj, dict) for obj in originals):
58 raise ValueError("object merge requires two JSON objects")
59 return json_lines([{**originals[0], **originals[1]}])
60 values = [parse_json(row) for row in rows if row.strip()]
61 if op == "slurp":
62 return json_lines([values])
63 if op == "to_tsv":
64 return json_to_tabular(values)
65 output = []
66 if any(not isinstance(v, (dict, list)) for v in values):
67 raise ValueError("expected objects or arrays")
68 for item in values:
69 entries = item.items() if isinstance(item, dict) else enumerate(item)
70 output.append([{"key": key, "value": value} for key, value in entries])
71 return json_lines(output)
72
73
74def split_table(op, data, params):
75 rows = lines(data)
76 if op == "split_rows":
77 size = params["rows"]
78 p = params["header_lines"]
79 return json_lines(
80 {"part": i // size + 1, "rows": rows[:p] + rows[p + i:p + i + size]}
81 for i in range(0, max(0, len(rows) - p), size)
82 )
83 header = rows[:1] if params["header"] else []
84 groups = defaultdict(list)
85 column = params["column"] - 1
86 for line in rows[len(header):]:
87 fields = line.split("\t")
88 if len(fields) <= column:
89 raise ValueError("grouping column missing in a TSV row")
90 groups[fields[column]].append(line)
91 return json_lines({"key": key, "rows": header + subset} for key, subset in sorted(groups.items()))
92
93
94def join_tables(inputs, params):
95 key_index = params["key_column"] - 1
96 value_index = params["value_column"] - 1
97 fill = params["fill"]
98 if any(c in fill for c in "\0\t\r\n"):
99 raise ValueError("fill must be a single TSV field")
100 tables = []
101 for port in ("source", "reference", "third"):
102 table = {}
103 for row in lines(inputs[port]):
104 fields = row.split("\t")
105 if len(fields) <= max(key_index, value_index):
106 raise ValueError("selected join column missing")
107 key = fields[key_index]
108 if not key or key in table:
109 raise ValueError("expected unique nonempty join keys per table")
110 table[key] = fields[value_index]
111 tables.append(table)
112 all_keys = sorted(set().union(*(t.keys() for t in tables)))
113 return "".join("\t".join([key, *(table.get(key, fill) for table in tables)]) + "\n"
114 for key in all_keys).encode()
115
116
117WORKBOOK_PARTS = {
118 "[Content_Types].xml": (
119 '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
120 '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
121 '<Default Extension="xml" ContentType="application/xml"/>'
122 '<Override PartName="/xl/workbook.xml" '
123 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
124 '<Override PartName="/xl/worksheets/sheet1.xml" '
125 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>'
126 ),
127 "_rels/.rels": (
128 '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
129 '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
130 'Target="xl/workbook.xml"/></Relationships>'
131 ),
132 "xl/workbook.xml": (
133 '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
134 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
135 '<sheets><sheet name="Data" sheetId="1" r:id="rId1"/></sheets></workbook>'
136 ),
137 "xl/_rels/workbook.xml.rels": (
138 '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
139 '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
140 'Target="worksheets/sheet1.xml"/></Relationships>'
141 ),
142}
143
144
145def xlsx(data):
146 rows = csv.reader(io.StringIO(text(data), newline=""), strict=True)
147 worksheet = ET.Element("worksheet", xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main")
148 sheet_data = ET.SubElement(worksheet, "sheetData")
149 cells = 0
150 try:
151 for index, fields in enumerate(rows, 1):
152 cells += len(fields)
153 if index > 25000 or len(fields) > 256 or cells > 25000:
154 raise ValueError("workbook rows, columns or cells exceed limits")
155 row = ET.SubElement(sheet_data, "row", r=str(index))
156 for value in fields:
157 if len(value) > 32767 or any(ord(c) < 32 and c not in "\t\n\r" or c in "\ufffe\uffff" for c in value):
158 raise ValueError("invalid workbook cell")
159 cell = ET.SubElement(row, "c", t="inlineStr")
160 content = ET.SubElement(ET.SubElement(cell, "is"), "t", {"xml:space": "preserve"})
161 content.text = value
162 except csv.Error as error:
163 raise ValueError("invalid CSV") from error
164 output = io.BytesIO()
165 with ZipFile(output, "w") as archive:
166 for path, value in WORKBOOK_PARTS.items():
167 archive.writestr(ZipInfo(path), value, compress_type=ZIP_DEFLATED)
168 archive.writestr(ZipInfo("xl/worksheets/sheet1.xml"), ET.tostring(worksheet, encoding="utf-8"),
169 compress_type=ZIP_DEFLATED)
170 return output.getvalue()
171
172
173def execute(operation, inputs, params):
174 op = operation
175 if op.startswith("jq_"):
176 return jq(op[3:], inputs)
177 if op in {"split_rows", "split_column"}:
178 return split_table(op, inputs["source"], params)
179 if op == "multijoin":
180 return join_tables(inputs, params)
181 if op == "xlsx":
182 return xlsx(inputs["source"])
183 raise ValueError("unknown table operation")