src/tables.py
1"""Bounded text, JSON and table adapters for selected Galaxy programs."""
2
3from collections import defaultdict
4import csv
5import io
6import json
7import math
8from xml.etree import ElementTree as ET
9from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
10
11from contract import MAX_OUTPUT, lines, text
12
13
14def finite(value):
15 n = float(value)
16 if not math.isfinite(n):
17 raise ValueError("nonfinite JSON number")
18 return n
19
20
21def invalid_constant(value):
22 raise ValueError(f"invalid JSON constant: {value}")
23
24
25def parse_json(value):
26 try:
27 return json.loads(value, parse_float=finite, parse_constant=invalid_constant)
28 except RecursionError as error:
29 raise ValueError("JSON nesting too deep") from error
30
31
32def compact(value):
33 try:
34 return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
35 except RecursionError as error:
36 raise ValueError("JSON nesting too deep") from error
37
38
39def json_lines(values):
40 output = bytearray()
41 for value in values:
42 row = compact(value).encode()
43 if len(output) + len(row) + 1 > MAX_OUTPUT:
44 raise ValueError("JSON result exceeds 8 MiB")
45 output.extend(row)
46 output.append(10)
47 return bytes(output)
48
49
50def json_to_tabular(rows):
51 output = []
52 for row in rows:
53 if not isinstance(row, list) or any(isinstance(v, (list, dict)) for v in row):
54 raise ValueError("TSV conversion requires a JSON array of scalar values per line")
55 fields = ["" if v is None else v if isinstance(v, str) else compact(v) for v in row]
56 escape = lambda value: value.replace("\\", "\\\\").replace("\t", "\\t").replace("\r", "\\r").replace("\n", "\\n")
57 output.append("\t".join(map(escape, fields)) + "\n")
58 return "".join(output).encode()
59
60
61def jq(op, inputs):
62 rows = lines(inputs["source"])
63 if op == "raw_lines":
64 return json_lines(rows)
65 if op == "merge_objects":
66 originals = [parse_json(text(inputs[port])) for port in ("source", "reference")]
67 if any(not isinstance(obj, dict) for obj in originals):
68 raise ValueError("object merge requires two JSON objects")
69 return json_lines([{**originals[0], **originals[1]}])
70 values = [parse_json(row) for row in rows if row.strip()]
71 if op == "slurp":
72 return json_lines([values])
73 if op == "to_tsv":
74 return json_to_tabular(values)
75 output = []
76 if any(not isinstance(v, (dict, list)) for v in values):
77 raise ValueError("expected objects or arrays")
78 for item in values:
79 entries = item.items() if isinstance(item, dict) else enumerate(item)
80 output.append([{"key": key, "value": value} for key, value in entries])
81 return json_lines(output)
82
83
84def split_table(op, data, params):
85 rows = lines(data)
86 if op == "split_rows":
87 size = params["rows"]
88 p = params["header_lines"]
89 return json_lines(
90 {"part": i // size + 1, "rows": rows[:p] + rows[p + i:p + i + size]}
91 for i in range(0, max(0, len(rows) - p), size)
92 )
93 header = rows[:1] if params["header"] else []
94 groups = defaultdict(list)
95 column = params["column"] - 1
96 for line in rows[len(header):]:
97 fields = line.split("\t")
98 if len(fields) <= column:
99 raise ValueError("grouping column missing in a TSV row")
100 groups[fields[column]].append(line)
101 return json_lines({"key": key, "rows": header + subset} for key, subset in sorted(groups.items()))
102
103
104def join_tables(inputs, params):
105 key_index = params["key_column"] - 1
106 value_index = params["value_column"] - 1
107 fill = params["fill"]
108 if any(c in fill for c in "\0\t\r\n"):
109 raise ValueError("fill must be a single TSV field")
110 tables = []
111 for port in ("source", "reference", "third"):
112 table = {}
113 for row in lines(inputs[port]):
114 fields = row.split("\t")
115 if len(fields) <= max(key_index, value_index):
116 raise ValueError("selected join column missing")
117 key = fields[key_index]
118 if not key or key in table:
119 raise ValueError("expected unique nonempty join keys per table")
120 table[key] = fields[value_index]
121 tables.append(table)
122 all_keys = sorted(set().union(*(t.keys() for t in tables)))
123 return "".join("\t".join([key, *(table.get(key, fill) for table in tables)]) + "\n"
124 for key in all_keys).encode()
125
126
127WORKBOOK_PARTS = {
128 "[Content_Types].xml": (
129 '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
130 '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
131 '<Default Extension="xml" ContentType="application/xml"/>'
132 '<Override PartName="/xl/workbook.xml" '
133 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
134 '<Override PartName="/xl/worksheets/sheet1.xml" '
135 'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>'
136 ),
137 "_rels/.rels": (
138 '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
139 '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
140 'Target="xl/workbook.xml"/></Relationships>'
141 ),
142 "xl/workbook.xml": (
143 '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
144 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
145 '<sheets><sheet name="Data" sheetId="1" r:id="rId1"/></sheets></workbook>'
146 ),
147 "xl/_rels/workbook.xml.rels": (
148 '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
149 '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
150 'Target="worksheets/sheet1.xml"/></Relationships>'
151 ),
152}
153
154
155def xlsx(data):
156 rows = csv.reader(io.StringIO(text(data), newline=""), strict=True)
157 worksheet = ET.Element("worksheet", xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main")
158 sheet_data = ET.SubElement(worksheet, "sheetData")
159 cells = 0
160 try:
161 for index, fields in enumerate(rows, 1):
162 cells += len(fields)
163 if index > 25000 or len(fields) > 256 or cells > 25000:
164 raise ValueError("workbook rows, columns or cells exceed limits")
165 row = ET.SubElement(sheet_data, "row", r=str(index))
166 for value in fields:
167 if len(value) > 32767 or any(ord(c) < 32 and c not in "\t\n\r" or c in "\ufffe\uffff" for c in value):
168 raise ValueError("invalid workbook cell")
169 cell = ET.SubElement(row, "c", t="inlineStr")
170 content = ET.SubElement(ET.SubElement(cell, "is"), "t", {"xml:space": "preserve"})
171 content.text = value
172 except csv.Error as error:
173 raise ValueError("invalid CSV") from error
174 output = io.BytesIO()
175 with ZipFile(output, "w") as archive:
176 for path, value in WORKBOOK_PARTS.items():
177 archive.writestr(ZipInfo(path), value, compress_type=ZIP_DEFLATED)
178 archive.writestr(ZipInfo("xl/worksheets/sheet1.xml"), ET.tostring(worksheet, encoding="utf-8"),
179 compress_type=ZIP_DEFLATED)
180 return output.getvalue()
181
182
183def execute(operation, inputs, params):
184 op = operation
185 if op.startswith("jq_"):
186 return jq(op[3:], inputs)
187 if op in {"split_rows", "split_column"}:
188 return split_table(op, inputs["source"], params)
189 if op == "multijoin":
190 return join_tables(inputs, params)
191 if op == "xlsx":
192 return xlsx(inputs["source"])
193 raise ValueError("unknown table operation")