Source code

Download source archive ↓

src/tables.py

1"""Bounded text, JSON and table adapters for selected Galaxy programs."""
2
3from collections import defaultdict
4import csv
5import io
6import json
7import math
8from xml.etree import ElementTree as ET
9from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
10
11from contract import MAX_OUTPUT, lines, text
12
13
14def finite(value):
15    n = float(value)
16    if not math.isfinite(n):
17        raise ValueError("nonfinite JSON number")
18    return n
19
20
21def invalid_constant(value):
22    raise ValueError(f"invalid JSON constant: {value}")
23
24
25def parse_json(value):
26    try:
27        return json.loads(value, parse_float=finite, parse_constant=invalid_constant)
28    except RecursionError as error:
29        raise ValueError("JSON nesting too deep") from error
30
31
32def compact(value):
33    try:
34        return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
35    except RecursionError as error:
36        raise ValueError("JSON nesting too deep") from error
37
38
39def json_lines(values):
40    output = bytearray()
41    for value in values:
42        row = compact(value).encode()
43        if len(output) + len(row) + 1 > MAX_OUTPUT:
44            raise ValueError("JSON result exceeds 8 MiB")
45        output.extend(row)
46        output.append(10)
47    return bytes(output)
48
49
50def json_to_tabular(rows):
51    output = []
52    for row in rows:
53        if not isinstance(row, list) or any(isinstance(v, (list, dict)) for v in row):
54            raise ValueError("TSV conversion requires a JSON array of scalar values per line")
55        fields = ["" if v is None else v if isinstance(v, str) else compact(v) for v in row]
56        escape = lambda value: value.replace("\\", "\\\\").replace("\t", "\\t").replace("\r", "\\r").replace("\n", "\\n")
57        output.append("\t".join(map(escape, fields)) + "\n")
58    return "".join(output).encode()
59
60
61def jq(op, inputs):
62    rows = lines(inputs["source"])
63    if op == "raw_lines":
64        return json_lines(rows)
65    if op == "merge_objects":
66        originals = [parse_json(text(inputs[port])) for port in ("source", "reference")]
67        if any(not isinstance(obj, dict) for obj in originals):
68            raise ValueError("object merge requires two JSON objects")
69        return json_lines([{**originals[0], **originals[1]}])
70    values = [parse_json(row) for row in rows if row.strip()]
71    if op == "slurp":
72        return json_lines([values])
73    if op == "to_tsv":
74        return json_to_tabular(values)
75    output = []
76    if any(not isinstance(v, (dict, list)) for v in values):
77        raise ValueError("expected objects or arrays")
78    for item in values:
79        entries = item.items() if isinstance(item, dict) else enumerate(item)
80        output.append([{"key": key, "value": value} for key, value in entries])
81    return json_lines(output)
82
83
84def split_table(op, data, params):
85    rows = lines(data)
86    if op == "split_rows":
87        size = params["rows"]
88        p = params["header_lines"]
89        return json_lines(
90            {"part": i // size + 1, "rows": rows[:p] + rows[p + i:p + i + size]}
91            for i in range(0, max(0, len(rows) - p), size)
92        )
93    header = rows[:1] if params["header"] else []
94    groups = defaultdict(list)
95    column = params["column"] - 1
96    for line in rows[len(header):]:
97        fields = line.split("\t")
98        if len(fields) <= column:
99            raise ValueError("grouping column missing in a TSV row")
100        groups[fields[column]].append(line)
101    return json_lines({"key": key, "rows": header + subset} for key, subset in sorted(groups.items()))
102
103
104def join_tables(inputs, params):
105    key_index = params["key_column"] - 1
106    value_index = params["value_column"] - 1
107    fill = params["fill"]
108    if any(c in fill for c in "\0\t\r\n"):
109        raise ValueError("fill must be a single TSV field")
110    tables = []
111    for port in ("source", "reference", "third"):
112        table = {}
113        for row in lines(inputs[port]):
114            fields = row.split("\t")
115            if len(fields) <= max(key_index, value_index):
116                raise ValueError("selected join column missing")
117            key = fields[key_index]
118            if not key or key in table:
119                raise ValueError("expected unique nonempty join keys per table")
120            table[key] = fields[value_index]
121        tables.append(table)
122    all_keys = sorted(set().union(*(t.keys() for t in tables)))
123    return "".join("\t".join([key, *(table.get(key, fill) for table in tables)]) + "\n"
124                   for key in all_keys).encode()
125
126
127WORKBOOK_PARTS = {
128    "[Content_Types].xml": (
129        '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
130        '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
131        '<Default Extension="xml" ContentType="application/xml"/>'
132        '<Override PartName="/xl/workbook.xml" '
133        'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
134        '<Override PartName="/xl/worksheets/sheet1.xml" '
135        'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>'
136    ),
137    "_rels/.rels": (
138        '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
139        '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
140        'Target="xl/workbook.xml"/></Relationships>'
141    ),
142    "xl/workbook.xml": (
143        '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
144        'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
145        '<sheets><sheet name="Data" sheetId="1" r:id="rId1"/></sheets></workbook>'
146    ),
147    "xl/_rels/workbook.xml.rels": (
148        '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
149        '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
150        'Target="worksheets/sheet1.xml"/></Relationships>'
151    ),
152}
153
154
155def xlsx(data):
156    rows = csv.reader(io.StringIO(text(data), newline=""), strict=True)
157    worksheet = ET.Element("worksheet", xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main")
158    sheet_data = ET.SubElement(worksheet, "sheetData")
159    cells = 0
160    try:
161        for index, fields in enumerate(rows, 1):
162            cells += len(fields)
163            if index > 25000 or len(fields) > 256 or cells > 25000:
164                raise ValueError("workbook rows, columns or cells exceed limits")
165            row = ET.SubElement(sheet_data, "row", r=str(index))
166            for value in fields:
167                if len(value) > 32767 or any(ord(c) < 32 and c not in "\t\n\r" or c in "\ufffe\uffff" for c in value):
168                    raise ValueError("invalid workbook cell")
169                cell = ET.SubElement(row, "c", t="inlineStr")
170                content = ET.SubElement(ET.SubElement(cell, "is"), "t", {"xml:space": "preserve"})
171                content.text = value
172    except csv.Error as error:
173        raise ValueError("invalid CSV") from error
174    output = io.BytesIO()
175    with ZipFile(output, "w") as archive:
176        for path, value in WORKBOOK_PARTS.items():
177            archive.writestr(ZipInfo(path), value, compress_type=ZIP_DEFLATED)
178        archive.writestr(ZipInfo("xl/worksheets/sheet1.xml"), ET.tostring(worksheet, encoding="utf-8"),
179                         compress_type=ZIP_DEFLATED)
180    return output.getvalue()
181
182
183def execute(operation, inputs, params):
184    op = operation
185    if op.startswith("jq_"):
186        return jq(op[3:], inputs)
187    if op in {"split_rows", "split_column"}:
188        return split_table(op, inputs["source"], params)
189    if op == "multijoin":
190        return join_tables(inputs, params)
191    if op == "xlsx":
192        return xlsx(inputs["source"])
193    raise ValueError("unknown table operation")