Source code

Download source archive ↓

src/tables.py

1"""Bounded text, JSON and table adapters for selected Galaxy programs."""
2
3from collections import defaultdict
4import csv
5import io
6import json
7import math
8from xml.etree import ElementTree as ET
9from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo
10
11from contract import lines, text
12
13
14def finite(value):
15    n = float(value)
16    if not math.isfinite(n):
17        raise ValueError("nonfinite JSON number")
18    return n
19
20
21def invalid_constant(value):
22    raise ValueError(f"invalid JSON constant: {value}")
23
24
25def parse_json(value):
26    try:
27        return json.loads(value, parse_float=finite, parse_constant=invalid_constant)
28    except RecursionError as error:
29        raise ValueError("JSON nesting too deep") from error
30
31
32def compact(value):
33    return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
34
35
36def json_lines(values):
37    return "".join(compact(v) + "\n" for v in values).encode()
38
39
40def json_to_tabular(rows):
41    output = []
42    for row in rows:
43        if not isinstance(row, list) or any(isinstance(v, (list, dict)) for v in row):
44            raise ValueError("TSV conversion requires a JSON array of scalar values per line")
45        fields = ["" if v is None else v if isinstance(v, str) else compact(v) for v in row]
46        escape = lambda value: value.replace("\\", "\\\\").replace("\t", "\\t").replace("\r", "\\r").replace("\n", "\\n")
47        output.append("\t".join(map(escape, fields)) + "\n")
48    return "".join(output).encode()
49
50
51def jq(op, inputs):
52    rows = lines(inputs["source"])
53    if op == "raw_lines":
54        return json_lines(rows)
55    if op == "merge_objects":
56        originals = [parse_json(text(inputs[port])) for port in ("source", "reference")]
57        if any(not isinstance(obj, dict) for obj in originals):
58            raise ValueError("object merge requires two JSON objects")
59        return json_lines([{**originals[0], **originals[1]}])
60    values = [parse_json(row) for row in rows if row.strip()]
61    if op == "slurp":
62        return json_lines([values])
63    if op == "to_tsv":
64        return json_to_tabular(values)
65    output = []
66    if any(not isinstance(v, (dict, list)) for v in values):
67        raise ValueError("expected objects or arrays")
68    for item in values:
69        entries = item.items() if isinstance(item, dict) else enumerate(item)
70        output.append([{"key": key, "value": value} for key, value in entries])
71    return json_lines(output)
72
73
74def split_table(op, data, params):
75    rows = lines(data)
76    if op == "split_rows":
77        size = params["rows"]
78        p = params["header_lines"]
79        return json_lines(
80            {"part": i // size + 1, "rows": rows[:p] + rows[p + i:p + i + size]}
81            for i in range(0, max(0, len(rows) - p), size)
82        )
83    header = rows[:1] if params["header"] else []
84    groups = defaultdict(list)
85    column = params["column"] - 1
86    for line in rows[len(header):]:
87        fields = line.split("\t")
88        if len(fields) <= column:
89            raise ValueError("grouping column missing in a TSV row")
90        groups[fields[column]].append(line)
91    return json_lines({"key": key, "rows": header + subset} for key, subset in sorted(groups.items()))
92
93
94def join_tables(inputs, params):
95    key_index = params["key_column"] - 1
96    value_index = params["value_column"] - 1
97    fill = params["fill"]
98    if any(c in fill for c in "\0\t\r\n"):
99        raise ValueError("fill must be a single TSV field")
100    tables = []
101    for port in ("source", "reference", "third"):
102        table = {}
103        for row in lines(inputs[port]):
104            fields = row.split("\t")
105            if len(fields) <= max(key_index, value_index):
106                raise ValueError("selected join column missing")
107            key = fields[key_index]
108            if not key or key in table:
109                raise ValueError("expected unique nonempty join keys per table")
110            table[key] = fields[value_index]
111        tables.append(table)
112    all_keys = sorted(set().union(*(t.keys() for t in tables)))
113    return "".join("\t".join([key, *(table.get(key, fill) for table in tables)]) + "\n"
114                   for key in all_keys).encode()
115
116
117WORKBOOK_PARTS = {
118    "[Content_Types].xml": (
119        '<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
120        '<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
121        '<Default Extension="xml" ContentType="application/xml"/>'
122        '<Override PartName="/xl/workbook.xml" '
123        'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>'
124        '<Override PartName="/xl/worksheets/sheet1.xml" '
125        'ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>'
126    ),
127    "_rels/.rels": (
128        '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
129        '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
130        'Target="xl/workbook.xml"/></Relationships>'
131    ),
132    "xl/workbook.xml": (
133        '<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" '
134        'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">'
135        '<sheets><sheet name="Data" sheetId="1" r:id="rId1"/></sheets></workbook>'
136    ),
137    "xl/_rels/workbook.xml.rels": (
138        '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">'
139        '<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" '
140        'Target="worksheets/sheet1.xml"/></Relationships>'
141    ),
142}
143
144
145def xlsx(data):
146    rows = csv.reader(io.StringIO(text(data), newline=""), strict=True)
147    worksheet = ET.Element("worksheet", xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main")
148    sheet_data = ET.SubElement(worksheet, "sheetData")
149    cells = 0
150    try:
151        for index, fields in enumerate(rows, 1):
152            cells += len(fields)
153            if index > 25000 or len(fields) > 256 or cells > 25000:
154                raise ValueError("workbook rows, columns or cells exceed limits")
155            row = ET.SubElement(sheet_data, "row", r=str(index))
156            for value in fields:
157                if len(value) > 32767 or any(ord(c) < 32 and c not in "\t\n\r" or c in "\ufffe\uffff" for c in value):
158                    raise ValueError("invalid workbook cell")
159                cell = ET.SubElement(row, "c", t="inlineStr")
160                content = ET.SubElement(ET.SubElement(cell, "is"), "t", {"xml:space": "preserve"})
161                content.text = value
162    except csv.Error as error:
163        raise ValueError("invalid CSV") from error
164    output = io.BytesIO()
165    with ZipFile(output, "w") as archive:
166        for path, value in WORKBOOK_PARTS.items():
167            archive.writestr(ZipInfo(path), value, compress_type=ZIP_DEFLATED)
168        archive.writestr(ZipInfo("xl/worksheets/sheet1.xml"), ET.tostring(worksheet, encoding="utf-8"),
169                         compress_type=ZIP_DEFLATED)
170    return output.getvalue()
171
172
173def execute(operation, inputs, params):
174    op = operation
175    if op.startswith("jq_"):
176        return jq(op[3:], inputs)
177    if op in {"split_rows", "split_column"}:
178        return split_table(op, inputs["source"], params)
179    if op == "multijoin":
180        return join_tables(inputs, params)
181    if op == "xlsx":
182        return xlsx(inputs["source"])
183    raise ValueError("unknown table operation")