Source code

Download source archive ↓

src/datamash.py

1"""Bounded single-metric and row operations from the Galaxy GNU Datamash wrapper."""
2
3from collections import Counter
4from itertools import groupby
5import math
6import statistics
7
8from contract import lines
9
10
11def numeric(value):
12    if len(value) > 128:
13        raise ValueError("numeric field exceeds 128 characters")
14    number = float(value)
15    if not math.isfinite(number) or abs(number) > 1e70:
16        raise ValueError("expected a finite number with magnitude at most 1e70")
17    return number
18
19
20def quantile(values, probability):
21    sorted_values = sorted(values)
22    index = (len(values) - 1) * probability
23    lower = int(index)
24    upper = math.ceil(index)
25    return sorted_values[lower] + (index - lower) * (sorted_values[upper] - sorted_values[lower])
26
27
28def moments(values, operation):
29    count = len(values)
30    average = statistics.fmean(values)
31    offsets = [value - average for value in values]
32    scale = max(map(abs, offsets))
33    if scale == 0:
34        return "NA"
35    normalized = [offset / scale for offset in offsets]
36    variance = statistics.fmean(x * x for x in normalized)
37    if operation.endswith("skew"):
38        population = statistics.fmean(x ** 3 for x in normalized) / variance ** 1.5
39        if operation == "pskew":
40            return population
41        return population * math.sqrt(count * (count - 1)) / (count - 2) if count > 2 else "NA"
42
43    population = statistics.fmean(x ** 4 for x in normalized) / variance ** 2 - 3
44    if operation == "pkurt":
45        return population
46    if count < 4:
47        return "NA"
48    return (count - 1) * ((count + 1) * population + 6) / ((count - 2) * (count - 3))
49
50
51NUMERIC = {
52    "sum": math.fsum,
53    "min": min,
54    "max": max,
55    "absmin": lambda v: min(v, key=abs),
56    "absmax": lambda v: max(v, key=abs),
57    "mean": statistics.fmean,
58    "pstdev": statistics.pstdev,
59    "sstdev": statistics.stdev,
60    "median": statistics.median,
61    "q1": lambda v: quantile(v, 0.25),
62    "q3": lambda v: quantile(v, 0.75),
63    "iqr": lambda v: quantile(v, 0.75) - quantile(v, 0.25),
64    "pvar": statistics.pvariance,
65    "svar": statistics.variance,
66    "mode": lambda v: min(Counter(v).items(), key=lambda item: (-item[1], item[0]))[0],
67    "antimode": lambda v: min(Counter(v).items(), key=lambda item: (item[1], item[0]))[0],
68}
69
70TEXT = {
71    "count": len,
72    "countunique": lambda v: len(set(v)),
73    "unique": lambda v: ",".join(sorted(set(v))),
74    "collapse": ",".join,
75}
76
77
78def aggregate(operation, values, skip_missing):
79    if skip_missing:
80        values = [v for v in values if v.lower() not in {"na", "nan"}]
81    if not values:
82        return "0" if operation in {"count", "countunique"} else "NA"
83    if operation in TEXT:
84        return str(TEXT[operation](values))
85    numbers = [numeric(v) for v in values]
86    if operation in ("sstdev", "svar") and len(numbers) < 2:
87        return "NA"
88    if operation == "mad":
89        center = statistics.median(numbers)
90        value = 1.4826 * statistics.median(abs(n - center) for n in numbers)
91    elif operation in NUMERIC:
92        value = NUMERIC[operation](numbers)
93    else:
94        value = moments(numbers, operation)
95    return value if isinstance(value, str) else format(value or 0, ".12g")
96
97
98def execute(operation, inputs, params):
99    rows = [row.split("\t") for row in lines(inputs["source"])]
100    if operation == "reverse":
101        output = ["\t".join(reversed(row)) for row in rows]
102        return "".join(row + "\n" for row in output).encode()
103    column = params["column"] - 1
104    group = params["group_column"] - 1
105    header = rows.pop(0) if rows and params["header"] else None
106    if any(len(row) <= max(column, group) for row in rows):
107        raise ValueError("a selected table column is missing")
108    if group >= 0:
109        key = lambda row: row[group]
110        grouped = groupby(sorted(rows, key=key), key=key)
111    else:
112        grouped = [(None, rows)] if rows else []
113    output = []
114    if header is not None:
115        if len(header) <= max(column, group):
116            raise ValueError("a selected header column is missing")
117        fields = [f"{operation}({header[column]})"]
118        if group >= 0:
119            fields.insert(0, f"GroupBy({header[group]})")
120        output.append("\t".join(fields))
121    for key, members in grouped:
122        value = aggregate(operation, [row[column] for row in members], params["skip_missing"])
123        output.append(value if key is None else key + "\t" + value)
124    return "".join(row + "\n" for row in output).encode()