src/datamash.py
1"""Bounded single-metric and row operations from the Galaxy GNU Datamash wrapper."""
2
3from collections import Counter
4from itertools import groupby
5import math
6import statistics
7
8from contract import lines
9
10
11def numeric(value):
12 if len(value) > 128:
13 raise ValueError("numeric field exceeds 128 characters")
14 number = float(value)
15 if not math.isfinite(number) or abs(number) > 1e70:
16 raise ValueError("expected a finite number with magnitude at most 1e70")
17 return number
18
19
20def quantile(values, probability):
21 sorted_values = sorted(values)
22 index = (len(values) - 1) * probability
23 lower = int(index)
24 upper = math.ceil(index)
25 return sorted_values[lower] + (index - lower) * (sorted_values[upper] - sorted_values[lower])
26
27
28def moments(values, operation):
29 count = len(values)
30 average = statistics.fmean(values)
31 offsets = [value - average for value in values]
32 scale = max(map(abs, offsets))
33 if scale == 0:
34 return "NA"
35 normalized = [offset / scale for offset in offsets]
36 variance = statistics.fmean(x * x for x in normalized)
37 if operation.endswith("skew"):
38 population = statistics.fmean(x ** 3 for x in normalized) / variance ** 1.5
39 if operation == "pskew":
40 return population
41 return population * math.sqrt(count * (count - 1)) / (count - 2) if count > 2 else "NA"
42
43 population = statistics.fmean(x ** 4 for x in normalized) / variance ** 2 - 3
44 if operation == "pkurt":
45 return population
46 if count < 4:
47 return "NA"
48 return (count - 1) * ((count + 1) * population + 6) / ((count - 2) * (count - 3))
49
50
51NUMERIC = {
52 "sum": math.fsum,
53 "min": min,
54 "max": max,
55 "absmin": lambda v: min(v, key=abs),
56 "absmax": lambda v: max(v, key=abs),
57 "mean": statistics.fmean,
58 "pstdev": statistics.pstdev,
59 "sstdev": statistics.stdev,
60 "median": statistics.median,
61 "q1": lambda v: quantile(v, 0.25),
62 "q3": lambda v: quantile(v, 0.75),
63 "iqr": lambda v: quantile(v, 0.75) - quantile(v, 0.25),
64 "pvar": statistics.pvariance,
65 "svar": statistics.variance,
66 "mode": lambda v: min(Counter(v).items(), key=lambda item: (-item[1], item[0]))[0],
67 "antimode": lambda v: min(Counter(v).items(), key=lambda item: (item[1], item[0]))[0],
68}
69
70TEXT = {
71 "count": len,
72 "countunique": lambda v: len(set(v)),
73 "unique": lambda v: ",".join(sorted(set(v))),
74 "collapse": ",".join,
75}
76
77
78def aggregate(operation, values, skip_missing):
79 if skip_missing:
80 values = [v for v in values if v.lower() not in {"na", "nan"}]
81 if not values:
82 return "0" if operation in {"count", "countunique"} else "NA"
83 if operation in TEXT:
84 return str(TEXT[operation](values))
85 numbers = [numeric(v) for v in values]
86 if operation in ("sstdev", "svar") and len(numbers) < 2:
87 return "NA"
88 if operation == "mad":
89 center = statistics.median(numbers)
90 value = 1.4826 * statistics.median(abs(n - center) for n in numbers)
91 elif operation in NUMERIC:
92 value = NUMERIC[operation](numbers)
93 else:
94 value = moments(numbers, operation)
95 return value if isinstance(value, str) else format(value or 0, ".12g")
96
97
98def execute(operation, inputs, params):
99 rows = [row.split("\t") for row in lines(inputs["source"])]
100 if operation == "reverse":
101 output = ["\t".join(reversed(row)) for row in rows]
102 return "".join(row + "\n" for row in output).encode()
103 column = params["column"] - 1
104 group = params["group_column"] - 1
105 header = rows.pop(0) if rows and params["header"] else None
106 if any(len(row) <= max(column, group) for row in rows):
107 raise ValueError("a selected table column is missing")
108 if group >= 0:
109 key = lambda row: row[group]
110 grouped = groupby(sorted(rows, key=key), key=key)
111 else:
112 grouped = [(None, rows)] if rows else []
113 output = []
114 if header is not None:
115 if len(header) <= max(column, group):
116 raise ValueError("a selected header column is missing")
117 fields = [f"{operation}({header[column]})"]
118 if group >= 0:
119 fields.insert(0, f"GroupBy({header[group]})")
120 output.append("\t".join(fields))
121 for key, members in grouped:
122 value = aggregate(operation, [row[column] for row in members], params["skip_missing"])
123 output.append(value if key is None else key + "\t" + value)
124 return "".join(row + "\n" for row in output).encode()