Source code

Download source archive ↓

src/rolling.py

1"""Bounded trailing and centered windows adapted from Galaxy's Pandas wrapper."""
2
3from collections import defaultdict
4import math
5import statistics
6
7from contract import MAX_OUTPUT, lines
8from datamash import moments, numeric, quantile
9
10COSINE = {
11    "blackman": (0.42, -0.50, 0.08),
12    "hamming": (0.54, -0.46),
13    "blackmanharris": (0.35875, -0.48829, 0.14128, -0.01168),
14    "nuttall": (0.3635819, -0.4891775, 0.1365995, -0.0106411),
15}
16
17STATISTICS = {
18    "sum": math.fsum,
19    "mean": statistics.fmean,
20    "median": statistics.median,
21    "min": min,
22    "max": max,
23    "std": statistics.stdev,
24    "var": statistics.variance,
25}
26WINDOWS = {"triang", "bartlett", "parzen", "bohman", "barthann", *COSINE}
27
28
29def weight(method, index, size):
30    fractional = index / (size - 1)
31    d = abs(2 * fractional - 1)
32    if method in COSINE:
33        return math.fsum(c * math.cos(2 * math.pi * k * fractional) for k, c in enumerate(COSINE[method]))
34    if method == "triang":
35        return 1 - d * (size - 1) / (size + size % 2)
36    if method == "bartlett":
37        return 1 - d
38    if method == "parzen":
39        u = d * (size - 1) / size
40        return 1 - 6*u*u + 6*u**3 if u <= 0.5 else 2 * (1 - u)**3
41    if method == "bohman":
42        return (1 - d) * math.cos(math.pi * d) + math.sin(math.pi * d) / math.pi if d < 1 else 0.0
43    return 0.62 - 0.24*d - 0.38 * math.cos(2 * math.pi * fractional)
44
45
46def statistic(method, values, secondary, params):
47    if method == "count":
48        return sum(x is not None for x in values)
49    if None in values:
50        return None
51    if method in STATISTICS:
52        return STATISTICS[method](values)
53    if method == "quantile":
54        return quantile(values, params["quantile"])
55    if method in ("skew", "kurt"):
56        n = len(values)
57        if n < (3 if method == "skew" else 4):
58            return None
59        if min(values) == max(values):
60            return 0 if method == "skew" else -3
61        return moments(values, "sskew" if method == "skew" else "skurt")
62    if None in secondary:
63        return None
64    if method == "cov":
65        return statistics.covariance(values, secondary)
66    if min(values) == max(values) or min(secondary) == max(secondary):
67        return None
68    return statistics.correlation(values, secondary)
69
70
71def calc(method, values, secondary, weights, params):
72    if weights is None:
73        return statistic(method, values, secondary, params)
74    denominator = math.fsum(weights)
75    if None in values or abs(denominator) < 1e-14:
76        return None
77    return math.fsum(v * w for v, w in zip(values, weights)) / denominator
78
79
80def execute(method, inputs, params):
81    rows = lines(inputs["source"])
82    groups = defaultdict(list)
83    result = ["NA"] * len(rows)
84    window = params["window"]
85    group, value, second = params["group_column"], params["value_column"], params.get("second_column", 0)
86    last = max(group, value, second)
87    for i, row in enumerate(rows):
88        fields = row.split("\t", last)
89        if len(fields) < last:
90            raise ValueError("selected rolling column missing")
91        x = fields[value - 1]
92        w = fields[second - 1] if second else ""
93        groups[fields[group - 1] if group else ""].append(
94            (i, None if x in ("", "NA") else numeric(x), None if w in ("", "NA") else numeric(w))
95        )
96    lookback = window // 2 if params["center"] else window - 1
97    weights = [weight(method, i, window) for i in range(window)] if method in WINDOWS else None
98    for batch in groups.values():
99        for start in range(len(batch) - window + 1):
100            segment = batch[start:start + window]
101            position = segment[lookback][0]
102            val = calc(method, [r[1] for r in segment], [r[2] for r in segment], weights, params)
103            if val is not None and val != "NA":
104                if not math.isfinite(val):
105                    raise ValueError("rolling statistic overflow")
106                result[position] = format(val or 0, ".12g")
107    output = bytearray()
108    for row, v in zip(rows, result):
109        data = (row + "\t" + v + "\n").encode()
110        if len(output) + len(data) > MAX_OUTPUT:
111            raise ValueError("rolling output exceeds 8 MiB")
112        output.extend(data)
113    return bytes(output)