Source code

Download source archive ↓

src/rdflib/plugins/sparql/aggregates.py

1"""
2Aggregation functions
3"""
4
5from __future__ import annotations
6
7from decimal import Decimal
8from typing import (
9    Any,
10    Callable,
11    Dict,
12    Iterable,
13    List,
14    Mapping,
15    MutableMapping,
16    Optional,
17    Set,
18    Tuple,
19    TypeVar,
20    Union,
21    overload,
22)
23
24from rdflib.namespace import XSD
25from rdflib.plugins.sparql.datatypes import type_promotion
26from rdflib.plugins.sparql.evalutils import _eval, _val
27from rdflib.plugins.sparql.operators import numeric
28from rdflib.plugins.sparql.parserutils import CompValue
29from rdflib.plugins.sparql.sparql import FrozenBindings, NotBoundError, SPARQLTypeError
30from rdflib.term import BNode, Identifier, Literal, URIRef, Variable
31
32
33class Accumulator:
34    """abstract base class for different aggregation functions"""
35
36    def __init__(self, aggregation: CompValue):
37        self.get_value: Callable[[], Optional[Literal]]
38        self.update: Callable[[FrozenBindings, Aggregator], None]
39        self.var = aggregation.res
40        self.expr = aggregation.vars
41        if not aggregation.distinct:
42            # type error: Cannot assign to a method
43            self.use_row = self.dont_care  # type: ignore[method-assign]
44            self.distinct = False
45        else:
46            self.distinct = aggregation.distinct
47            self.seen: Set[Any] = set()
48
49    def dont_care(self, row: FrozenBindings) -> bool:
50        """skips distinct test"""
51        return True
52
53    def use_row(self, row: FrozenBindings) -> bool:
54        """tests distinct with set"""
55        return _eval(self.expr, row) not in self.seen
56
57    def set_value(self, bindings: MutableMapping[Variable, Identifier]) -> None:
58        """sets final value in bindings"""
59        # type error: Incompatible types in assignment (expression has type "Optional[Literal]", target has type "Identifier")
60        bindings[self.var] = self.get_value()  # type: ignore[assignment]
61
62
63class Counter(Accumulator):
64    def __init__(self, aggregation: CompValue):
65        super(Counter, self).__init__(aggregation)
66        self.value = 0
67        if self.expr == "*":
68            # cannot eval "*" => always use the full row
69            # type error: Cannot assign to a method
70            self.eval_row = self.eval_full_row  # type: ignore[assignment]
71
72    def update(self, row: FrozenBindings, aggregator: Aggregator) -> None:
73        try:
74            val = self.eval_row(row)
75        except NotBoundError:
76            # skip UNDEF
77            return
78        self.value += 1
79        if self.distinct:
80            self.seen.add(val)
81
82    def get_value(self) -> Literal:
83        return Literal(self.value)
84
85    def eval_row(self, row: FrozenBindings) -> Identifier:
86        return _eval(self.expr, row)
87
88    def eval_full_row(self, row: FrozenBindings) -> FrozenBindings:
89        return row
90
91    def use_row(self, row: FrozenBindings) -> bool:
92        try:
93            return self.eval_row(row) not in self.seen
94        except NotBoundError:
95            # happens when counting zero optional nodes. See issue #2229
96            return False
97
98
99@overload
100def type_safe_numbers(*args: int) -> Tuple[int]: ...
101
102
103@overload
104def type_safe_numbers(
105    *args: Union[Decimal, float, int]
106) -> Tuple[Union[float, int]]: ...
107
108
109def type_safe_numbers(*args: Union[Decimal, float, int]) -> Iterable[Union[float, int]]:
110    if any(isinstance(arg, float) for arg in args) and any(
111        isinstance(arg, Decimal) for arg in args
112    ):
113        return map(float, args)
114    # type error: Incompatible return value type (got "Tuple[Union[Decimal, float, int], ...]", expected "Iterable[Union[float, int]]")
115    # NOTE on type error: if args contains a Decimal it will nopt get here.
116    return args  # type: ignore[return-value]
117
118
119class Sum(Accumulator):
120    def __init__(self, aggregation: CompValue):
121        super(Sum, self).__init__(aggregation)
122        self.value = 0
123        self.datatype: Optional[str] = None
124
125    def update(self, row: FrozenBindings, aggregator: Aggregator) -> None:
126        try:
127            value = _eval(self.expr, row)
128            dt = self.datatype
129            if dt is None:
130                dt = value.datatype
131            else:
132                # type error: Argument 1 to "type_promotion" has incompatible type "str"; expected "URIRef"
133                dt = type_promotion(dt, value.datatype)  # type: ignore[arg-type]
134            self.datatype = dt
135            self.value = sum(type_safe_numbers(self.value, numeric(value)))
136            if self.distinct:
137                self.seen.add(value)
138        except NotBoundError:
139            # skip UNDEF
140            pass
141
142    def get_value(self) -> Literal:
143        return Literal(self.value, datatype=self.datatype)
144
145
146class Average(Accumulator):
147    def __init__(self, aggregation: CompValue):
148        super(Average, self).__init__(aggregation)
149        self.counter = 0
150        self.sum = 0
151        self.datatype: Optional[str] = None
152
153    def update(self, row: FrozenBindings, aggregator: Aggregator) -> None:
154        try:
155            value = _eval(self.expr, row)
156            dt = self.datatype
157            self.sum = sum(type_safe_numbers(self.sum, numeric(value)))
158            if dt is None:
159                dt = value.datatype
160            else:
161                # type error: Argument 1 to "type_promotion" has incompatible type "str"; expected "URIRef"
162                dt = type_promotion(dt, value.datatype)  # type: ignore[arg-type]
163            self.datatype = dt
164            if self.distinct:
165                self.seen.add(value)
166            self.counter += 1
167        # skip UNDEF or BNode => SPARQLTypeError
168        except NotBoundError:
169            pass
170        except SPARQLTypeError:
171            pass
172
173    def get_value(self) -> Literal:
174        if self.counter == 0:
175            return Literal(0)
176        if self.datatype in (XSD.float, XSD.double):
177            return Literal(self.sum / self.counter)
178        else:
179            return Literal(Decimal(self.sum) / Decimal(self.counter))
180
181
182class Extremum(Accumulator):
183    """abstract base class for Minimum and Maximum"""
184
185    def __init__(self, aggregation: CompValue):
186        self.compare: Callable[[Any, Any], Any]
187        super(Extremum, self).__init__(aggregation)
188        self.value: Any = None
189        # DISTINCT would not change the value for MIN or MAX
190        # type error: Cannot assign to a method
191        self.use_row = self.dont_care  # type: ignore[method-assign]
192
193    def set_value(self, bindings: MutableMapping[Variable, Identifier]) -> None:
194        if self.value is not None:
195            # simply do not set if self.value is still None
196            bindings[self.var] = Literal(self.value)
197
198    def update(self, row: FrozenBindings, aggregator: Aggregator) -> None:
199        try:
200            if self.value is None:
201                self.value = _eval(self.expr, row)
202            else:
203                # self.compare is implemented by Minimum/Maximum
204                self.value = self.compare(self.value, _eval(self.expr, row))
205        # skip UNDEF or BNode => SPARQLTypeError
206        except NotBoundError:
207            pass
208        except SPARQLTypeError:
209            pass
210
211
212_ValueT = TypeVar("_ValueT", Variable, BNode, URIRef, Literal)
213
214
215class Minimum(Extremum):
216    def compare(self, val1: _ValueT, val2: _ValueT) -> _ValueT:
217        return min(val1, val2, key=_val)
218
219
220class Maximum(Extremum):
221    def compare(self, val1: _ValueT, val2: _ValueT) -> _ValueT:
222        return max(val1, val2, key=_val)
223
224
225class Sample(Accumulator):
226    """takes the first eligible value"""
227
228    def __init__(self, aggregation):
229        super(Sample, self).__init__(aggregation)
230        # DISTINCT would not change the value
231        # type error: Cannot assign to a method
232        self.use_row = self.dont_care  # type: ignore[method-assign]
233
234    def update(self, row: FrozenBindings, aggregator: Aggregator) -> None:
235        try:
236            # set the value now
237            aggregator.bindings[self.var] = _eval(self.expr, row)
238            # and skip this accumulator for future rows
239            del aggregator.accumulators[self.var]
240        except NotBoundError:
241            pass
242
243    def get_value(self) -> None:
244        # set None if no value was set
245        return None
246
247
248class GroupConcat(Accumulator):
249    value: List[Literal]
250
251    def __init__(self, aggregation: CompValue):
252        super(GroupConcat, self).__init__(aggregation)
253        # only GROUPCONCAT needs to have a list as accumulator
254        self.value = []
255        if aggregation.separator is None:
256            self.separator = " "
257        else:
258            self.separator = aggregation.separator
259
260    def update(self, row: FrozenBindings, aggregator: Aggregator) -> None:
261        try:
262            value = _eval(self.expr, row)
263            # skip UNDEF
264            if isinstance(value, NotBoundError):
265                return
266            self.value.append(value)
267            if self.distinct:
268                self.seen.add(value)
269        # skip UNDEF
270        # NOTE: It seems like this is not the way undefined values occur, they
271        # come through not as exceptions but as values. This is left here
272        # however as it may occur in some cases.
273        # TODO: Consider removing this.
274        except NotBoundError:
275            pass
276
277    def get_value(self) -> Literal:
278        return Literal(self.separator.join(str(v) for v in self.value))
279
280
281class Aggregator:
282    """combines different Accumulator objects"""
283
284    accumulator_classes = {
285        "Aggregate_Count": Counter,
286        "Aggregate_Sample": Sample,
287        "Aggregate_Sum": Sum,
288        "Aggregate_Avg": Average,
289        "Aggregate_Min": Minimum,
290        "Aggregate_Max": Maximum,
291        "Aggregate_GroupConcat": GroupConcat,
292    }
293
294    def __init__(self, aggregations: List[CompValue]):
295        self.bindings: Dict[Variable, Identifier] = {}
296        self.accumulators: Dict[str, Accumulator] = {}
297        for a in aggregations:
298            accumulator_class = self.accumulator_classes.get(a.name)
299            if accumulator_class is None:
300                raise Exception("Unknown aggregate function " + a.name)
301            self.accumulators[a.res] = accumulator_class(a)
302
303    def update(self, row: FrozenBindings) -> None:
304        """update all own accumulators"""
305        # SAMPLE accumulators may delete themselves
306        # => iterate over list not generator
307
308        for acc in list(self.accumulators.values()):
309            if acc.use_row(row):
310                acc.update(row, self)
311
312    def get_bindings(self) -> Mapping[Variable, Identifier]:
313        """calculate and set last values"""
314        for acc in self.accumulators.values():
315            acc.set_value(self.bindings)
316        return self.bindings