Source code

Download source archive ↓

upstream-extended/iuc/tools/table_compute/scripts/table_compute.py

1#!/usr/bin/env python3
2"""
3Table Compute tool - a wrapper around pandas with parameter input validation.
4"""
5
6
7__version__ = "0.9.2"
8
9import csv
10import math
11from sys import argv
12
13import numpy as np
14import pandas as pd
15from safety import Safety
16
17if len(argv) == 2 and argv[1] == "--version":
18    print(__version__)
19    exit(0)
20
21# The import below should be generated in the same directory as
22# the table_compute.py script.
23# It is placed here so that the --version switch does not fail
24import userconfig as uc  # noqa: I100,I202
25
26
27class Utils:
28    @staticmethod
29    def getOneValueMathOp(op_name):
30        "Returns a simple one value math operator such as log, sqrt, etc"
31        return getattr(math, op_name)
32
33    @staticmethod
34    def getVectorPandaOp(op_name):
35        "Returns a valid DataFrame vector operator"
36        return getattr(pd.DataFrame, op_name)
37
38    @staticmethod
39    def getTwoValuePandaOp(op_name, pd_obj):
40        "Returns a valid two value DataFrame or Series operator"
41        return getattr(type(pd_obj), "__" + op_name + "__")
42
43    @staticmethod
44    def readcsv(filedict, narm):
45        data = pd.read_csv(
46            filedict["file"],
47            header=filedict["header"],
48            index_col=filedict["row_names"],
49            keep_default_na=narm,
50            nrows=filedict["nrows"],
51            skipfooter=filedict["skipfooter"],
52            skip_blank_lines=filedict["skip_blank_lines"],
53            sep='\t'
54        )
55        # Fix whitespace issues in index or column names
56        data.columns = [col.strip() if type(col) is str else col
57                        for col in data.columns]
58        data.index = [row.strip() if type(row) is str else row
59                      for row in data.index]
60        return data
61
62    @staticmethod
63    def rangemaker(tab):
64        # e.g. "1:3,2:-2" specifies "1,2,3,2,1,0,-1,-2" to give [0,1,2,1,0,-1,-2]
65        # Positive indices are decremented by 1 to reference 0-base numbering
66        # Negative indices are unaltered, so that -1 refers to the last column
67        out = []
68        err_mess = None
69        for ranges in tab.split(","):
70            nums = ranges.split(":")
71            if len(nums) == 1:
72                numb = int(nums[0])
73                # Positive numbers get decremented.
74                # i.e. column "3" refers to index 2
75                #      column "-1" still refers to index -1
76                if numb != 0:
77                    out.append(numb if (numb < 0) else (numb - 1))
78                else:
79                    err_mess = "Please do not use 0 as an index"
80            elif len(nums) == 2:
81                left, right = map(int, nums)
82                if 0 in (left, right):
83                    err_mess = "Please do not use 0 as an index"
84                elif left < right:
85                    if left > 0:  # and right > 0 too
86                        # 1:3 to 0,1,2
87                        out.extend(range(left - 1, right))
88                    elif right < 0:  # and left < 0 too
89                        # -3:-1 to -3,-2,-1
90                        out.extend(range(left, right + 1))
91                    elif left < 0 and right > 0:
92                        # -2:2 to -2,-1,0,1
93                        out.extend(range(left, 0))
94                        out.extend(range(0, right))
95                elif right < left:
96                    if right > 0:  # and left > 0
97                        # 3:1 to 2,1,0
98                        out.extend(range(left - 1, right - 2, -1))
99                    elif left < 0:  # and right < 0
100                        # -1:-3 to -1,-2,-3
101                        out.extend(range(left, right - 1, -1))
102                    elif right < 0 and left > 0:
103                        # 2:-2 to 1,0,-1,-2
104                        out.extend(range(left - 1, right - 1, -1))
105                else:
106                    err_mess = "%s should not be equal or contain a zero" % nums
107            if err_mess:
108                print(err_mess)
109                return None
110        return out
111
112
113# Set decimal precision
114pd.options.display.precision = uc.Default["precision"]
115
116user_mode = uc.Default["user_mode"]
117user_mode_single = None
118out_table = None
119params = uc.Data["params"]
120
121if user_mode == "single":
122    # Read in TSV file
123    data = Utils.readcsv(uc.Data["tables"][0], uc.Default["narm"])
124    user_mode_single = params["user_mode_single"]
125
126    if user_mode_single == "precision":
127        # Useful for changing decimal precision on write out
128        out_table = data
129
130    elif user_mode_single == "select":
131        cols_specified = params["select_cols_wanted"]
132        rows_specified = params["select_rows_wanted"]
133
134        # Select all indexes if empty array of values
135        if cols_specified:
136            cols_specified = Utils.rangemaker(cols_specified)
137        else:
138            cols_specified = range(len(data.columns))
139        if rows_specified:
140            rows_specified = Utils.rangemaker(rows_specified)
141        else:
142            rows_specified = range(len(data))
143
144        # do not use duplicate indexes
145        # e.g. [2,3,2,5,5,4,2] to [2,3,5,4]
146        nodupes_col = not params["select_cols_unique"]
147        nodupes_row = not params["select_rows_unique"]
148
149        if nodupes_col:
150            cols_specified = [x for i, x in enumerate(cols_specified)
151                              if x not in cols_specified[:i]]
152        if nodupes_row:
153            rows_specified = [x for i, x in enumerate(rows_specified)
154                              if x not in rows_specified[:i]]
155
156        out_table = data.iloc[rows_specified, cols_specified]
157
158    elif user_mode_single == "filtersumval":
159        mode = params["filtersumval_mode"]
160        axis = params["filtersumval_axis"]
161        operation = params["filtersumval_op"]
162        compare_operation = params["filtersumval_compare"]
163        value = params["filtersumval_against"]
164        minmatch = params["filtersumval_minmatch"]
165
166        if mode == "operation":
167            # Perform axis operation
168            summary_op = Utils.getVectorPandaOp(operation)
169            axis_summary = summary_op(data, axis=axis)
170            # Perform vector comparison
171            compare_op = Utils.getTwoValuePandaOp(
172                compare_operation, axis_summary
173            )
174            axis_bool = compare_op(axis_summary, value)
175
176        elif mode == "element":
177            if operation.startswith("str_"):
178                data = data.astype("str")
179                value = str(value)
180                # Convert str_eq to eq
181                operation = operation[4:]
182            else:
183                value = float(value)
184
185            op = Utils.getTwoValuePandaOp(operation, data)
186            bool_mat = op(data, value)
187            axis_bool = np.sum(bool_mat, axis=axis) >= minmatch
188
189        out_table = data.loc[:, axis_bool] if axis == 0 else data.loc[axis_bool, :]
190
191    elif user_mode_single == "matrixapply":
192        # 0 - column, 1 - row
193        axis = params["matrixapply_dimension"]
194        # sd, mean, max, min, sum, median, summary
195        operation = params["matrixapply_op"]
196
197        if operation is None:
198            use_custom = params["matrixapply_custom"]
199            if use_custom:
200                custom_func = params["matrixapply_custom_func"]
201
202                def fun(vec):
203                    """Dummy Function"""
204                    return vec
205
206                ss = Safety(custom_func, ['vec'], 'pd.Series')
207                fun_string = ss.generateFunction()
208                exec(fun_string)  # SUPER DUPER SAFE...
209
210                out_table = data.apply(fun, axis)
211            else:
212                print("No operation given")
213                exit(-1)
214        else:
215            op = getattr(pd.DataFrame, operation)
216            out_table = op(data, axis)
217
218    elif user_mode_single == "element":
219        # lt, gt, ge, etc.
220        operation = params["element_op"]
221        bool_mat = None
222        if operation is not None:
223            if operation == "rowcol":
224                # Select all indexes if empty array of values
225                if "element_cols" in params:
226                    cols_specified = Utils.rangemaker(params["element_cols"])
227                else:
228                    cols_specified = range(len(data.columns))
229                if "element_rows" in params:
230                    rows_specified = Utils.rangemaker(params["element_rows"])
231                else:
232                    rows_specified = range(len(data))
233
234                # Inclusive selection:
235                # - True: Giving a row or column will match all elements in that row or column
236                # - False: Give a row or column will match only elements in both those rows or columns
237                inclusive = params["element_inclusive"]
238
239                # Create a bool matrix (intialised to False) with selected
240                # rows and columns set to True
241                bool_mat = data.copy()
242                bool_mat[:] = False
243                if inclusive:
244                    bool_mat.iloc[rows_specified, :] = True
245                    bool_mat.iloc[:, cols_specified] = True
246                else:
247                    bool_mat.iloc[rows_specified, cols_specified] = True
248
249            else:
250                op = Utils.getTwoValuePandaOp(operation, data)
251                value = params["element_value"]
252                try:
253                    # Could be numeric
254                    value = float(value)
255                except ValueError:
256                    pass
257                # generate filter matrix of True/False values
258                bool_mat = op(data, value)
259        else:
260            # implement no filtering through a filter matrix filled with
261            # True values.
262            bool_mat = np.full(data.shape, True)
263
264        # Get the main processing mode
265        mode = params["element_mode"]
266        if mode == "replace":
267            replacement_val = params["element_replace"]
268            out_table = data.mask(
269                bool_mat,
270                data.where(bool_mat).applymap(
271                    lambda x: replacement_val.format(elem=x)
272                )
273            )
274        elif mode == "modify":
275            mod_op = Utils.getOneValueMathOp(params["element_modify_op"])
276            out_table = data.mask(
277                bool_mat, data.where(bool_mat).applymap(mod_op)
278            )
279        elif mode == "scale":
280            scale_op = Utils.getTwoValuePandaOp(
281                params["element_scale_op"], data
282            )
283            scale_value = params["element_scale_value"]
284            out_table = data.mask(
285                bool_mat, scale_op(data.where(bool_mat), scale_value)
286            )
287        elif mode == "custom":
288            element_customop = params["element_customop"]
289
290            def fun(elem):
291                """Dummy Function"""
292                return elem
293
294            ss = Safety(element_customop, ['elem'])
295            fun_string = ss.generateFunction()
296            exec(fun_string)  # SUPER DUPER SAFE...
297
298            out_table = data.mask(
299                bool_mat, data.where(bool_mat).applymap(fun)
300            )
301        else:
302            print("No such element mode!", mode)
303            exit(-1)
304
305    elif user_mode_single == "fulltable":
306        general_mode = params["mode"]
307
308        if general_mode == "transpose":
309            out_table = data.T
310        elif general_mode == "melt":
311            melt_ids = params["MELT"]["melt_ids"]
312            melt_values = params["MELT"]["melt_values"]
313
314            out_table = pd.melt(data, id_vars=melt_ids, value_vars=melt_values)
315        elif general_mode == "pivot":
316            pivot_index = params["PIVOT"]["pivot_index"]
317            pivot_column = params["PIVOT"]["pivot_columns"]
318            pivot_values = params["PIVOT"]["pivot_values"]
319            pivot_aggfunc = params["PIVOT"]["pivot_aggfunc"]
320
321            if not pivot_aggfunc:
322                out_table = data.pivot(
323                    index=pivot_index, columns=pivot_column, values=pivot_values
324                )
325            else:
326                out_table = data.pivot_table(
327                    index=pivot_index, columns=pivot_column, values=pivot_values,
328                    aggfunc=pivot_aggfunc
329                )
330
331        elif general_mode == "custom":
332            custom_func = params["fulltable_customop"]
333
334            def fun(tableau):
335                """Dummy Function"""
336                return tableau
337
338            ss = Safety(custom_func, ['table'], 'pd.DataFrame')
339            fun_string = ss.generateFunction()
340            exec(fun_string)  # SUPER DUPER SAFE...
341
342            out_table = fun(data)
343
344    else:
345        print("No such mode!", user_mode_single)
346        exit(-1)
347
348
349elif user_mode == "multiple":
350
351    table_sections = uc.Data["tables"]
352
353    if not table_sections:
354        print("Multiple table sets not given!")
355        exit(-1)
356
357    reader_skip = uc.Default["reader_skip"]
358
359    # Data
360    table = []
361    # 1-based handlers for users "table1", "table2", etc.
362    table_names = []
363    # Actual 0-based references "table[0]", "table[1]", etc.
364    table_names_real = []
365
366    # Read and populate tables
367    for x, t_sect in enumerate(table_sections):
368        tmp = Utils.readcsv(t_sect, uc.Default["narm"])
369        table.append(tmp)
370        table_names.append("table" + str(x + 1))
371        table_names_real.append("table[" + str(x) + "]")
372
373    custom_op = params["fulltable_customop"]
374    ss = Safety(custom_op, table_names, 'pd.DataFrame')
375    fun_string = ss.generateFunction()
376    # Change the argument to table
377    fun_string = fun_string.replace("fun(table1):", "fun():")
378    # table1 to table[1]
379    for name, name_real in zip(table_names, table_names_real):
380        fun_string = fun_string.replace(name, name_real)
381
382    fun_string = fun_string.replace("fun():", "fun(table):")
383    exec(fun_string)  # SUPER DUPER SAFE...
384    out_table = fun(table)
385
386else:
387    print("No such mode!", user_mode)
388    exit(-1)
389
390if not isinstance(out_table, (pd.DataFrame, pd.Series)):
391    print('The specified operation did not result in a table to return.')
392    raise RuntimeError(
393        'The operation did not generate a pd.DataFrame or pd.Series to return.'
394    )
395out_parameters = {
396    "sep": "\t",
397    "float_format": "%%.%df" % pd.options.display.precision,
398    "header": uc.Default["out_headers_col"],
399    "index": uc.Default["out_headers_row"]
400}
401if user_mode_single not in ('matrixapply', None):
402    out_parameters["quoting"] = csv.QUOTE_NONE
403
404out_table.to_csv(uc.Default["outtable"], **out_parameters)