upstream-extended/iuc/tools/table_compute/scripts/table_compute.py
1#!/usr/bin/env python3
2"""
3Table Compute tool - a wrapper around pandas with parameter input validation.
4"""
5
6
7__version__ = "0.9.2"
8
9import csv
10import math
11from sys import argv
12
13import numpy as np
14import pandas as pd
15from safety import Safety
16
17if len(argv) == 2 and argv[1] == "--version":
18 print(__version__)
19 exit(0)
20
21# The import below should be generated in the same directory as
22# the table_compute.py script.
23# It is placed here so that the --version switch does not fail
24import userconfig as uc # noqa: I100,I202
25
26
27class Utils:
28 @staticmethod
29 def getOneValueMathOp(op_name):
30 "Returns a simple one value math operator such as log, sqrt, etc"
31 return getattr(math, op_name)
32
33 @staticmethod
34 def getVectorPandaOp(op_name):
35 "Returns a valid DataFrame vector operator"
36 return getattr(pd.DataFrame, op_name)
37
38 @staticmethod
39 def getTwoValuePandaOp(op_name, pd_obj):
40 "Returns a valid two value DataFrame or Series operator"
41 return getattr(type(pd_obj), "__" + op_name + "__")
42
43 @staticmethod
44 def readcsv(filedict, narm):
45 data = pd.read_csv(
46 filedict["file"],
47 header=filedict["header"],
48 index_col=filedict["row_names"],
49 keep_default_na=narm,
50 nrows=filedict["nrows"],
51 skipfooter=filedict["skipfooter"],
52 skip_blank_lines=filedict["skip_blank_lines"],
53 sep='\t'
54 )
55 # Fix whitespace issues in index or column names
56 data.columns = [col.strip() if type(col) is str else col
57 for col in data.columns]
58 data.index = [row.strip() if type(row) is str else row
59 for row in data.index]
60 return data
61
62 @staticmethod
63 def rangemaker(tab):
64 # e.g. "1:3,2:-2" specifies "1,2,3,2,1,0,-1,-2" to give [0,1,2,1,0,-1,-2]
65 # Positive indices are decremented by 1 to reference 0-base numbering
66 # Negative indices are unaltered, so that -1 refers to the last column
67 out = []
68 err_mess = None
69 for ranges in tab.split(","):
70 nums = ranges.split(":")
71 if len(nums) == 1:
72 numb = int(nums[0])
73 # Positive numbers get decremented.
74 # i.e. column "3" refers to index 2
75 # column "-1" still refers to index -1
76 if numb != 0:
77 out.append(numb if (numb < 0) else (numb - 1))
78 else:
79 err_mess = "Please do not use 0 as an index"
80 elif len(nums) == 2:
81 left, right = map(int, nums)
82 if 0 in (left, right):
83 err_mess = "Please do not use 0 as an index"
84 elif left < right:
85 if left > 0: # and right > 0 too
86 # 1:3 to 0,1,2
87 out.extend(range(left - 1, right))
88 elif right < 0: # and left < 0 too
89 # -3:-1 to -3,-2,-1
90 out.extend(range(left, right + 1))
91 elif left < 0 and right > 0:
92 # -2:2 to -2,-1,0,1
93 out.extend(range(left, 0))
94 out.extend(range(0, right))
95 elif right < left:
96 if right > 0: # and left > 0
97 # 3:1 to 2,1,0
98 out.extend(range(left - 1, right - 2, -1))
99 elif left < 0: # and right < 0
100 # -1:-3 to -1,-2,-3
101 out.extend(range(left, right - 1, -1))
102 elif right < 0 and left > 0:
103 # 2:-2 to 1,0,-1,-2
104 out.extend(range(left - 1, right - 1, -1))
105 else:
106 err_mess = "%s should not be equal or contain a zero" % nums
107 if err_mess:
108 print(err_mess)
109 return None
110 return out
111
112
113# Set decimal precision
114pd.options.display.precision = uc.Default["precision"]
115
116user_mode = uc.Default["user_mode"]
117user_mode_single = None
118out_table = None
119params = uc.Data["params"]
120
121if user_mode == "single":
122 # Read in TSV file
123 data = Utils.readcsv(uc.Data["tables"][0], uc.Default["narm"])
124 user_mode_single = params["user_mode_single"]
125
126 if user_mode_single == "precision":
127 # Useful for changing decimal precision on write out
128 out_table = data
129
130 elif user_mode_single == "select":
131 cols_specified = params["select_cols_wanted"]
132 rows_specified = params["select_rows_wanted"]
133
134 # Select all indexes if empty array of values
135 if cols_specified:
136 cols_specified = Utils.rangemaker(cols_specified)
137 else:
138 cols_specified = range(len(data.columns))
139 if rows_specified:
140 rows_specified = Utils.rangemaker(rows_specified)
141 else:
142 rows_specified = range(len(data))
143
144 # do not use duplicate indexes
145 # e.g. [2,3,2,5,5,4,2] to [2,3,5,4]
146 nodupes_col = not params["select_cols_unique"]
147 nodupes_row = not params["select_rows_unique"]
148
149 if nodupes_col:
150 cols_specified = [x for i, x in enumerate(cols_specified)
151 if x not in cols_specified[:i]]
152 if nodupes_row:
153 rows_specified = [x for i, x in enumerate(rows_specified)
154 if x not in rows_specified[:i]]
155
156 out_table = data.iloc[rows_specified, cols_specified]
157
158 elif user_mode_single == "filtersumval":
159 mode = params["filtersumval_mode"]
160 axis = params["filtersumval_axis"]
161 operation = params["filtersumval_op"]
162 compare_operation = params["filtersumval_compare"]
163 value = params["filtersumval_against"]
164 minmatch = params["filtersumval_minmatch"]
165
166 if mode == "operation":
167 # Perform axis operation
168 summary_op = Utils.getVectorPandaOp(operation)
169 axis_summary = summary_op(data, axis=axis)
170 # Perform vector comparison
171 compare_op = Utils.getTwoValuePandaOp(
172 compare_operation, axis_summary
173 )
174 axis_bool = compare_op(axis_summary, value)
175
176 elif mode == "element":
177 if operation.startswith("str_"):
178 data = data.astype("str")
179 value = str(value)
180 # Convert str_eq to eq
181 operation = operation[4:]
182 else:
183 value = float(value)
184
185 op = Utils.getTwoValuePandaOp(operation, data)
186 bool_mat = op(data, value)
187 axis_bool = np.sum(bool_mat, axis=axis) >= minmatch
188
189 out_table = data.loc[:, axis_bool] if axis == 0 else data.loc[axis_bool, :]
190
191 elif user_mode_single == "matrixapply":
192 # 0 - column, 1 - row
193 axis = params["matrixapply_dimension"]
194 # sd, mean, max, min, sum, median, summary
195 operation = params["matrixapply_op"]
196
197 if operation is None:
198 use_custom = params["matrixapply_custom"]
199 if use_custom:
200 custom_func = params["matrixapply_custom_func"]
201
202 def fun(vec):
203 """Dummy Function"""
204 return vec
205
206 ss = Safety(custom_func, ['vec'], 'pd.Series')
207 fun_string = ss.generateFunction()
208 exec(fun_string) # SUPER DUPER SAFE...
209
210 out_table = data.apply(fun, axis)
211 else:
212 print("No operation given")
213 exit(-1)
214 else:
215 op = getattr(pd.DataFrame, operation)
216 out_table = op(data, axis)
217
218 elif user_mode_single == "element":
219 # lt, gt, ge, etc.
220 operation = params["element_op"]
221 bool_mat = None
222 if operation is not None:
223 if operation == "rowcol":
224 # Select all indexes if empty array of values
225 if "element_cols" in params:
226 cols_specified = Utils.rangemaker(params["element_cols"])
227 else:
228 cols_specified = range(len(data.columns))
229 if "element_rows" in params:
230 rows_specified = Utils.rangemaker(params["element_rows"])
231 else:
232 rows_specified = range(len(data))
233
234 # Inclusive selection:
235 # - True: Giving a row or column will match all elements in that row or column
236 # - False: Give a row or column will match only elements in both those rows or columns
237 inclusive = params["element_inclusive"]
238
239 # Create a bool matrix (intialised to False) with selected
240 # rows and columns set to True
241 bool_mat = data.copy()
242 bool_mat[:] = False
243 if inclusive:
244 bool_mat.iloc[rows_specified, :] = True
245 bool_mat.iloc[:, cols_specified] = True
246 else:
247 bool_mat.iloc[rows_specified, cols_specified] = True
248
249 else:
250 op = Utils.getTwoValuePandaOp(operation, data)
251 value = params["element_value"]
252 try:
253 # Could be numeric
254 value = float(value)
255 except ValueError:
256 pass
257 # generate filter matrix of True/False values
258 bool_mat = op(data, value)
259 else:
260 # implement no filtering through a filter matrix filled with
261 # True values.
262 bool_mat = np.full(data.shape, True)
263
264 # Get the main processing mode
265 mode = params["element_mode"]
266 if mode == "replace":
267 replacement_val = params["element_replace"]
268 out_table = data.mask(
269 bool_mat,
270 data.where(bool_mat).applymap(
271 lambda x: replacement_val.format(elem=x)
272 )
273 )
274 elif mode == "modify":
275 mod_op = Utils.getOneValueMathOp(params["element_modify_op"])
276 out_table = data.mask(
277 bool_mat, data.where(bool_mat).applymap(mod_op)
278 )
279 elif mode == "scale":
280 scale_op = Utils.getTwoValuePandaOp(
281 params["element_scale_op"], data
282 )
283 scale_value = params["element_scale_value"]
284 out_table = data.mask(
285 bool_mat, scale_op(data.where(bool_mat), scale_value)
286 )
287 elif mode == "custom":
288 element_customop = params["element_customop"]
289
290 def fun(elem):
291 """Dummy Function"""
292 return elem
293
294 ss = Safety(element_customop, ['elem'])
295 fun_string = ss.generateFunction()
296 exec(fun_string) # SUPER DUPER SAFE...
297
298 out_table = data.mask(
299 bool_mat, data.where(bool_mat).applymap(fun)
300 )
301 else:
302 print("No such element mode!", mode)
303 exit(-1)
304
305 elif user_mode_single == "fulltable":
306 general_mode = params["mode"]
307
308 if general_mode == "transpose":
309 out_table = data.T
310 elif general_mode == "melt":
311 melt_ids = params["MELT"]["melt_ids"]
312 melt_values = params["MELT"]["melt_values"]
313
314 out_table = pd.melt(data, id_vars=melt_ids, value_vars=melt_values)
315 elif general_mode == "pivot":
316 pivot_index = params["PIVOT"]["pivot_index"]
317 pivot_column = params["PIVOT"]["pivot_columns"]
318 pivot_values = params["PIVOT"]["pivot_values"]
319 pivot_aggfunc = params["PIVOT"]["pivot_aggfunc"]
320
321 if not pivot_aggfunc:
322 out_table = data.pivot(
323 index=pivot_index, columns=pivot_column, values=pivot_values
324 )
325 else:
326 out_table = data.pivot_table(
327 index=pivot_index, columns=pivot_column, values=pivot_values,
328 aggfunc=pivot_aggfunc
329 )
330
331 elif general_mode == "custom":
332 custom_func = params["fulltable_customop"]
333
334 def fun(tableau):
335 """Dummy Function"""
336 return tableau
337
338 ss = Safety(custom_func, ['table'], 'pd.DataFrame')
339 fun_string = ss.generateFunction()
340 exec(fun_string) # SUPER DUPER SAFE...
341
342 out_table = fun(data)
343
344 else:
345 print("No such mode!", user_mode_single)
346 exit(-1)
347
348
349elif user_mode == "multiple":
350
351 table_sections = uc.Data["tables"]
352
353 if not table_sections:
354 print("Multiple table sets not given!")
355 exit(-1)
356
357 reader_skip = uc.Default["reader_skip"]
358
359 # Data
360 table = []
361 # 1-based handlers for users "table1", "table2", etc.
362 table_names = []
363 # Actual 0-based references "table[0]", "table[1]", etc.
364 table_names_real = []
365
366 # Read and populate tables
367 for x, t_sect in enumerate(table_sections):
368 tmp = Utils.readcsv(t_sect, uc.Default["narm"])
369 table.append(tmp)
370 table_names.append("table" + str(x + 1))
371 table_names_real.append("table[" + str(x) + "]")
372
373 custom_op = params["fulltable_customop"]
374 ss = Safety(custom_op, table_names, 'pd.DataFrame')
375 fun_string = ss.generateFunction()
376 # Change the argument to table
377 fun_string = fun_string.replace("fun(table1):", "fun():")
378 # table1 to table[1]
379 for name, name_real in zip(table_names, table_names_real):
380 fun_string = fun_string.replace(name, name_real)
381
382 fun_string = fun_string.replace("fun():", "fun(table):")
383 exec(fun_string) # SUPER DUPER SAFE...
384 out_table = fun(table)
385
386else:
387 print("No such mode!", user_mode)
388 exit(-1)
389
390if not isinstance(out_table, (pd.DataFrame, pd.Series)):
391 print('The specified operation did not result in a table to return.')
392 raise RuntimeError(
393 'The operation did not generate a pd.DataFrame or pd.Series to return.'
394 )
395out_parameters = {
396 "sep": "\t",
397 "float_format": "%%.%df" % pd.options.display.precision,
398 "header": uc.Default["out_headers_col"],
399 "index": uc.Default["out_headers_row"]
400}
401if user_mode_single not in ('matrixapply', None):
402 out_parameters["quoting"] = csv.QUOTE_NONE
403
404out_table.to_csv(uc.Default["outtable"], **out_parameters)