From 23fd74c2c5a745e719fdb47b03fd19c945246450 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Wed, 19 Sep 2007 14:49:02 +0000 Subject: [PATCH] Better error handling for tools that use columnlist tool parameter. --- tools/filters/sorter.py | 2 +- tools/filters/uniq.py | 2 +- tools/plotting/histogram.py | 7 +- tools/plotting/scatterplot.py | 13 +- tools/stats/cor.py | 2 +- tools/stats/grouping.py | 341 +++++++++++++++++----------------- 6 files changed, 189 insertions(+), 178 deletions(-) diff --git a/tools/filters/sorter.py b/tools/filters/sorter.py index 808db2dab8a..dcd4489c10d 100644 --- a/tools/filters/sorter.py +++ b/tools/filters/sorter.py @@ -48,7 +48,7 @@ def main(): return -2 column = opts.get("-cols") - if column == None: + if column == None or column == 'None': print "Sort column not specified." return -3 diff --git a/tools/filters/uniq.py b/tools/filters/uniq.py index 723b8fdfefd..49855f77e33 100644 --- a/tools/filters/uniq.py +++ b/tools/filters/uniq.py @@ -67,7 +67,7 @@ def main(): return -3 columns = opts.get("-c") - if columns == None: + if columns == None or columns == 'None': print "Columns not specified." return -4 diff --git a/tools/plotting/histogram.py b/tools/plotting/histogram.py index ab022c9da78..91baebd0265 100644 --- a/tools/plotting/histogram.py +++ b/tools/plotting/histogram.py @@ -12,8 +12,11 @@ def main(): # Handle input params in_fname = sys.argv[1] - out_fname = sys.argv[2] - column = int( sys.argv[3] ) - 1 + out_fname = sys.argv[2] + try: + column = int( sys.argv[3] ) - 1 + except: + stop_err( "..Column not specified, your query does not contain a column of numerical data." ) title = sys.argv[4] xlab = sys.argv[5] breaks = int( sys.argv[6] ) diff --git a/tools/plotting/scatterplot.py b/tools/plotting/scatterplot.py index f082eb9ff48..ec909c01c8e 100644 --- a/tools/plotting/scatterplot.py +++ b/tools/plotting/scatterplot.py @@ -4,17 +4,18 @@ import sys from rpy import * -def fail( message ): - print >> sys.stderr, message +def stop_err(msg): + sys.stderr.write(msg) sys.exit() def main(): in_fname = sys.argv[1] out_fname = sys.argv[2] - - columns = int( sys.argv[3] ) - 1, int( sys.argv[4] ) - 1 - + try: + columns = int( sys.argv[3] ) - 1, int( sys.argv[4] ) - 1 + except: + stop_err( "..Columns not specified, your query does not contain a column of numerical data." ) title = sys.argv[5] xlab = sys.argv[6] ylab = sys.argv[7] @@ -36,7 +37,7 @@ def main(): break fields = line.split( "\t" ) if len( fields ) <= column: - return fail( "Column %d on line %d missing, line: %s" % ( column+1, i, line ) ) + stop_err( "Column %d on line %d missing, line: %s" % ( column+1, i, line ) ) val = fields[column] if val.lower() == "na": row.append( float( "nan" ) ) diff --git a/tools/stats/cor.py b/tools/stats/cor.py index ff466d7559a..2460b9260de 100644 --- a/tools/stats/cor.py +++ b/tools/stats/cor.py @@ -22,7 +22,7 @@ def main(): try: columns = map( int, sys.argv[3].split( ',' ) ) except: - stop_err("Problem determining columns '%s'" %sys.argv[3]) + stop_err( "Problem determining columns, perhaps your query does not contain a column of numerical data." ) matrix = [] skipped_lines = 0 diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index ad9bd785263..d5808634db5 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -6,185 +6,192 @@ This tool provides the SQL "group by" functionality. import sys, string, re, commands, tempfile from rpy import * -inputfile = sys.argv[2] - -ops = [] -cols = [] -elems = [] - -for var in sys.argv[4:]: - ops.append(var.split()[0]) - cols.append(var.split()[1]) - -""" -At this point, ops and cols will look something like this: -ops: ['mean', 'min', 'c'] -cols: ['1', '3', '4'] -""" - -for i, line in enumerate( file ( inputfile )): - line = line.rstrip('\r\n') - if len( line )>0 and not line.startswith( '#' ): - elems = line.split( '\t' ) - break - if i == 30: - break # Hopefully we'll never get here... - -if len( elems )<1: - print >> sys.stderr, "The data in your input dataset is either missing or not formatted properly." +def stop_err(msg): + sys.stderr.write(msg) sys.exit() -group_col = int( sys.argv[3] )-1 - -for k,col in enumerate(cols): - col = int(col)-1 - if ops[k] != 'c': - """ - We'll get here only if the user didn't choose 'Concatenate', which is the - only aggregation function that can be used on columns containing strings. - """ - try: - float( elems[col] ) - except: - print >> sys.stderr, "Operation '%s' cannot be performed on non-numeric column %d containing value %s." %(ops[k], col+1, elems[col]) - sys.exit() - -tmpfile = tempfile.NamedTemporaryFile() - -try: - """ - The -k option for the Posix sort command is as follows: - -k, --key=POS1[,POS2] - start a key at POS1, end it at POS2 (origin 1) - In other words, column positions start at 1 rather than 0, so - we need to add 1 to group_col. - """ - command_line = "sort -f -k " + str(group_col+1) + " -o " + tmpfile.name + " " + inputfile -except Exception, exc: - print >> sys.stderr, 'Initialization error -> %s' % exc - sys.exit() - -error_code, stdout = commands.getstatusoutput(command_line) - -if error_code != 0: - print >> sys.stderr, "Sorting input dataset resulted in error: ", error_code, stdout - sys.exit() +def main(): + inputfile = sys.argv[2] -prev_item = "" -prev_vals = [] -skipped_lines = 0 -first_invalid_line = 0 -invalid_line = '' -invalid_value = '' -invalid_column = 0 -fout = open(sys.argv[1], "w") - -for ii, line in enumerate( file( tmpfile.name )): - if line and not line.startswith( '#' ): - try: - fields = line.split("\t") - item = fields[group_col] - if prev_item != "": - """ - At this level, we're grouping on values (item and prev_item) in group_col - """ - if item == prev_item: + ops = [] + cols = [] + elems = [] + + for var in sys.argv[4:]: + ops.append(var.split()[0]) + cols.append(var.split()[1]) + + """ + At this point, ops and cols will look something like this: + ops: ['mean', 'min', 'c'] + cols: ['1', '3', '4'] + """ + + for i, line in enumerate( file ( inputfile )): + line = line.rstrip('\r\n') + if len( line )>0 and not line.startswith( '#' ): + elems = line.split( '\t' ) + break + if i == 30: + break # Hopefully we'll never get here... + + if len( elems )<1: + print >> sys.stderr, "The data in your input dataset is either missing or not formatted properly." + sys.exit() + + try: + group_col = int( sys.argv[3] )-1 + except: + stop_err( "Group column not specified." ) + + for k,col in enumerate(cols): + col = int(col)-1 + if ops[k] != 'c': + """ + We'll get here only if the user didn't choose 'Concatenate', which is the + only aggregation function that can be used on columns containing strings. + """ + try: + float( elems[col] ) + except: + stop_err( "Operation '%s' cannot be performed on non-numeric column %d containing value %s." %( ops[k], col+1, elems[col] )) + + tmpfile = tempfile.NamedTemporaryFile() + + try: + """ + The -k option for the Posix sort command is as follows: + -k, --key=POS1[,POS2] + start a key at POS1, end it at POS2 (origin 1) + In other words, column positions start at 1 rather than 0, so + we need to add 1 to group_col. + """ + command_line = "sort -f -k " + str(group_col+1) + " -o " + tmpfile.name + " " + inputfile + except Exception, exc: + stop_err( 'Initialization error -> %s' %str(exc) ) + + error_code, stdout = commands.getstatusoutput(command_line) + + if error_code != 0: + stop_err( "Sorting input dataset resulted in error: %s: %s" %( error_code, stdout )) + + prev_item = "" + prev_vals = [] + skipped_lines = 0 + first_invalid_line = 0 + invalid_line = '' + invalid_value = '' + invalid_column = 0 + fout = open(sys.argv[1], "w") + + for ii, line in enumerate( file( tmpfile.name )): + if line and not line.startswith( '#' ): + try: + fields = line.split("\t") + item = fields[group_col] + if prev_item != "": """ - Keep iterating and storing values until a new value is encountered. + At this level, we're grouping on values (item and prev_item) in group_col """ - for i, col in enumerate(cols): - col = int(col)-1 - valid = True + if item == prev_item: """ - Before appending the current value, make sure it is numeric if the - operation for the column requires it. + Keep iterating and storing values until a new value is encountered. """ - if ops[i] != 'c': - try: - float( fields[col].strip()) - except: - valid = False - skipped_lines += 1 - if not first_invalid_line: - first_invalid_line = ii+1 - invalid_value = fields[col] - invalid_column = col+1 - if valid: - prev_vals[i].append(fields[col].strip()) - else: + for i, col in enumerate(cols): + col = int(col)-1 + valid = True + """ + Before appending the current value, make sure it is numeric if the + operation for the column requires it. + """ + if ops[i] != 'c': + try: + float( fields[col].strip()) + except: + valid = False + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = ii+1 + invalid_value = fields[col] + invalid_column = col+1 + if valid: + prev_vals[i].append(fields[col].strip()) + else: + """ + When a new value is encountered, write the previous value and the + corresponding aggregate values into the output file. This works + due to the sort on group_col we've applied to the data above. + """ + out_str = prev_item + + for i, op in enumerate( ops ): + rfunc = "r." + op + if op != 'c': + for j, elem in enumerate( prev_vals[i] ): + prev_vals[i][j] = float( elem ) + rout = "%.2f" %( eval( rfunc )( prev_vals[i] )) + else: + rout = eval( rfunc )( prev_vals[i] ) + + out_str += "\t" + str(rout) + + print >>fout, out_str + + prev_item = item + prev_vals = [] + for col in cols: + col = int(col)-1 + val_list = [] + val_list.append(fields[col].strip()) + prev_vals.append(val_list) + else: """ - When a new value is encountered, write the previous value and the - corresponding aggregate values into the output file. This works - due to the sort on group_col we've applied to the data above. + This only occurs once, right at the start of the iteration. """ - out_str = prev_item - - for i, op in enumerate( ops ): - rfunc = "r." + op - if op != 'c': - for j, elem in enumerate( prev_vals[i] ): - prev_vals[i][j] = float( elem ) - rout = "%.2f" %( eval( rfunc )( prev_vals[i] )) - else: - rout = eval( rfunc )( prev_vals[i] ) - - out_str += "\t" + str(rout) - - print >>fout, out_str - - prev_item = item - prev_vals = [] + prev_item = item for col in cols: col = int(col)-1 val_list = [] val_list.append(fields[col].strip()) prev_vals.append(val_list) - else: - """ - This only occurs once, right at the start of the iteration. - """ - prev_item = item - for col in cols: - col = int(col)-1 - val_list = [] - val_list.append(fields[col].strip()) - prev_vals.append(val_list) + + except Exception, exc: + stop_err( "Error executing aggregation functions: %s" %str(exc) ) + else: + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = ii+1 + + """ + Handle the last grouped value + """ + out_str = prev_item + + for i, op in enumerate(ops): + rfunc = "r." + op + if op != 'c': + for j, elem in enumerate( prev_vals[i] ): + prev_vals[i][j] = float( elem ) + rout = "%.2f" %( eval( rfunc )( prev_vals[i] )) + else: + rout = eval( rfunc )( prev_vals[i] ) + + out_str += "\t" + str( rout ) + + print >>fout, out_str + + """ + Generate a useful info message. + """ + msg = "--Group by c%d: " %(group_col+1) + for i,op in enumerate(ops): + if op == 'c': + op = 'concat' + msg += op + "[c" + cols[i] + "] " + if skipped_lines > 0: + msg+= "--skipped %d invalid lines starting with line %d. Value '%s' in column %d is not numeric." % ( skipped_lines, first_invalid_line, invalid_value, invalid_column ) + + print msg - except Exception, exc: - print >> sys.stderr, "Error executing aggregation functions: %s" %exc - sys.exit() - else: - skipped_lines += 1 - if not first_invalid_line: - first_invalid_line = ii+1 - -""" -Handle the last grouped value -""" -out_str = prev_item - -for i, op in enumerate(ops): - rfunc = "r." + op - if op != 'c': - for j, elem in enumerate( prev_vals[i] ): - prev_vals[i][j] = float( elem ) - rout = "%.2f" %( eval( rfunc )( prev_vals[i] )) - else: - rout = eval( rfunc )( prev_vals[i] ) - - out_str += "\t" + str( rout ) - -print >>fout, out_str - -""" -Generate a useful info message. -""" -msg = "--Group by c%d: " %(group_col+1) -for i,op in enumerate(ops): - if op == 'c': - op = 'concat' - msg += op + "[c" + cols[i] + "] " -if skipped_lines > 0: - msg+= "--skipped %d invalid lines starting with line %d. Value '%s' in column %d is not numeric." % ( skipped_lines, first_invalid_line, invalid_value, invalid_column ) - -print msg +if __name__ == "__main__": + main()