From 6d72d59f0958e7eed0c23a0956aee975e50e9242 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Fri, 17 Aug 2007 14:23:42 +0000 Subject: [PATCH] Fixes for the grouping tool. --- test-data/1.tabular | 6 + tools/stats/grouping.py | 237 +++++++++++++++++++++++++-------------- tools/stats/grouping.xml | 55 +++++---- 3 files changed, 192 insertions(+), 106 deletions(-) create mode 100644 test-data/1.tabular diff --git a/test-data/1.tabular b/test-data/1.tabular new file mode 100644 index 00000000000..71b7e41197b --- /dev/null +++ b/test-data/1.tabular @@ -0,0 +1,6 @@ +chr22 1000 NM_17 +chr22 2000 NM_18 +chr10 2200 NM_10 +chr10 hap test +chr10 1200 NM_11 +chr22 1600 NM_19 \ No newline at end of file diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index 5b6c28bc6af..02b7d9398a7 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -6,114 +6,181 @@ This tool provides the SQL "group by" functionality. import sys, string, re, commands, tempfile from rpy import * -fout = open(sys.argv[1], "w") inputfile = sys.argv[2] ops = [] cols = [] +elems = [] + for var in sys.argv[4:]: ops.append(var.split()[0]) cols.append(var.split()[1]) -if os.path.exists( inputfile ): - for line in open( inputfile ): - line = line.strip() - if line and not line.startswith( '#' ): - elems = line.split( '\t' ) - break -else: - print 'The data file you selected for filtering does not exist.' +""" +At this point, ops and cols will look something like this: +ops: ['mean', 'min', 'c'] +cols: ['1', '3', '4'] +""" + +for i, line in enumerate( file ( inputfile )): + line = line.rstrip('\r\n') + if len( line )>0 and not line.startswith( '#' ): + elems = line.split( '\t' ) + break + if i == 30: + break # Hopefully we'll never get here... + +if len( elems )<1: + print >> sys.stderr, "The data in your input dataset is either missing or not formatted properly." sys.exit() - -groupcol = string.atoi(sys.argv[3]) -if groupcol > len( elems ): - print >> sys.stderr, "Column %d does not exist." %(groupcol) - sys.exit() -groupcol = groupcol-1 + +group_col = int( sys.argv[3] )-1 for k,col in enumerate(cols): - col = int(col) - if col > len( elems ): - print >> sys.stderr, "Column %d does not exist." %(col) - sys.exit() - else: - if ops[k] != "c": - try: - assert float(elems[col-1]) - except: - print >> sys.stderr, "Operation '%s' cannot be performed on non-numeric column %d." %(ops[k],col) - sys.exit() + col = int(col)-1 + if ops[k] != 'c': + """ + We'll get here only if the user didn't choose 'Concatenate', which is the + only aggregation function that can be used on columns containing strings. + """ + try: + map( float, elems[col] ) + except: + print >> sys.stderr, "Operation '%s' cannot be performed on non-numeric column %d containing value %s." %(ops[k], col+1, elems[col]) + sys.exit() + +tmpfile = tempfile.NamedTemporaryFile() -tmpfile = tempfile.NamedTemporaryFile() try: - commandline = "sort -f "+"+"+str(groupcol)+" -o "+tmpfile.name+" "+inputfile + """ + The -k option for the Posix sort command is as follows: + -k, --key=POS1[,POS2] + start a key at POS1, end it at POS2 (origin 1) + In other words, column positions start at 1 rather than 0, so + we need to add 1 to group_col. + """ + command_line = "sort -f -k " + str(group_col+1) + " -o " + tmpfile.name + " " + inputfile except Exception, exc: - print >>sys.stdout, 'Initialization error -> %s' % exc + print >> sys.stderr, 'Initialization error -> %s' % exc sys.exit() -errorcode, stdout = commands.getstatusoutput(commandline) -previtem = "" -prevvals = [] +error_code, stdout = commands.getstatusoutput(command_line) + +if error_code != 0: + print >> sys.stderr, "Sorting input dataset resulted in error: ", error_code, stdout + sys.exit() + +prev_item = "" +prev_vals = [] skipped_lines = 0 first_invalid_line = 0 -invalid_line = None +invalid_line = '' +fout = open(sys.argv[1], "w") -for line in open(tmpfile.name): - if line and (not line.startswith( '#' )) and line != '': +for ii, line in enumerate( file( tmpfile.name )): + if line and not line.startswith( '#' ): try: fields = line.split("\t") - item = fields[groupcol] - if previtem != "": - if item == previtem: #Keep iterating and storing values till a new item is encountered. - previtem = item - for i,col in enumerate(cols): - col = string.atoi(col) - col = col-1 - prevvals[i].append(fields[col].strip()) - else: #When a new item is encountered, write the previous item and the corresponding aggregate values into the output file. - outstr = previtem - try: - for i,op in enumerate(ops): - rfunc = "r." + op - if op != 'c': - for j,elem in enumerate(prevvals[i]): - prevvals[i][j] = float(elem) - rout = "%.2f" %(eval(rfunc)(prevvals[i])) - else: - rout = eval(rfunc)(prevvals[i]) - outstr += "\t" + str(rout) - print >>fout, outstr - except: - skipped_lines += 1 - previtem = item - prevvals = [] - for col in cols: - col = string.atoi(col) - col = col-1 - vallist = [] - vallist.append(fields[col].strip()) - prevvals.append(vallist) - else: #visited only once right at the start of the iteration. - previtem = item - for col in cols: - col = string.atoi(col) - col = col-1 - vallist = [] - vallist.append(fields[col].strip()) - prevvals.append(vallist) - except: - pass + item = fields[group_col] + if prev_item != "": + """ + At this level, we're grouping on values (item and prev_item) in group_col + """ + if item == prev_item: + """ + Keep iterating and storing values until a new value is encountered. + """ + for i, col in enumerate(cols): + col = int(col)-1 + valid = True + """ + Before appending the current value, make sure it is numeric if the + operation for the column requires it. + """ + if ops[i] != 'c': + try: + float( fields[col].strip()) + except: + valid = False + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = ii+1 + if valid: + prev_vals[i].append(fields[col].strip()) + else: + """ + When a new value is encountered, write the previous value and the + corresponding aggregate values into the output file. This works + due to the sort on group_col we've applied to the data above. + """ + out_str = prev_item -outstr = previtem -for i,op in enumerate(ops): + for i, op in enumerate( ops ): + rfunc = "r." + op + if op != 'c': + for j, elem in enumerate( prev_vals[i] ): + prev_vals[i][j] = float( elem ) + rout = "%.2f" %( eval( rfunc )( prev_vals[i] )) + else: + rout = eval( rfunc )( prev_vals[i] ) + + out_str += "\t" + str(rout) + + print >>fout, out_str + + prev_item = item + prev_vals = [] + for col in cols: + col = int(col)-1 + val_list = [] + val_list.append(fields[col].strip()) + prev_vals.append(val_list) + else: + """ + This only occurs once, right at the start of the iteration. + """ + prev_item = item + for col in cols: + col = int(col)-1 + val_list = [] + val_list.append(fields[col].strip()) + prev_vals.append(val_list) + + except Exception, exc: + print >> sys.stderr, "Error executing aggregation functions: %s" %exc + sys.exit() + else: + skipped_lines += 1 + if not first_invalid_line: + first_invalid_line = ii+1 + +""" +Handle the last grouped value +""" +out_str = prev_item + +for i, op in enumerate(ops): rfunc = "r." + op if op != 'c': - for j,elem in enumerate(prevvals[i]): - prevvals[i][j] = float(elem) - rout = "%.2f" %(eval(rfunc)(prevvals[i])) + for j, elem in enumerate( prev_vals[i] ): + prev_vals[i][j] = float( elem ) + rout = "%.2f" %( eval( rfunc )( prev_vals[i] )) else: - rout = eval(rfunc)(prevvals[i]) - outstr += "\t" + str(rout) -print >>fout, outstr + rout = eval( rfunc )( prev_vals[i] ) + + out_str += "\t" + str( rout ) -print "Group by column %d" %(groupcol+1) +print >>fout, out_str + +""" +Generate a useful info message. +""" +msg = "--Group by c%d: " %(group_col+1) +for i,op in enumerate(ops): + if op == 'c': + op = 'concat' + msg += op + "[c" + cols[i] + "] " +if skipped_lines > 0: + msg+= "--skipped %d blank/comment/invalid lines starting with line #%d. " %( skipped_lines, first_invalid_line ) + +print msg diff --git a/tools/stats/grouping.xml b/tools/stats/grouping.xml index d196ddb5f05..f3256917886 100644 --- a/tools/stats/grouping.xml +++ b/tools/stats/grouping.xml @@ -2,39 +2,49 @@ data by a column and perform aggregate operation on other columns. grouping.py - "$out_file1" - "$input" - "$groupcol" + $out_file1 + $input1 + $groupcol #for $op in $operations '${op.optype} ${op.opcol}' #end for - - - + + + + + + - - - - - - - - + + + + + + + + + - - - + + - + - + + + + + + + + @@ -49,6 +59,8 @@ This tool allows you to group the input dataset by a particular column and perform aggregate functions like Mean, Sum, Max, Min and Concatenate on other columns. +- All invalid, blank and comment lines are skipped when performing the aggregate functions. The number of skipped lines is displayed in the resulting history item. + ----- **Example** @@ -63,7 +75,8 @@ This tool allows you to group the input dataset by a particular column and perfo - running this tool with **Group by column 1**, Operations **Mean on column 2** and **Concatenate on column 3** will return:: - chr10 1700.00 ['NM_10', 'NM_11'] - chr22 1533.33 ['NM_17', 'NM_18', 'NM_19'] + chr10 1700.00 ['NM_11', 'NM_10'] + chr22 1533.33 ['NM_17', 'NM_19', 'NM_18'] +