mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-01 15:37:32 +08:00
Fixes for the grouping tool.
This commit is contained in:
Greg Von Kuster
@@ -0,0 +1,6 @@
|
||||
chr22 1000 NM_17
|
||||
chr22 2000 NM_18
|
||||
chr10 2200 NM_10
|
||||
chr10 hap test
|
||||
chr10 1200 NM_11
|
||||
chr22 1600 NM_19
|
||||
+152
-85
@@ -6,114 +6,181 @@ This tool provides the SQL "group by" functionality.
|
||||
import sys, string, re, commands, tempfile
|
||||
from rpy import *
|
||||
|
||||
fout = open(sys.argv[1], "w")
|
||||
inputfile = sys.argv[2]
|
||||
|
||||
ops = []
|
||||
cols = []
|
||||
elems = []
|
||||
|
||||
for var in sys.argv[4:]:
|
||||
ops.append(var.split()[0])
|
||||
cols.append(var.split()[1])
|
||||
|
||||
if os.path.exists( inputfile ):
|
||||
for line in open( inputfile ):
|
||||
line = line.strip()
|
||||
if line and not line.startswith( '#' ):
|
||||
elems = line.split( '\t' )
|
||||
break
|
||||
else:
|
||||
print 'The data file you selected for filtering does not exist.'
|
||||
"""
|
||||
At this point, ops and cols will look something like this:
|
||||
ops: ['mean', 'min', 'c']
|
||||
cols: ['1', '3', '4']
|
||||
"""
|
||||
|
||||
for i, line in enumerate( file ( inputfile )):
|
||||
line = line.rstrip('\r\n')
|
||||
if len( line )>0 and not line.startswith( '#' ):
|
||||
elems = line.split( '\t' )
|
||||
break
|
||||
if i == 30:
|
||||
break # Hopefully we'll never get here...
|
||||
|
||||
if len( elems )<1:
|
||||
print >> sys.stderr, "The data in your input dataset is either missing or not formatted properly."
|
||||
sys.exit()
|
||||
|
||||
groupcol = string.atoi(sys.argv[3])
|
||||
if groupcol > len( elems ):
|
||||
print >> sys.stderr, "Column %d does not exist." %(groupcol)
|
||||
sys.exit()
|
||||
groupcol = groupcol-1
|
||||
|
||||
group_col = int( sys.argv[3] )-1
|
||||
|
||||
for k,col in enumerate(cols):
|
||||
col = int(col)
|
||||
if col > len( elems ):
|
||||
print >> sys.stderr, "Column %d does not exist." %(col)
|
||||
sys.exit()
|
||||
else:
|
||||
if ops[k] != "c":
|
||||
try:
|
||||
assert float(elems[col-1])
|
||||
except:
|
||||
print >> sys.stderr, "Operation '%s' cannot be performed on non-numeric column %d." %(ops[k],col)
|
||||
sys.exit()
|
||||
col = int(col)-1
|
||||
if ops[k] != 'c':
|
||||
"""
|
||||
We'll get here only if the user didn't choose 'Concatenate', which is the
|
||||
only aggregation function that can be used on columns containing strings.
|
||||
"""
|
||||
try:
|
||||
map( float, elems[col] )
|
||||
except:
|
||||
print >> sys.stderr, "Operation '%s' cannot be performed on non-numeric column %d containing value %s." %(ops[k], col+1, elems[col])
|
||||
sys.exit()
|
||||
|
||||
tmpfile = tempfile.NamedTemporaryFile()
|
||||
|
||||
tmpfile = tempfile.NamedTemporaryFile()
|
||||
try:
|
||||
commandline = "sort -f "+"+"+str(groupcol)+" -o "+tmpfile.name+" "+inputfile
|
||||
"""
|
||||
The -k option for the Posix sort command is as follows:
|
||||
-k, --key=POS1[,POS2]
|
||||
start a key at POS1, end it at POS2 (origin 1)
|
||||
In other words, column positions start at 1 rather than 0, so
|
||||
we need to add 1 to group_col.
|
||||
"""
|
||||
command_line = "sort -f -k " + str(group_col+1) + " -o " + tmpfile.name + " " + inputfile
|
||||
except Exception, exc:
|
||||
print >>sys.stdout, 'Initialization error -> %s' % exc
|
||||
print >> sys.stderr, 'Initialization error -> %s' % exc
|
||||
sys.exit()
|
||||
errorcode, stdout = commands.getstatusoutput(commandline)
|
||||
|
||||
previtem = ""
|
||||
prevvals = []
|
||||
error_code, stdout = commands.getstatusoutput(command_line)
|
||||
|
||||
if error_code != 0:
|
||||
print >> sys.stderr, "Sorting input dataset resulted in error: ", error_code, stdout
|
||||
sys.exit()
|
||||
|
||||
prev_item = ""
|
||||
prev_vals = []
|
||||
skipped_lines = 0
|
||||
first_invalid_line = 0
|
||||
invalid_line = None
|
||||
invalid_line = ''
|
||||
fout = open(sys.argv[1], "w")
|
||||
|
||||
for line in open(tmpfile.name):
|
||||
if line and (not line.startswith( '#' )) and line != '':
|
||||
for ii, line in enumerate( file( tmpfile.name )):
|
||||
if line and not line.startswith( '#' ):
|
||||
try:
|
||||
fields = line.split("\t")
|
||||
item = fields[groupcol]
|
||||
if previtem != "":
|
||||
if item == previtem: #Keep iterating and storing values till a new item is encountered.
|
||||
previtem = item
|
||||
for i,col in enumerate(cols):
|
||||
col = string.atoi(col)
|
||||
col = col-1
|
||||
prevvals[i].append(fields[col].strip())
|
||||
else: #When a new item is encountered, write the previous item and the corresponding aggregate values into the output file.
|
||||
outstr = previtem
|
||||
try:
|
||||
for i,op in enumerate(ops):
|
||||
rfunc = "r." + op
|
||||
if op != 'c':
|
||||
for j,elem in enumerate(prevvals[i]):
|
||||
prevvals[i][j] = float(elem)
|
||||
rout = "%.2f" %(eval(rfunc)(prevvals[i]))
|
||||
else:
|
||||
rout = eval(rfunc)(prevvals[i])
|
||||
outstr += "\t" + str(rout)
|
||||
print >>fout, outstr
|
||||
except:
|
||||
skipped_lines += 1
|
||||
previtem = item
|
||||
prevvals = []
|
||||
for col in cols:
|
||||
col = string.atoi(col)
|
||||
col = col-1
|
||||
vallist = []
|
||||
vallist.append(fields[col].strip())
|
||||
prevvals.append(vallist)
|
||||
else: #visited only once right at the start of the iteration.
|
||||
previtem = item
|
||||
for col in cols:
|
||||
col = string.atoi(col)
|
||||
col = col-1
|
||||
vallist = []
|
||||
vallist.append(fields[col].strip())
|
||||
prevvals.append(vallist)
|
||||
except:
|
||||
pass
|
||||
item = fields[group_col]
|
||||
if prev_item != "":
|
||||
"""
|
||||
At this level, we're grouping on values (item and prev_item) in group_col
|
||||
"""
|
||||
if item == prev_item:
|
||||
"""
|
||||
Keep iterating and storing values until a new value is encountered.
|
||||
"""
|
||||
for i, col in enumerate(cols):
|
||||
col = int(col)-1
|
||||
valid = True
|
||||
"""
|
||||
Before appending the current value, make sure it is numeric if the
|
||||
operation for the column requires it.
|
||||
"""
|
||||
if ops[i] != 'c':
|
||||
try:
|
||||
float( fields[col].strip())
|
||||
except:
|
||||
valid = False
|
||||
skipped_lines += 1
|
||||
if not first_invalid_line:
|
||||
first_invalid_line = ii+1
|
||||
if valid:
|
||||
prev_vals[i].append(fields[col].strip())
|
||||
else:
|
||||
"""
|
||||
When a new value is encountered, write the previous value and the
|
||||
corresponding aggregate values into the output file. This works
|
||||
due to the sort on group_col we've applied to the data above.
|
||||
"""
|
||||
out_str = prev_item
|
||||
|
||||
outstr = previtem
|
||||
for i,op in enumerate(ops):
|
||||
for i, op in enumerate( ops ):
|
||||
rfunc = "r." + op
|
||||
if op != 'c':
|
||||
for j, elem in enumerate( prev_vals[i] ):
|
||||
prev_vals[i][j] = float( elem )
|
||||
rout = "%.2f" %( eval( rfunc )( prev_vals[i] ))
|
||||
else:
|
||||
rout = eval( rfunc )( prev_vals[i] )
|
||||
|
||||
out_str += "\t" + str(rout)
|
||||
|
||||
print >>fout, out_str
|
||||
|
||||
prev_item = item
|
||||
prev_vals = []
|
||||
for col in cols:
|
||||
col = int(col)-1
|
||||
val_list = []
|
||||
val_list.append(fields[col].strip())
|
||||
prev_vals.append(val_list)
|
||||
else:
|
||||
"""
|
||||
This only occurs once, right at the start of the iteration.
|
||||
"""
|
||||
prev_item = item
|
||||
for col in cols:
|
||||
col = int(col)-1
|
||||
val_list = []
|
||||
val_list.append(fields[col].strip())
|
||||
prev_vals.append(val_list)
|
||||
|
||||
except Exception, exc:
|
||||
print >> sys.stderr, "Error executing aggregation functions: %s" %exc
|
||||
sys.exit()
|
||||
else:
|
||||
skipped_lines += 1
|
||||
if not first_invalid_line:
|
||||
first_invalid_line = ii+1
|
||||
|
||||
"""
|
||||
Handle the last grouped value
|
||||
"""
|
||||
out_str = prev_item
|
||||
|
||||
for i, op in enumerate(ops):
|
||||
rfunc = "r." + op
|
||||
if op != 'c':
|
||||
for j,elem in enumerate(prevvals[i]):
|
||||
prevvals[i][j] = float(elem)
|
||||
rout = "%.2f" %(eval(rfunc)(prevvals[i]))
|
||||
for j, elem in enumerate( prev_vals[i] ):
|
||||
prev_vals[i][j] = float( elem )
|
||||
rout = "%.2f" %( eval( rfunc )( prev_vals[i] ))
|
||||
else:
|
||||
rout = eval(rfunc)(prevvals[i])
|
||||
outstr += "\t" + str(rout)
|
||||
print >>fout, outstr
|
||||
rout = eval( rfunc )( prev_vals[i] )
|
||||
|
||||
out_str += "\t" + str( rout )
|
||||
|
||||
print "Group by column %d" %(groupcol+1)
|
||||
print >>fout, out_str
|
||||
|
||||
"""
|
||||
Generate a useful info message.
|
||||
"""
|
||||
msg = "--Group by c%d: " %(group_col+1)
|
||||
for i,op in enumerate(ops):
|
||||
if op == 'c':
|
||||
op = 'concat'
|
||||
msg += op + "[c" + cols[i] + "] "
|
||||
if skipped_lines > 0:
|
||||
msg+= "--skipped %d blank/comment/invalid lines starting with line #%d. " %( skipped_lines, first_invalid_line )
|
||||
|
||||
print msg
|
||||
|
||||
+34
-21
@@ -2,39 +2,49 @@
|
||||
<description>data by a column and perform aggregate operation on other columns.</description>
|
||||
<command interpreter="python2.4">
|
||||
grouping.py
|
||||
"$out_file1"
|
||||
"$input"
|
||||
"$groupcol"
|
||||
$out_file1
|
||||
$input1
|
||||
$groupcol
|
||||
#for $op in $operations
|
||||
'${op.optype}
|
||||
${op.opcol}'
|
||||
#end for
|
||||
</command>
|
||||
<inputs>
|
||||
<param format="tabular, interval" name="input" type="data" label="Select data" help="Query missing? See TIP below."/>
|
||||
<param name="groupcol" size="40" type="integer" value="1" label="Group by column" />
|
||||
<repeat name="operations" title="Operation">
|
||||
<page>
|
||||
<param format="tabular" name="input1" type="data" label="Select data" help="Query missing? See TIP below."/>
|
||||
</page>
|
||||
<page>
|
||||
<param name="groupcol" label="Group by column" type="select" dynamic_options="get_columns( input1 )" />
|
||||
<repeat name="operations" title="Operation">
|
||||
<param name="optype" type="select" label="Type">
|
||||
<option value="mean">Mean</option>
|
||||
<option value="max">Maximum</option>
|
||||
<option value="min">Minimum</option>
|
||||
<option value="sum">Sum</option>
|
||||
<option value="c">Concatenate</option>
|
||||
</param>
|
||||
<param name="opcol" size="40" type="integer" value="2" label="On column" />
|
||||
</repeat>
|
||||
<option value="mean">Mean</option>
|
||||
<option value="max">Maximum</option>
|
||||
<option value="min">Minimum</option>
|
||||
<option value="sum">Sum</option>
|
||||
<option value="c">Concatenate</option>
|
||||
</param>
|
||||
<param name="opcol" label="On column" type="select" dynamic_options="get_columns( input1 )" />
|
||||
</repeat>
|
||||
</page>
|
||||
</inputs>
|
||||
|
||||
|
||||
<outputs>
|
||||
<data format="input" name="out_file1"/>
|
||||
<data format="input" name="out_file1" metadata_source="input1" />
|
||||
</outputs>
|
||||
<tests>
|
||||
<!-- Test valid data -->
|
||||
<test>
|
||||
<param name="input" value="1.bed"/>
|
||||
<param name="input1" value="1.bed"/>
|
||||
<param name="groupcol" value="1"/>
|
||||
<param name="operations" value="mean 2"/>
|
||||
<output name="out_file1" file="groupby_1.bed"/>
|
||||
<output name="out_file1" file="groupby_out1.dat"/>
|
||||
</test>
|
||||
<!-- Test data with an invalid value in a column -->
|
||||
<test>
|
||||
<param name="input1" value="1.tabular"/>
|
||||
<param name="groupcol" value="1"/>
|
||||
<param name="operations" value="mean 2,c 3"/>
|
||||
<output name="out_file1" file="groupby_out2.dat"/>
|
||||
</test>
|
||||
</tests>
|
||||
<help>
|
||||
@@ -49,6 +59,8 @@
|
||||
|
||||
This tool allows you to group the input dataset by a particular column and perform aggregate functions like Mean, Sum, Max, Min and Concatenate on other columns.
|
||||
|
||||
- All invalid, blank and comment lines are skipped when performing the aggregate functions. The number of skipped lines is displayed in the resulting history item.
|
||||
|
||||
-----
|
||||
|
||||
**Example**
|
||||
@@ -63,7 +75,8 @@ This tool allows you to group the input dataset by a particular column and perfo
|
||||
|
||||
- running this tool with **Group by column 1**, Operations **Mean on column 2** and **Concatenate on column 3** will return::
|
||||
|
||||
chr10 1700.00 ['NM_10', 'NM_11']
|
||||
chr22 1533.33 ['NM_17', 'NM_18', 'NM_19']
|
||||
chr10 1700.00 ['NM_11', 'NM_10']
|
||||
chr22 1533.33 ['NM_17', 'NM_19', 'NM_18']
|
||||
</help>
|
||||
<code file="grouping_code.py" />
|
||||
</tool>
|
||||
|
||||
Reference in New Issue
Block a user