opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Enhanced column join to allow user to specify fill values for empty columns, either one value for all or by column

Browse Source
This commit is contained in:
Kelly Vincent
2010-05-05 11:34:29 -04:00
parent 07f96e096d
commit 1c56e2e768
2 changed files with 167 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/new_operations/column_join.py
+95 -25
View File
@@ -13,7 +13,19 @@ usage: %prog output input1 input2 column1[,column2[,column3[,...]]] hinge1[,hing
"""
import os, re, sys, tempfile
import optparse, os, re, struct, sys, tempfile
try:
simple_json_exception = None
from galaxy import eggs
from galaxy.util.bunch import Bunch
from galaxy.util import stringify_dictionary_keys
import pkg_resources
pkg_resources.require("simplejson")
import simplejson
except Exception, e:
simplejson_exception = e
simplejson = None
def stop_err( msg ):
sys.stderr.write( msg )
@@ -136,17 +148,50 @@ def min_chr_pos( chr_pos ):
return '%s\t%s' % tuple( min_ref_pos ), min_loc
def __main__():
output = sys.argv[1]
input1 = sys.argv[2]
input2 = sys.argv[3]
hinge = int( sys.argv[4] )
cols = [ int( c ) for c in sys.argv[5].split( ',' ) ]
inputs = sys.argv[6:]
assert len( cols ) > 2, 'You need to select at least one column in addition to the first two'
parser = optparse.OptionParser()
parser.add_option( '', '--output', dest='output', help='' )
parser.add_option( '', '--input1', dest='input1', help='' )
parser.add_option( '', '--input2', dest='input2', help='' )
parser.add_option( '', '--hinge', dest='hinge', help='' )
parser.add_option( '', '--columns', dest='columns', help='' )
parser.add_option( '', '--fill_options_file', dest='fill_options_file', default=None, help='' )
(options, args) = parser.parse_args()
output = options.output
input1 = options.input1
input2 = options.input2
hinge = int( options.hinge )
cols = [ int( c ) for c in str( options.columns ).split( ',' ) if int( c ) > hinge ]
inputs = [ input1, input2 ]
if options.fill_options_file == "None":
inputs.extend( args )
else:
try:
col = int( args[0] )
except ValueError:
inputs.extend( args )
fill_options = None
if options.fill_options_file != "None" and options.fill_options_file is not None:
try:
if simplejson is None:
raise simplejson_exception
fill_options = Bunch( **stringify_dictionary_keys( simplejson.load( open( options.fill_options_file ) ) ) )
except Exception, e:
print "Warning: Ignoring fill options due to simplejson error (%s)." % e
if fill_options is None:
fill_options = Bunch()
if 'file1_columns' not in fill_options:
fill_options.file1_columns = None
if fill_options and fill_options.file1_columns:
fill_empty = {}
for col in cols:
fill_empty[ col ] = fill_options.file1_columns[ col - 1 ]
else:
fill_empty = None
assert len( cols ) > 0, 'You need to select at least one column in addition to the hinge'
delimiter = '\t'
# make sure all files are sorted in same way, ascending
tmp_input_files = []
input_files = [ input1, input2 ]
input_files.extend( inputs )
input_files = inputs[:]
for in_file in input_files:
tmp_file = tempfile.NamedTemporaryFile()
tmp_file_name = tmp_file.name
@@ -162,10 +207,9 @@ def __main__():
current_lines = [ f.readline() for f in tmp_input_files ]
last_lines = ''.join( current_lines ).strip()
last_loc = -1
i = 0
while last_lines:
# get the "minimum" hinge, which should come first, and the file location in list
hinges = [ '\t'.join( line.split( '\t' )[ :hinge ] ) for line in current_lines ]
hinges = [ delimiter.join( line.split( delimiter )[ :hinge ] ) for line in current_lines ]
hinge_dict = {}
for i in range( len( hinges ) ):
if not hinge_dict.has_key( hinges[ i ] ):
@@ -174,33 +218,59 @@ def __main__():
hinges = [ h for h in hinges if h ]
current, loc = hinges[0], hinge_dict[ hinges[0] ]
# first output empty columns for vertical alignment (account for "missing" files)
# write output if trailing empty columns
current_data = []
if current != old_current:
last_loc = -1
if loc - last_loc > 1:
current_data = [ '' for col in range( ( loc - last_loc - 1 ) * len( [ col for col in cols if col > hinge ] ) ) ]
# fill trailing empty columns with appropriate fill value
if not first_line:
if last_loc < len( inputs ) - 1:
if not fill_empty:
filler = [ '' for col in range( ( len( inputs ) - last_loc - 1 ) * len( cols ) ) ]
else:
filler = [ fill_empty[ cols[ col % len( cols ) ] ] for col in range( ( len( inputs ) - last_loc - 1 ) * len( cols ) ) ]
fout.write( '%s%s' % ( delimiter, delimiter.join( filler ) ) )
# insert line break before current line
fout.write( '\n' )
# fill leading empty columns with appropriate fill value
if loc > 0:
if not fill_empty:
current_data = [ '' for col in range( loc * len( cols ) ) ]
else:
current_data = [ fill_empty[ cols[ col % len( cols ) ] ] for col in range( loc * len( cols ) ) ]
else:
current_data = []
if loc - last_loc > 1:
if not fill_empty:
current_data = [ '' for col in range( ( loc - last_loc - 1 ) * len( cols ) ) ]
else:
current_data = [ fill_empty[ cols[ col % len( cols ) ] ] for col in range( ( loc - last_loc - 1 ) * len( cols ) ) ]
# now output actual data
split_line = current_lines[ loc ].strip().split( '\t' )
split_line = current_lines[ loc ].strip().split( delimiter )
if ''.join( split_line ):
# add actual data to be output below
for col in cols:
if col > hinge:
current_data.append( split_line[ col - 1 ] )
# grab next line for selected file
current_lines[ loc ] = tmp_input_files[ loc ].readline()
# write relevant data to file
if current == old_current:
if current_data:
fout.write( '\t%s' % '\t'.join( current_data ) )
fout.write( '%s%s' % ( delimiter, delimiter.join( current_data ) ) )
else:
if not first_line:
fout.write( '\n' )
fout.write( '%s\t%s' % ( current, '\t'.join( current_data ) ) )
first_line = False
fout.write( '%s%s%s' % ( current, delimiter, delimiter.join( current_data ) ) )
last_loc = loc
old_current = current
last_lines = ''.join( current_lines ).strip()
last_loc = loc
first_line = False
# fill trailing empty columns for final line
if last_loc < len( inputs ) - 1:
if not fill_empty:
filler = [ '' for col in range( ( len( inputs ) - last_loc - 1 ) * len( cols ) ) ]
else:
filler = [ fill_empty[ cols[ col % len( cols ) ] ] for col in range( ( len( inputs ) - last_loc - 1 ) * len( cols ) ) ]
fout.write( '%s%s' % ( delimiter, delimiter.join( filler ) ) )
fout.write( '\n' )
fout.close()
for f in tmp_input_files:
os.unlink( f.name )
fout.close()
if __name__ == "__main__" : __main__()
tools/new_operations/column_join.xml
+72 -14
View File
@@ -2,53 +2,111 @@
<description></description>
<command interpreter="python">
column_join.py
$output
$input1
$input2
$hinge
$columns
--output=$output
--input1=$input1
--input2=$input2
--hinge=$hinge
--columns=$columns
#if $fill_empty_columns.fill_empty_columns_switch == "fill_empty":
--fill_options_file=$fill_options_file
#end if
#for $f in $file_chooser:
${f.input}
#end for
</command>
<inputs>
<param name="input1" type="data" format="tabular" label="Choose the first file for the join" />
<param name="columns" type="data_column" data_ref="input1" multiple="true" numerical="false" label="Include these column" help="Multi-select list - hold the appropriate key while clicking to select multiple columns" />
<param name="hinge" type="data_column" data_ref="input1" multiple="false" numerical="false" label="Use this column and columns to left the 'hinge' (matching data for each join)" help="All columns to left of selected column (plus selected column) will be used. Select 2 for pileup" />
<param name="columns" type="data_column" data_ref="input1" multiple="true" numerical="false" label="Include these column" help="Multi-select list - hold the appropriate key while clicking to select multiple columns" />
<conditional name="fill_empty_columns">
<param name="fill_empty_columns_switch" type="select" label="Fill empty columns">
<option value="no_fill" selected="True">No</option>
<option value="fill_empty">Yes</option>
</param>
<when value="no_fill" />
<when value="fill_empty">
<conditional name="do_fill_empty_columns">
<param name="column_fill_type" type="select" label="Fill Columns by">
<option value="single_fill_value" selected="True">Single fill value</option>
<option value="fill_value_by_column">Values by column</option>
</param>
<when value="single_fill_value">
<param type="text" name="fill_value" label="Fill value" value="." />
</when>
<when value="fill_value_by_column">
<repeat name="column_fill" title="Fill Column">
<param name="column_number" label="Column" type="data_column" data_ref="input1" />
<param type="text" name="fill_value" value="." />
</repeat>
</when>
</conditional>
</when>
</conditional>
<param name="input2" type="data" format="tabular" label="Choose the second file for the join" />
<repeat name="file_chooser" title="Additional Input">
<param name="input" label="Additional input file" type="data" format="tabular" />
</repeat>
</inputs>
<configfiles>
<configfile name="fill_options_file">&lt;%
import simplejson
%&gt;
#set $__fill_options = {}
#if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty':
#if $fill_empty_columns['do_fill_empty_columns']['column_fill_type'] == 'single_fill_value':
#set $__start_fill = $fill_empty_columns['do_fill_empty_columns']['fill_value'].value
#else:
#set $__start_fill = ""
#end if
#set $__fill_options['file1_columns'] = [ __start_fill for i in range( int( $input1.metadata.columns ) ) ]
#if $fill_empty_columns['do_fill_empty_columns']['column_fill_type'] == 'fill_value_by_column':
#for column_fill in $fill_empty_columns['do_fill_empty_columns']['column_fill']:
#set $__fill_options['file1_columns'][ int( column_fill['column_number'].value ) - 1 ] = column_fill['fill_value'].value
#end for
#end if
#end if
${simplejson.dumps( __fill_options )}
</configfile>
</configfiles>
<outputs>
<data name="output" format="tabular" />
</outputs>
<tests>
<test>
<param name="input1" value="column_join_in1.pileup" ftype="pileup" />
<param name="hinge" value="2" />
<param name="columns" value="1,2,3,4,5,7" />
<param name="hinge" value="1,2" />
<param name="fill_empty_columns_switch" value="fill_empty" />
<param name="column_fill_type" value="single_fill_value" />
<param name="fill_value" value="?" />
<param name="input2" value="column_join_in2.pileup" ftype="pileup" />
<param name="input" value="column_join_in3.pileup" ftype="pileup" />
<output name="output" file="column_join_out1.pileup" ftype="tabular" />
</test>
<test>
<param name="input1" value="column_join_in4.pileup" ftype="pileup" />
<param name="hinge" value="2" />
<param name="columns" value="1,2,3,4" />
<param name="hinge" value="1,2" />
<param name="fill_empty_columns_switch" value="no_fill" />
<param name="input2" value="column_join_in5.pileup" ftype="pileup" />
<param name="input" value="column_join_in6.pileup" ftype="pileup" />
<output name="output" file="column_join_out2.pileup" ftype="tabular" />
</test>
<!-- This test is failing for an unclear reason (the column values do not get
passed into the script), but passes in the browser
<test>
<param name="input1" value="column_join_in7.pileup" ftype="pileup" />
<param name="columns" value="1,2,3,4,5" />
<param name="hinge" value="1,2" />
<param name="input2" value="column_join_in8.pileup" ftype="pileup" />
<param name="input" value="column_join_in9.pileup" ftype="pileup" />
<param name="input1" value="column_join_in7.pileup" ftype="tabular" />
<param name="hinge" value="2" />
<param name="columns" value="3,4,5" />
<param name="fill_empty_columns_switch" value="fill_empty" />
<param name="column_fill_type" value="fill_value_by_column" />
<param name="column_number" value="5" />
<param name="fill_value" value="X" />
<param name="input2" value="column_join_in8.pileup" ftype="tabular" />
<param name="input" value="column_join_in9.pileup" ftype="tabular" />
<output name="output" file="column_join_out3.pileup" ftype="tabular" />
</test>
</tests>
--> </tests>
<help>
**What it does**