mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Add the ability for the tabular join tool to fill in empty columns.
Working tests are needed.
This commit is contained in:
Daniel Blankenberg
+69
-6
@@ -8,9 +8,22 @@ User can also opt to have have non-joining rows of file1 echoed.
|
||||
|
||||
"""
|
||||
|
||||
import optparse, os, sys, tempfile, struct
|
||||
import optparse, os, sys, tempfile, struct
|
||||
import psyco_full
|
||||
|
||||
try:
|
||||
simple_json_exception = None
|
||||
from galaxy import eggs
|
||||
from galaxy.util.bunch import Bunch
|
||||
from galaxy.util import stringify_dictionary_keys
|
||||
import pkg_resources
|
||||
pkg_resources.require("simplejson")
|
||||
import simplejson
|
||||
except Exception, e:
|
||||
simplejson_exception = e
|
||||
simplejson = None
|
||||
|
||||
|
||||
class OffsetList:
|
||||
def __init__( self, filesize = 0, fmt = None ):
|
||||
self.file = tempfile.NamedTemporaryFile( 'w+b' )
|
||||
@@ -235,7 +248,22 @@ class BufferedIndex:
|
||||
for offset in self.buffered_offsets[identifier]:
|
||||
yield self.index.get_line_by_offset( offset )
|
||||
|
||||
def join_files( filename1, column1, filename2, column2, out_filename, split = None, buffer = 1000000, keep_unmatched = False, keep_partial = False, index_depth = 3 ):
|
||||
|
||||
def fill_empty_columns( line, split, fill_values ):
|
||||
if not fill_values:
|
||||
return line
|
||||
filled_columns = []
|
||||
for i, field in enumerate( line.split( split ) ):
|
||||
if field or i >= len( fill_values ):
|
||||
filled_columns.append( field )
|
||||
else:
|
||||
filled_columns.append( fill_values[i] )
|
||||
if len( fill_values ) > len( filled_columns ):
|
||||
filled_columns.extend( fill_values[ len( filled_columns ) : ] )
|
||||
return split.join( filled_columns )
|
||||
|
||||
|
||||
def join_files( filename1, column1, filename2, column2, out_filename, split = None, buffer = 1000000, keep_unmatched = False, keep_partial = False, index_depth = 3, fill_options = None ):
|
||||
#return identifier based upon line
|
||||
def get_identifier_by_line( line, column, split = None ):
|
||||
if isinstance( line, str ):
|
||||
@@ -243,6 +271,8 @@ def join_files( filename1, column1, filename2, column2, out_filename, split = No
|
||||
if column < len( fields ):
|
||||
return fields[column]
|
||||
return None
|
||||
if fill_options is None:
|
||||
fill_options = Bunch( fill_unjoined_only = True, file1_columns = None, file2_columns = None )
|
||||
out = open( out_filename, 'w+b' )
|
||||
index = BufferedIndex( filename2, column2, split, buffer, index_depth )
|
||||
for line1 in open( filename1, 'rb' ):
|
||||
@@ -250,12 +280,21 @@ def join_files( filename1, column1, filename2, column2, out_filename, split = No
|
||||
if identifier:
|
||||
written = False
|
||||
for line2 in index.get_lines_by_identifier( identifier ):
|
||||
out.write( "%s%s%s\n" % ( line1.rstrip( '\r\n' ), split, line2.rstrip( '\r\n' ) ) )
|
||||
if not fill_options.fill_unjoined_only:
|
||||
out.write( "%s%s%s\n" % ( fill_empty_columns( line1.rstrip( '\r\n' ), split, fill_options.file1_columns ), split, fill_empty_columns( line2.rstrip( '\r\n' ), split, fill_options.file2_columns ) ) )
|
||||
else:
|
||||
out.write( "%s%s%s\n" % ( line1.rstrip( '\r\n' ), split, line2.rstrip( '\r\n' ) ) )
|
||||
written = True
|
||||
if not written and keep_unmatched:
|
||||
out.write( "%s\n" % ( line1.rstrip( '\r\n' ) ) )
|
||||
out.write( fill_empty_columns( line1.rstrip( '\r\n' ), split, fill_options.file1_columns ) )
|
||||
if fill_options:
|
||||
out.write( fill_empty_columns( "", split, fill_options.file2_columns ) )
|
||||
out.write( "\n" )
|
||||
elif keep_partial:
|
||||
out.write( "%s\n" % ( line1.rstrip( '\r\n' ) ) )
|
||||
out.write( fill_empty_columns( line1.rstrip( '\r\n' ), split, fill_options.file1_columns ) )
|
||||
if fill_options:
|
||||
out.write( fill_empty_columns( "", split, fill_options.file2_columns ) )
|
||||
out.write( "\n" )
|
||||
out.close()
|
||||
|
||||
def main():
|
||||
@@ -284,9 +323,33 @@ def main():
|
||||
dest='keep_unmatched',
|
||||
default=False,
|
||||
help='Keep rows in first input which are not joined with the second input.')
|
||||
parser.add_option(
|
||||
'-f','--fill_options_file',
|
||||
dest='fill_options_file',
|
||||
type='str',default=None,
|
||||
help='Fill empty columns with a values from a JSONified file.')
|
||||
|
||||
|
||||
options, args = parser.parse_args()
|
||||
|
||||
fill_options = None
|
||||
if options.fill_options_file is not None:
|
||||
try:
|
||||
if simplejson is None:
|
||||
raise simplejson_exception
|
||||
fill_options = Bunch( **stringify_dictionary_keys( simplejson.load( open( options.fill_options_file ) ) ) ) #simplejson.load( open( options.fill_options_file ) )
|
||||
except Exception, e:
|
||||
print "Warning: Ignoring fill options due to simplejson error (%s)." % e
|
||||
if fill_options is None:
|
||||
fill_options = Bunch()
|
||||
if 'fill_unjoined_only' not in fill_options:
|
||||
fill_options.fill_unjoined_only = True
|
||||
if 'file1_columns' not in fill_options:
|
||||
fill_options.file1_columns = None
|
||||
if 'file2_columns' not in fill_options:
|
||||
fill_options.file2_columns = None
|
||||
|
||||
|
||||
try:
|
||||
filename1 = args[0]
|
||||
filename2 = args[1]
|
||||
@@ -300,6 +363,6 @@ def main():
|
||||
#Character for splitting fields and joining lines
|
||||
split = "\t"
|
||||
|
||||
return join_files( filename1, column1, filename2, column2, out_filename, split, options.buffer, options.keep_unmatched, options.keep_partial, options.index_depth )
|
||||
return join_files( filename1, column1, filename2, column2, out_filename, split, options.buffer, options.keep_unmatched, options.keep_partial, options.index_depth, fill_options = fill_options )
|
||||
|
||||
if __name__ == "__main__": main()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<tool id="join1" name="Join two Queries" version="2.0.0">
|
||||
<tool id="join1" name="Join two Queries" version="2.0.1">
|
||||
<description>side by side on a specified field</description>
|
||||
<command interpreter="python">join.py $input1 $input2 $field1 $field2 $out_file1 $unmatched $partial --index_depth=3 --buffer=50000000</command>
|
||||
<command interpreter="python">join.py $input1 $input2 $field1 $field2 $out_file1 $unmatched $partial --index_depth=3 --buffer=50000000 --fill_options_file=$fill_options_file</command>
|
||||
<inputs>
|
||||
<param format="tabular" name="input1" type="data" label="Join"/>
|
||||
<param name="field1" label="using column" type="data_column" data_ref="input1" />
|
||||
@@ -14,7 +14,66 @@
|
||||
<option value="-p">Yes</option>
|
||||
<option value="" selected="true">No</option>
|
||||
</param>
|
||||
<conditional name="fill_empty_columns">
|
||||
<param name="fill_empty_columns_switch" type="select" label="Fill empty columns">
|
||||
<option value="no_fill" selected="True">No</option>
|
||||
<option value="fill_empty">Yes</option>
|
||||
</param>
|
||||
<when value="no_fill">
|
||||
<!-- do nothing -->
|
||||
</when>
|
||||
<when value="fill_empty">
|
||||
<param type="select" name="fill_columns_by" label="Only fill unjoined rows">
|
||||
<option value="fill_unjoined_only" selected="True">Yes</option>
|
||||
<option value="fill_all">No</option>
|
||||
</param>
|
||||
<conditional name="do_fill_empty_columns">
|
||||
<param name="column_fill_type" type="select" label="Fill Columns by">
|
||||
<option value="single_fill_value" selected="True">Single fill value</option>
|
||||
<option value="fill_value_by_column">Values by column</option>
|
||||
</param>
|
||||
<when value="single_fill_value">
|
||||
<param type="text" name="fill_value" label="Fill value" value="."/>
|
||||
</when>
|
||||
<when value="fill_value_by_column">
|
||||
<repeat name="column_fill1" title="Fill Column for Input 1">
|
||||
<param name="column_number1" label="Column" type="data_column" data_ref="input1" />
|
||||
<param type="text" name="fill_value1" value="."/>
|
||||
</repeat>
|
||||
<repeat name="column_fill2" title="Fill Column for Input 2">
|
||||
<param name="column_number2" label="Column" type="data_column" data_ref="input2" />
|
||||
<param type="text" name="fill_value2" value="."/>
|
||||
</repeat>
|
||||
</when>
|
||||
</conditional>
|
||||
</when>
|
||||
</conditional>
|
||||
</inputs>
|
||||
<configfiles>
|
||||
<configfile name="fill_options_file"><%
|
||||
import simplejson
|
||||
%>#set $__fill_options = {}
|
||||
#if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty':
|
||||
#set $__fill_options['fill_unjoined_only'] = $fill_empty_columns['fill_columns_by'].value == 'fill_unjoined_only'
|
||||
#if $fill_empty_columns['do_fill_empty_columns']['column_fill_type'] == 'single_fill_value':
|
||||
#set $__start_fill = $fill_empty_columns['do_fill_empty_columns']['fill_value'].value
|
||||
#else:
|
||||
#set $__start_fill = ""
|
||||
#end if
|
||||
#set $__fill_options['file1_columns'] = [ $__start_fill for i in range( int( $input1.metadata.columns ) ) ]
|
||||
#set $__fill_options['file2_columns'] = [ $__start_fill for i in range( int( $input2.metadata.columns ) ) ]
|
||||
#if $fill_empty_columns['do_fill_empty_columns']['column_fill_type'] == 'fill_value_by_column':
|
||||
#for column_fill1 in $fill_empty_columns['do_fill_empty_columns']['column_fill1']:
|
||||
#set $__fill_options['file1_columns'][ int( column_fill1['column_number1'].value ) - 1 ] = column_fill1['fill_value1'].value
|
||||
#end for
|
||||
#for column_fill2 in $fill_empty_columns['do_fill_empty_columns']['column_fill2']:
|
||||
#set $__fill_options['file2_columns'][ int( column_fill2['column_number2'].value ) - 1 ] = column_fill2['fill_value2'].value
|
||||
#end for
|
||||
#end if
|
||||
#end if
|
||||
${simplejson.dumps( __fill_options )}
|
||||
</configfile>
|
||||
</configfiles>
|
||||
<outputs>
|
||||
<data format="input" name="out_file1" metadata_source="input1" />
|
||||
</outputs>
|
||||
@@ -26,6 +85,7 @@
|
||||
<param name="field2" value="2"/>
|
||||
<param name="unmatched" value=""/>
|
||||
<param name="partial" value=""/>
|
||||
<param name="fill_empty_columns_switch" value="no_fill"/>
|
||||
<output name="out_file1" file="joiner_out1.bed"/>
|
||||
</test>
|
||||
<test>
|
||||
@@ -35,8 +95,22 @@
|
||||
<param name="field2" value="2"/>
|
||||
<param name="unmatched" value="Yes"/>
|
||||
<param name="partial" value="Yes"/>
|
||||
<param name="fill_empty_columns_switch" value="no_fill"/>
|
||||
<output name="out_file1" file="joiner_out2.bed"/>
|
||||
</test>
|
||||
<!--this throws an exception in the test framework - we need tests for the fill empty columns enhancements <test>
|
||||
<param name="input1" value="1.bed"/>
|
||||
<param name="input2" value="2.bed"/>
|
||||
<param name="field1" value="2"/>
|
||||
<param name="field2" value="2"/>
|
||||
<param name="unmatched" value="Yes"/>
|
||||
<param name="partial" value="Yes"/>
|
||||
<param name="fill_empty_columns_switch" value="fill_empty"/>
|
||||
<param name="fill_columns_by" value="fill_all"/>
|
||||
<param name="column_fill_type" value="single_fill_value"/>
|
||||
<param name="fill_value" value="~"/>
|
||||
<output name="out_file1" file="joiner_out3.bed"/>
|
||||
</test>-->
|
||||
</tests>
|
||||
<help>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user