Add ability for data_source tools to append parameters passed in the initial response to the value of URL prior to Galaxy's post to the URL. This is a cleaner method for Biomart and also gets GBrowse to wrok.

lib/galaxy/tools/__init__.py

@@ -231,8 +231,6 @@ class Tool:
         # data_source tool
         if self.tool_type == "data_source":
             self.URL_method = root.get( "URL_method", "get" ) # get is the default
-            # TODO: Biomart hack - eliminate when they encode URL - they'll let us know when...
-            self.add_to_URL = root.get( "add_to_URL", None )
             self.param_trans_dict = {}
             req_param_trans = root.find( "request_param_translation" )
             if req_param_trans is not None:
@@ -255,6 +253,22 @@ class Tool:
                                 galaxy_format = format.get( "galaxy_format" )                        
                                 format_trans_dict[ remote_format ] = galaxy_format
                             trans_list.append( format_trans_dict )
+                    elif req_param.get( "galaxy_name" ) == "URL":
+                        # Some remote data sources ( e.g., Gbrowse ) send parameters back to
+                        # Galaxy in the initial response that must be added to URL prior to
+                        # Galaxy sending the secondary request to the URL.  The tag set looks
+                        # asomething like:
+                        # <add_to_url>
+                        #    <param_from_source name="d" missing="" />
+                        # </add_to_url>
+                        add_to_url = req_param.find( "add_to_url" )
+                        if add_to_url is not None:
+                            add_to_url_dict = {}
+                            for param_from_source in add_to_url.findall( "param_from_source" ):
+                                name = param_from_source.get( "name" )
+                                value = param_from_source.get( "missing" ) # only used if the source doesn't send the param name                     
+                                add_to_url_dict[ name ] = value
+                            trans_list.append( add_to_url_dict )
                     self.param_trans_dict[ remote_name ] = trans_list
         # Command line (template). Optional for tools that do not invoke a local program  
         command = root.find("command")
@@ -1162,11 +1176,16 @@ class Tool:
                 description = param_dict.get( 'position', '' )
                 if not description:
                     description = 'unknown position'
+            gb_landmark_region = param_dict.get( 'q' )
             data_type = param_dict.get( 'data_type' )
             items = out_data.items()
             for name, data in items:
                 if organism and table and description:
+                    # This is UCSC
                     data.name  = '%s on %s: %s (%s)' % ( data.name, organism, table, description )
+                elif gb_landmark_region:
+                    # This is GBrowse
+                    data.name = '%s on %s' % ( data.name, gb_landmark_region )
                 data.info = info
                 data.dbkey = dbkey
                 try:

lib/galaxy/util/__init__.py

@@ -159,12 +159,33 @@ class Params:
                             try:
                                 # The Galaxy "data_type entry is special in that it can include the ability
                                 # to translate the format to a Galaxy supported format.  In the dict, this entry
-                                # looks something like:  {'hgta_outputType': ['data_type', 'bed', {'selectedFields': 'tabular'}] }
+                                # looks something like:  
+                                # {'hgta_outputType': ['data_type', 'bed', {'selectedFields': 'tabular'}] }
                                 format_trans_dict = tool.param_trans_dict[ key ][2]
                                 if value in format_trans_dict:
                                     new_value = format_trans_dict[ value ]
                             except:
                                 pass
+                        elif new_key == 'URL':
+                            # As above, the URL can include a set of params from the remote data source
+                            # that must be appended to the URL prior to the post.  In this case, the
+                            # dict entry would look something like:
+                            # ['URL', '', {'q': '', 's': '', 'd': '', 'dbkey': '', 't': ''}]
+                            try:
+                                add_to_url_dict = tool.param_trans_dict[ key ][2]
+                                if new_value.count( '?' ) == 0:
+                                    sep = '?'
+                                else:
+                                    sep = '&'
+                                for param_name, missing_value in add_to_url_dict.items():
+                                    param_value = params.get( param_name, None )
+                                    if not param_value and missing_value:
+                                        param_value = missing_value
+                                    if param_value:
+                                        new_value += '%s%s=%s' % ( sep, param_name, param_value )
+                                        sep = '&'
+                            except:
+                                pass
                         if not value and not new_value:
                             new_value = tool.param_trans_dict[ key ][1]
                 if key not in self.NEVER_SANITIZE and sanitize:
@@ -174,9 +195,6 @@ class Params:
             if tool and tool.tool_type == 'data_source':
                 # Add the tool's URL_method to params
                 self.__dict__[ 'URL_method' ] = tool.URL_method
-                # TODO: Biomart hack - eliminate when they encode URL - they'll let us know when...
-                if tool.add_to_URL is not None:
-                    self.__dict__[ 'add_to_URL' ] = tool.add_to_URL
                 for key, value in tool.param_trans_dict.items():
                     # Make sure that all translated values used in Galaxy are added to the params
                     galaxy_name = tool.param_trans_dict[ key ][0]

tools/data_source/biomart.xml

@@ -7,7 +7,7 @@
     TODO: Hack to get biomart to work - the 'add_to_URL' param can be eliminated when the Biomart team encodes URL prior to sending, meanwhile
     everything including and beyond the first '&' is truncated from URL.  They said they'll let us know when this is fixed at their end.
 -->
-<tool name="BioMart" id="biomart" tool_type="data_source" URL_method="get" add_to_URL="biomart_hack">
+<tool name="BioMart" id="biomart" tool_type="data_source" URL_method="get">
 	<description>Central server</description>
 	<command interpreter="python">data_source.py $output</command>
 	<inputs action="http://www.biomart.org/biomart/martview" check_values="false" method="get" target="_top">
@@ -15,7 +15,12 @@
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" />
     </inputs>
     <request_param_translation>
-        <request_param galaxy_name="URL" remote_name="URL" missing="" />
+        <request_param galaxy_name="URL" remote_name="URL" missing="">
+            <add_to_url>
+                <param_from_source name="_export" missing="1" />
+                <param_from_source name="GALAXY_URL" missing="0" />
+            </add_to_url>
+        </request_param>
         <request_param galaxy_name="dbkey" remote_name="dbkey" missing="?" />
         <request_param galaxy_name="organism" remote_name="organism" missing="" />
         <request_param galaxy_name="table" remote_name="table" missing="" />

tools/data_source/biomart_test.xml

@@ -7,7 +7,7 @@
     TODO: Hack to get biomart to work - the 'add_to_URL' param can be eliminated when the Biomart team encodes URL prior to sending, meanwhile
     everything including and beyond the first '&' is truncated from URL.  They said they'll let us know when this is fixed at their end.
 -->
-<tool name="BioMart" id="biomart_test" tool_type="data_source" URL_method="get" add_to_URL="biomart_hack">
+<tool name="BioMart" id="biomart_test" tool_type="data_source" URL_method="get">
 	<description>Test server</description>
 	<command interpreter="python">data_source.py $output</command>
 	<inputs action="http://test.biomart.org/biomart/martview" check_values="false" method="get" target="_top">
@@ -15,7 +15,12 @@
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner/biomart" />
 	</inputs>
     <request_param_translation>
-        <request_param galaxy_name="URL" remote_name="URL" missing="" />
+        <request_param galaxy_name="URL" remote_name="URL" missing="">
+            <add_to_url>
+                <param_from_source name="_export" missing="1" />
+                <param_from_source name="GALAXY_URL" missing="0" />
+            </add_to_url>
+        </request_param>
         <request_param galaxy_name="dbkey" remote_name="dbkey" missing="?" />
         <request_param galaxy_name="organism" remote_name="organism" missing="" />
         <request_param galaxy_name="table" remote_name="table" missing="" />

tools/data_source/data_source.py

@@ -33,12 +33,6 @@ def __main__():
     if not URL:
         open( filename, 'w' ).write( "" )
         stop_err( 'The remote data source application has not sent back a URL parameter in the request.' )
-    # TODO: Hack to get biomart to work - this can be eliminated when the Biomart team encodes URL prior to sending, meanwhile
-    # everything including and beyond the first '&' is truncated from URL.  They said they'll let us know when this is fixed
-    # at their end.
-    add_to_URL = params.get( 'add_to_URL', None )
-    if add_to_URL:
-        URL += '&_export=1&GALAXY_URL=0'
     URL_method = params.get( 'URL_method', None )
     out = open( filename, 'w' )
     CHUNK_SIZE = 2**20 # 1Mb 

tools/data_source/gbrowse_datasource.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-#Retreives data from GMOD and stores in a file. GBrowse parameters are provided in the input/output file.
-import urllib, sys, os, gzip, tempfile, shutil
-from galaxy import eggs
-from galaxy.datatypes import data
-
-assert sys.version_info[:2] >= ( 2, 4 )
-
-def stop_err( msg ):
-    sys.stderr.write( msg )
-    sys.exit()
-
-def __main__():
-    filename = sys.argv[1]
-    params = {}
-    
-    for line in open( filename, 'r' ):
-        try:
-            line = line.strip()
-            fields = line.split( '\t' )
-            params[ fields[0] ] = fields[1]
-        except:
-            continue
-    
-    URL = params.get( 'URL', None )
-    if not URL:
-        open( filename, 'w' ).write( "" )
-        stop_err( 'Datasource has not sent back a URL parameter.' )
-
-    for i, param in enumerate( params.keys() ):
-        if i == 0:
-            sep = '?'
-        else:
-            sep = '&'
-        if param != '__collected_datasets__':
-            URL += "%s%s=%s" % ( sep, param, params.get( param ) )
-
-    CHUNK_SIZE = 2**20 # 1Mb 
-    try:
-        page = urllib.urlopen( URL )
-    except Exception, exc:
-        raise Exception( 'Problems connecting to %s (%s)' % ( URL, exc ) )
-        sys.exit( 1 )
-    
-    fp = open( filename, 'wb' )
-    while 1:
-        chunk = page.read( CHUNK_SIZE )
-        if not chunk:
-            break
-        fp.write( chunk )
-    fp.close()      
-    
-if __name__ == "__main__": __main__()

tools/data_source/gbrowse_elegans.xml

@@ -1,13 +1,24 @@
 <?xml version="1.0"?>
-<tool name="C. Elegans" id="gbrowse_elegans">
+<tool name="C. Elegans" id="gbrowse_elegans" tool_type="data_source" URL_method="get">
 	<description>server</description>
-	<command interpreter="python">gbrowse_datasource.py $output</command>
-	<inputs action="http://www.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> 
+	<command interpreter="python">data_source.py $output</command>
+	<inputs action="http://dev.wormbase.org/db/seq/gbrowse/c_elegans/" check_values="false" method="get" target="_top"> 
 		<display>go to C. Elegans server $GALAXY_URL</display>
 		<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id=gbrowse_elegans" />
 	</inputs>
+    <request_param_translation>
+        <request_param galaxy_name="URL" remote_name="URL" missing="">
+            <add_to_url>
+                <param_from_source name="d" missing="" />
+                <param_from_source name="dbkey" missing="" />
+                <param_from_source name="q" missing="" />
+                <param_from_source name="s" missing="" />
+                <param_from_source name="t" missing="" />
+            </add_to_url>
+        </request_param>
+        <request_param galaxy_name="data_type" remote_name="data_type" missing="gff3" />
+    </request_param_translation>
 	<uihints minwidth="800"/>
-	<code file="gbrowse_filter_code.py"/>
 	<outputs>
 		<data name="output" format="txt" />
 	</outputs>

tools/data_source/gbrowse_filter_code.py

@@ -1,34 +0,0 @@
-# Code for direct connection to GMOD
-from galaxy.datatypes import sniff
-import urllib
-
-import logging
-log = logging.getLogger( __name__ )
-
-def exec_before_job( app, inp_data, out_data, param_dict, tool=None ):
-    """Sets the attributes of the data"""
-    gb_settings = urllib.unquote( param_dict.get( 't', None ) ) # t=CG+TS+ESTB+SAGE+EXPR+EXPR_PATTERN+SNPs+PolyA+BLASTX+LINK+ETILE
-    gb_landmark_region = urllib.unquote( param_dict.get( 'q' ) ) # q=IV:6070000..6100000&
-    gb_land_mark, gb_region = gb_landmark_region.split( ':' )
-    items = out_data.items()
-    for name, data in items:
-        data.name = "%s on %s" % ( data.name, gb_landmark_region )
-        data.dbkey = param_dict.get( 'dbkey', '?' )    
-        # Store GMOD / GBrowse parameters temporarily in output file
-        out = open( data.file_name, 'w' )
-        for key, value in param_dict.items():
-            out.write( "%s\t%s\n" % ( key, value ) )
-        out.close()
-        out_data[ name ] = data
-
-def exec_after_process( app, inp_data, out_data, param_dict, tool=None, stdout=None, stderr=None ):
-    """Verifies the data after the run"""
-    name, data = out_data.items()[0]
-    data.set_size()
-    if data.state == data.states.OK:
-        data.info = data.name
-    if data.extension == 'txt':
-        data_type = sniff.guess_ext( data.file_name, sniff_order=app.datatypes_registry.sniff_order )
-        data = app.datatypes_registry.change_datatype( data, data_type ) 
-    data.set_peek()
-    data.flush()