Merge next-stable.

lib/galaxy/datatypes/display_applications/__init__.py

@@ -1 +1,3 @@
-
+"""
+Contains functionality of the newer XML defined external display applications (not hardcoded into datatype classes).
+"""

lib/galaxy/datatypes/display_applications/link_generator.py

@@ -1,8 +1,11 @@
-"""Classes to generate links for display applications.
+"""Classes to generate links for old-style display applications.
 
 Separating Transaction based elements of display applications from datatypes.
 """
 
+#FIXME: The code contained within this file is for old-style display applications, but
+#this module namespace is intended to only handle the new-style display applications.
+
 import urllib
 
 # for the url_for hack
@@ -19,6 +22,8 @@ from galaxy.datatypes.interval import Interval, Gff, Wiggle, CustomTrack
 
 #TODO: these could be extended to handle file_function and parse/contain the builds.txt files
 
+#HACK: these duplicate functionality from the individual datatype classes themselves
+
 def get_display_app_link_generator( display_app_name ):
     """Returns an instance of the proper link generator class
     based on the display_app_name or DisplayAppLinkGenerator

lib/galaxy/model/__init__.py

@@ -3040,9 +3040,9 @@ class APIKeys( object ):
     pass
 
 class ToolShedRepository( object ):
-    api_collection_visible_keys = ( 'id', 'name', 'tool_shed', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes',
+    api_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes',
                                     'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' )
-    api_element_visible_keys = ( 'id', 'name', 'tool_shed', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes',
+    api_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes',
                                     'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' )
     installation_status = Bunch( NEW='New',
                                  CLONING='Cloning',
@@ -3079,10 +3079,8 @@ class ToolShedRepository( object ):
         self.dist_to_shed = dist_to_shed
         self.status = status
         self.error_message = error_message
-    def as_dict( self, trans ):
-        tsr_dict = self.get_api_value( view='element' )
-        tsr_dict[ 'id' ] = trans.security.encode_id( self.id )    
-        return tsr_dict
+    def as_dict( self, value_mapper=None ):
+        return self.get_api_value( view='element', value_mapper=value_mapper )
     def repo_files_directory( self, app ):
         repo_path = self.repo_path( app )
         if repo_path:
@@ -3182,7 +3180,7 @@ class ToolShedRepository( object ):
             try:
                 rval[ key ] = self.__getattribute__( key )
                 if key in value_mapper:
-                    rval[ key ] = value_mapper.get( key )( rval[ key ] )
+                    rval[ key ] = value_mapper.get( key, rval[ key ] )
             except AttributeError:
                 rval[ key ] = None
         return rval

lib/galaxy/web/__init__.py

@@ -1,7 +1,16 @@
 """
 The Galaxy web application framework
 """
-
-from framework import expose, json, json_pretty, require_login, require_admin, url_for, error, form, FormBuilder, expose_api, expose_api_raw
+from framework import expose
+from framework import json
+from framework import json_pretty
+from framework import require_login
+from framework import require_admin
+from framework import url_for
+from framework import error
+from framework import form
+from framework import FormBuilder
+from framework import expose_api
+from framework import expose_api_anonymous
+from framework import expose_api_raw
 from framework.base import httpexceptions
-

lib/galaxy/web/framework/__init__.py

@@ -106,7 +106,13 @@ def expose_api_raw( func ):
     """
     return expose_api( func, to_json=False )
 
-def expose_api( func, to_json=True ):
+def expose_api_anonymous( func, to_json=True ):
+    """
+    Expose this function via the API but don't require an API key.
+    """
+    return expose_api( func, to_json=to_json, key_required=False )
+
+def expose_api( func, to_json=True, key_required=True ):
     @wraps(func)
     def decorator( self, trans, *args, **kwargs ):
         def error( environ, start_response ):
@@ -114,7 +120,7 @@ def expose_api( func, to_json=True ):
             return error_message
         error_status = '403 Forbidden'
         ## If there is a user, we've authenticated a session.
-        if not trans.user and isinstance(trans.galaxy_session, Bunch):
+        if key_required and not trans.user and isinstance( trans.galaxy_session, Bunch ):
             # If trans.user is already set, don't check for a key.
             # This happens when we're authenticating using session instead of an API key.
             # The Bunch clause is used to prevent the case where there's no user, but there is a real session.

lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py

@@ -0,0 +1,339 @@
+import logging
+import urllib2
+from galaxy.util import json
+from galaxy import util
+from galaxy import web
+from galaxy.web.base.controller import BaseAPIController
+from tool_shed.galaxy_install import repository_util
+import tool_shed.util.shed_util_common as suc
+
+log = logging.getLogger( __name__ )
+
+def default_tool_shed_repository_value_mapper( trans, tool_shed_repository ):
+    value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ),
+                   'error_message' : tool_shed_repository.error_message or '' }
+    return value_mapper
+
+def get_message_for_no_shed_tool_config():
+    # This Galaxy instance is not configured with a shed-related tool panel configuration file.
+    message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a <toolbox> '
+    message += 'tag that includes a tool_path attribute value which is a directory relative to the Galaxy installation directory in order to '
+    message += 'automatically install tools from a tool shed into Galaxy (e.g., the file name shed_tool_conf.xml whose <toolbox> tag is '
+    message += '<toolbox tool_path="../shed_tools">).  For details, see the "Installation of Galaxy tool shed repository tools into a local '
+    message += 'Galaxy instance" section of the Galaxy tool shed wiki at http://wiki.galaxyproject.org/InstallingRepositoriesToGalaxy#'
+    message += 'Installing_Galaxy_tool_shed_repository_tools_into_a_local_Galaxy_instance.'
+    return message
+
+class ToolShedRepositoriesController( BaseAPIController ):
+    """RESTful controller for interactions with tool shed repositories."""
+
+    @web.expose_api
+    def index( self, trans, **kwd ):
+        """
+        GET /api/tool_shed_repositories
+        Display a list of dictionaries containing information about installed tool shed repositories.
+        """
+        # Example URL: http://localhost:8763/api/tool_shed_repositories
+        tool_shed_repository_dicts = []
+        try:
+            query = trans.sa_session.query( trans.app.model.ToolShedRepository ) \
+                                    .order_by( trans.app.model.ToolShedRepository.table.c.name ) \
+                                    .all()
+            for tool_shed_repository in query:
+                tool_shed_repository_dict = tool_shed_repository.get_api_value( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) )
+                tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
+                                                                  action='show',
+                                                                  id=trans.security.encode_id( tool_shed_repository.id ) )
+                tool_shed_repository_dicts.append( tool_shed_repository_dict )
+            return tool_shed_repository_dicts
+        except Exception, e:
+            message = "Error in the tool_shed_repositories API in index: %s" % str( e )
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return message
+
+    @web.expose_api
+    def show( self, trans, id, **kwd ):
+        """
+        GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id}
+        Display a dictionary containing information about a specified tool_shed_repository.
+
+        :param id: the encoded id of the ToolShedRepository object
+        """
+        # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e
+        try:
+            tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id )
+            tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) )
+            tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
+                                                              action='show',
+                                                              id=trans.security.encode_id( tool_shed_repository.id ) )
+            return tool_shed_repository_dict
+        except Exception, e:
+            message = "Error in tool_shed_repositories API in index: " + str( e )
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return message
+
+    @web.expose_api
+    def install_repository_revision( self, trans, payload, **kwd ):
+        """
+        POST /api/tool_shed_repositories/install_repository_revision
+        Install a specified repository revision from a specified tool shed into Galaxy.
+        
+        :param key: the current Galaxy admin user's API key
+        
+        The following parameters are included in the payload.
+        :param tool_shed_url (required): the base URL of the Tool Shed from which to install the Repository
+        :param name (required): the name of the Repository
+        :param owner (required): the owner of the Repository
+        :param changset_revision (required): the changset_revision of the RepositoryMetadata object associated with the Repository
+        :param new_tool_panel_section_label (optional): label of a new section to be added to the Galaxy tool panel in which to load
+                                                        tools contained in the Repository.  Either this parameter must be an empty string or
+                                                        the tool_panel_section_id parameter must be an empty string or both must be an empty
+                                                        string (both cannot be used simultaneously).
+        :param tool_panel_section_id (optional): id of the Galaxy tool panel section in which to load tools contained in the Repository.
+                                                 If this parameter is an empty string and the above new_tool_panel_section_label parameter is an
+                                                 empty string, tools will be loaded outside of any sections in the tool panel.  Either this
+                                                 parameter must be an empty string or the tool_panel_section_id parameter must be an empty string
+                                                 of both must be an empty string (both cannot be used simultaneously).
+        :param install_repository_dependencies (optional): Set to True if you want to install repository dependencies defined for the specified
+                                                           repository being installed.  The default setting is False.
+        :param install_tool_dependencies (optional): Set to True if you want to install tool dependencies defined for the specified repository being
+                                                     installed.  The default setting is False.
+        :param shed_tool_conf (optional): The shed-related tool panel configuration file configured in the "tool_config_file" setting in the Galaxy config file
+                                          (e.g., universe_wsgi.ini).  At least one shed-related tool panel config file is required to be configured. Setting
+                                          this parameter to a specific file enables you to choose where the specified repository will be installed because
+                                          the tool_path attribute of the <toolbox> from the specified file is used as the installation location
+                                          (e.g., <toolbox tool_path="../shed_tools">).  If this parameter is not set, a shed-related tool panel configuration
+                                          file will be selected automatically.
+        """
+        # Get the information about the repository to be installed from the payload.
+        tool_shed_url = payload.get( 'tool_shed_url', '' )
+        if not tool_shed_url:
+            raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." )
+        name = payload.get( 'name', '' )
+        if not name:
+            raise HTTPBadRequest( detail="Missing required parameter 'name'." )
+        owner = payload.get( 'owner', '' )
+        if not owner:
+            raise HTTPBadRequest( detail="Missing required parameter 'owner'." )
+        changeset_revision = payload.get( 'changeset_revision', '' )
+        if not changeset_revision:
+            raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." )
+        # Make sure this Galaxy instance is configured with a shed-related tool panel configuration file.
+        if not suc.have_shed_tool_conf_for_install( trans ):
+            message = get_message_for_no_shed_tool_config()
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        # Make sure the current user's API key proves he is an admin user in this Galaxy instance.
+        if not trans.user_is_admin():
+            raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' )
+        # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed.
+        installed_tool_shed_repositories = []
+        # Get all of the information necessary for installing the repository from the specified tool shed.
+        url = suc.url_join( tool_shed_url,
+                            'api/repositories/get_repository_revision_install_info?name=%s&owner=%s&changeset_revision=%s' % \
+                            ( name, owner, changeset_revision ) )
+        try:
+            response = urllib2.urlopen( url )
+            raw_text = response.read()
+            response.close()
+        except Exception, e:
+            message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
+                ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ), str( e ) )
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        if raw_text:
+            items = json.from_json_string( raw_text )
+            repository_dict = items[ 0 ]
+            repository_revision_dict = items[ 1 ]
+            repo_info_dict = items[ 2 ]
+        else:
+            message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \
+                ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ) )
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        repo_info_dicts = [ repo_info_dict ]
+        # Make sure the tool shed returned everything we need for installing the repository.
+        try:
+            has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ]
+        except:
+            raise HTTPBadRequest( detail="Missing required parameter 'has_repository_dependencies'." )
+        try:
+            includes_tools = repository_revision_dict[ 'includes_tools' ]
+        except:
+            raise HTTPBadRequest( detail="Missing required parameter 'includes_tools'." )
+        try:
+            includes_tool_dependencies = repository_revision_dict[ 'includes_tool_dependencies' ]
+        except:
+            raise HTTPBadRequest( detail="Missing required parameter 'includes_tool_dependencies'." )
+        try:
+            includes_tools_for_display_in_tool_panel = repository_revision_dict[ 'includes_tools_for_display_in_tool_panel' ]
+        except:
+            raise HTTPBadRequest( detail="Missing required parameter 'includes_tools_for_display_in_tool_panel'." )
+        # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository information.
+        install_repository_dependencies = payload.get( 'install_repository_dependencies', False )
+        install_tool_dependencies = payload.get( 'install_tool_dependencies', False )
+        new_tool_panel_section = payload.get( 'new_tool_panel_section_label', '' )
+        shed_tool_conf = payload.get( 'shed_tool_conf', None )
+        if shed_tool_conf:
+            # Get the tool_path setting.
+            index, shed_conf_dict = suc.get_shed_tool_conf_dict( trans.app, shed_tool_conf )
+            tool_path = shed_config_dict[ 'tool_path' ]
+        else:
+            # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting.
+            for shed_config_dict in trans.app.toolbox.shed_tool_confs:
+                # Don't use migrated_tools_conf.xml.
+                if shed_config_dict[ 'config_filename' ] != trans.app.config.migrated_tools_config:
+                    break
+            shed_tool_conf = shed_config_dict[ 'config_filename' ]
+            tool_path = shed_config_dict[ 'tool_path' ]
+        if not shed_tool_conf:
+            raise HTTPBadRequest( detail="Missing required parameter 'shed_tool_conf'." )
+        tool_panel_section_id = payload.get( 'tool_panel_section_id', '' )
+        if tool_panel_section_id not in [ None, '' ]:
+            tool_panel_section = trans.app.toolbox.tool_panel[ tool_panel_section_id ]
+        else:
+            tool_panel_section = ''
+        # Build the dictionary of information necessary for creating tool_shed_repository database records for each repository being installed.
+        installation_dict = dict( install_repository_dependencies=install_repository_dependencies,
+                                  new_tool_panel_section=new_tool_panel_section,
+                                  no_changes_checked=False,
+                                  reinstalling=False,
+                                  repo_info_dicts=repo_info_dicts,
+                                  tool_panel_section=tool_panel_section,
+                                  tool_path=tool_path,
+                                  tool_shed_url=tool_shed_url )
+        # Create the tool_shed_repository database records and gather additional information for repository installation.
+        created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \
+            repository_util.handle_tool_shed_repositories( trans, installation_dict, using_api=True )
+        if message and len( repo_info_dicts ) == 1:
+            # We're attempting to install a single repository that has already been installed into this Galaxy instance.
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        if created_or_updated_tool_shed_repositories:
+            # Build the dictionary of information necessary for installing the repositories.
+            installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories,
+                                      filtered_repo_info_dicts=filtered_repo_info_dicts,
+                                      has_repository_dependencies=has_repository_dependencies,
+                                      includes_tool_dependencies=includes_tool_dependencies,
+                                      includes_tools=includes_tools,
+                                      includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
+                                      install_repository_dependencies=install_repository_dependencies,
+                                      install_tool_dependencies=install_tool_dependencies,
+                                      message='',
+                                      new_tool_panel_section=new_tool_panel_section,
+                                      shed_tool_conf=shed_tool_conf,
+                                      status='done',
+                                      tool_panel_section=tool_panel_section,
+                                      tool_panel_section_keys=tool_panel_section_keys,
+                                      tool_path=tool_path,
+                                      tool_shed_url=tool_shed_url )
+            # Prepare the repositories for installation.  Even though this method receives a single combination of tool_shed_url, name, owner and
+            # changeset_revision, there may be multiple repositories for installation at this point because repository dependencies may have added
+            # additional repositories for installation along with the single specified repository.
+            encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = repository_util.initiate_repository_installation( trans, installation_dict )
+            # Install the repositories, keeping track of each one for later display.
+            for index, tool_shed_repository in enumerate( tool_shed_repositories ):
+                repo_info_dict = repo_info_dicts[ index ]
+                tool_panel_section_key = tool_panel_section_keys[ index ]
+                repository_util.install_tool_shed_repository( trans,
+                                                              tool_shed_repository,
+                                                              repo_info_dict,
+                                                              tool_panel_section_key,
+                                                              shed_tool_conf,
+                                                              tool_path,
+                                                              install_tool_dependencies,
+                                                              reinstalling=False )
+                tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) )
+                tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories',
+                                                                  action='show',
+                                                                  id=trans.security.encode_id( tool_shed_repository.id ) )
+                installed_tool_shed_repositories.append( tool_shed_repository_dict )
+        else:
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        # Display the list of installed repositories.
+        return installed_tool_shed_repositories
+
+    @web.expose_api
+    def install_repository_revisions( self, trans, payload, **kwd ):
+        """
+        POST /api/tool_shed_repositories/install_repository_revisions
+        Install one or more specified repository revisions from one or more specified tool sheds into Galaxy.  The received parameters
+        must be ordered lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.
+        
+        It's questionable whether this method is needed as the above method for installing a single repository can probably cover all
+        desired scenarios.  We'll keep this one around just in case...
+        
+        :param tool_shed_urls: the base URLs of the Tool Sheds from which to install a specified Repository
+        :param names: the names of the Repositories to be installed
+        :param owners: the owners of the Repositories to be installed
+        :param changset_revisions: the changset_revisions of each RepositoryMetadata object associated with each Repository to be installed
+        :param key: the current Galaxy admin user's API key
+        :param new_tool_panel_section_label: optional label of a new section to be added to the Galaxy tool panel in which to load
+                                             tools contained in the Repository.  Either this parameter must be an empty string or
+                                             the tool_panel_section_id parameter must be an empty string, as both cannot be used.
+        :param tool_panel_section_id: optional id of the Galaxy tool panel section in which to load tools contained in the Repository.
+                                      If not set, tools will be loaded outside of any sections in the tool panel.  Either this
+                                      parameter must be an empty string or the tool_panel_section_id parameter must be an empty string,
+                                      as both cannot be used.
+        """
+        if not suc.have_shed_tool_conf_for_install( trans ):
+            # This Galaxy instance is not configured with a shed-related tool panel configuration file.
+            message = get_message_for_no_shed_tool_config()
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        if not trans.user_is_admin():
+            raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' )
+        # Get the information about all of the repositories to be installed.
+        tool_shed_urls = util.listify( payload.get( 'tool_shed_urls', '' ) )
+        names = util.listify( payload.get( 'names', '' ) )
+        owners = util.listify( payload.get( 'owners', '' ) )
+        changeset_revisions = util.listify( payload.get( 'changeset_revisions', '' ) )
+        num_specified_repositories = len( tool_shed_urls )
+        if len( names ) != num_specified_repositories or \
+            len( owners ) != num_specified_repositories or \
+            len( changeset_revisions ) != num_specified_repositories:
+            message = 'Error in tool_shed_repositories API in install_repository_revisions: the received parameters must be ordered '
+            message += 'lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.'
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return dict( status='error', error=message )
+        # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain information
+        # about each of the repositories being installed.
+        # TODO: we may want to enhance this method to allow for each of the following to be associated with each repository instead of
+        # forcing all repositories to use the same settings.
+        install_repository_dependencies = payload.get( 'install_repository_dependencies', False )
+        install_tool_dependencies = payload.get( 'install_tool_dependencies', False )
+        new_tool_panel_section = payload.get( 'new_tool_panel_section_label', '' )
+        shed_tool_conf = payload.get( 'shed_tool_conf', None )
+        tool_path = payload.get( 'tool_path', None )
+        tool_panel_section_id = payload.get( 'tool_panel_section_id', '' )
+        all_installed_tool_shed_repositories = []
+        for index, tool_shed_url in enumerate( tool_shed_urls ):
+            current_payload = {}
+            current_payload[ 'tool_shed_url' ] = tool_shed_url
+            current_payload[ 'name' ] = names[ index ]
+            current_payload[ 'owner' ] = owners[ index ]
+            current_payload[ 'changeset_revision' ] = changeset_revisions[ index ]
+            current_payload[ 'install_repository_dependencies' ] = install_repository_dependencies
+            current_payload[ 'install_tool_dependencies' ] = install_tool_dependencies
+            current_payload[ 'new_tool_panel_section' ] = new_tool_panel_section
+            current_payload[ 'shed_tool_conf' ] = shed_tool_conf
+            current_payload[ 'tool_path' ] = tool_path
+            current_payload[ 'tool_panel_section_id' ] = tool_panel_section_id
+            installed_tool_shed_repositories = self.install_repository_revision( trans, **current_payload )
+            if isinstance( installed_tool_shed_repositories, dict ):
+                # We encountered an error.
+                return installed_tool_shed_repositories
+            elif isinstance( installed_tool_shed_repositories, list ):
+                all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories )
+        return all_installed_tool_shed_repositories
+    

lib/galaxy/webapps/galaxy/buildapp.py

@@ -104,12 +104,6 @@ def app_factory( global_conf, **kwargs ):
                                 name_prefix='group_',
                                 path_prefix='/api/groups/:group_id',
                                 parent_resources=dict( member_name='group', collection_name='groups' ) )
-    webapp.api_mapper.resource( 'content',
-                                'contents',
-                                controller='tool_shed_repository_contents',
-                                name_prefix='tool_shed_repository_',
-                                path_prefix='/api/tool_shed_repositories/:tool_shed_repository_id',
-                                parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) )
     _add_item_tags_controller( webapp, 
                                name_prefix="history_content_",
                                path_prefix='/api/histories/:history_id/contents/:history_content_id' )
@@ -142,7 +136,6 @@ def app_factory( global_conf, **kwargs ):
     webapp.api_mapper.resource( 'tool', 'tools', path_prefix='/api' )
     webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' )
     webapp.api_mapper.resource( 'genome', 'genomes', path_prefix='/api' )
-    webapp.api_mapper.resource( 'tool_shed_repository', 'tool_shed_repositories', path_prefix='/api' )
     webapp.api_mapper.resource( 'visualization', 'visualizations', path_prefix='/api' )
     webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' )
     webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' )
@@ -155,7 +148,14 @@ def app_factory( global_conf, **kwargs ):
     webapp.api_mapper.connect("workflow_dict", '/api/workflows/{workflow_id}/download', controller='workflows', action='workflow_dict', conditions=dict(method=['GET']))
     # Preserve the following download route for now for dependent applications  -- deprecate at some point
     webapp.api_mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET']))
-
+    # Galaxy API for tool shed features.
+    webapp.api_mapper.resource( 'tool_shed_repository',
+                                'tool_shed_repositories',
+                                controller='tool_shed_repositories',
+                                name_prefix='tool_shed_repository_',
+                                path_prefix='/api',
+                                new={ 'install_repository_revision' : 'POST' },
+                                parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) )
     # Connect logger from app
     if app.trace_logger:
         webapp.trace_logger = app.trace_logger

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -1,19 +1,35 @@
-import logging, os, shutil, tempfile, urllib2
+import logging
+import os
+import shutil
+import urllib2
 from admin import AdminGalaxy
-from galaxy import web, util, eggs, tools
-from galaxy.web.form_builder import SelectField, CheckboxField
-from galaxy.web.framework.helpers import iff, grids
+from galaxy import eggs
+from galaxy import web
+from galaxy import util
+from galaxy.web.form_builder import CheckboxField
+from galaxy.web.framework.helpers import grids
+from galaxy.web.framework.helpers import iff
 from galaxy.util import json
 from galaxy.model.orm import or_
 import tool_shed.util.shed_util_common as suc
-from tool_shed.util import common_install_util, data_manager_util, datatype_util, encoding_util, metadata_util
-from tool_shed.util import readme_util, repository_dependency_util, tool_dependency_util, tool_util, workflow_util
+from tool_shed.util import common_install_util
+from tool_shed.util import data_manager_util
+from tool_shed.util import datatype_util
+from tool_shed.util import encoding_util
+from tool_shed.util import metadata_util
+from tool_shed.util import readme_util
+from tool_shed.util import repository_dependency_util
+from tool_shed.util import tool_dependency_util
+from tool_shed.util import tool_util
+from tool_shed.util import workflow_util
 from tool_shed.galaxy_install import repository_util
 import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids
 import pkg_resources
 
 eggs.require( 'mercurial' )
-from mercurial import hg, ui, commands
+from mercurial import commands
+from mercurial import hg
+from mercurial import ui
 
 pkg_resources.require( 'elementtree' )
 from elementtree import ElementTree
@@ -373,91 +389,6 @@ class AdminToolshed( AdminGalaxy ):
         tool_version = tool_util.get_tool_version( app, guid )
         return tool_version.get_version_ids( app, reverse=True )
 
-    def handle_repository_contents( self, trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None,
-                                    tool_section=None, shed_tool_conf=None, reinstalling=False ):
-        """
-        Generate the metadata for the installed tool shed repository, among other things.  This method is called from Galaxy (never the tool shed)
-        when an admin is installing a new repository or reinstalling an uninstalled repository.
-        """
-        shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf )
-        metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
-                                                                                                   repository=tool_shed_repository,
-                                                                                                   changeset_revision=tool_shed_repository.changeset_revision,
-                                                                                                   repository_clone_url=repository_clone_url,
-                                                                                                   shed_config_dict=shed_config_dict,
-                                                                                                   relative_install_dir=relative_install_dir,
-                                                                                                   repository_files_dir=None,
-                                                                                                   resetting_all_metadata_on_repository=False,
-                                                                                                   updating_installed_repository=False,
-                                                                                                   persist=True )
-        tool_shed_repository.metadata = metadata_dict
-        trans.sa_session.add( tool_shed_repository )
-        trans.sa_session.flush()
-        if 'tool_dependencies' in metadata_dict and not reinstalling:
-            tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True )
-        if 'tools' in metadata_dict:
-            tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section )
-            sample_files = metadata_dict.get( 'sample_files', [] )
-            tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files )
-            tool_util.copy_sample_files( trans.app, tool_index_sample_files, tool_path=tool_path )
-            sample_files_copied = [ str( s ) for s in tool_index_sample_files ]
-            repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict )
-            if repository_tools_tups:
-                # Handle missing data table entries for tool parameters that are dynamically generated select lists.
-                repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups )
-                # Handle missing index files for tool parameters that are dynamically generated select lists.
-                repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app,
-                                                                                                  tool_path,
-                                                                                                  sample_files,
-                                                                                                  repository_tools_tups,
-                                                                                                  sample_files_copied )
-                # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
-                tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied )
-                tool_util.add_to_tool_panel( app=trans.app,
-                                             repository_name=tool_shed_repository.name,
-                                             repository_clone_url=repository_clone_url,
-                                             changeset_revision=tool_shed_repository.installed_changeset_revision,
-                                             repository_tools_tups=repository_tools_tups,
-                                             owner=tool_shed_repository.owner,
-                                             shed_tool_conf=shed_tool_conf,
-                                             tool_panel_dict=tool_panel_dict,
-                                             new_install=True )
-        if 'data_manager' in metadata_dict:
-            new_data_managers = data_manager_util.install_data_managers( trans.app,
-                                                                         trans.app.config.shed_data_manager_config_file,
-                                                                         metadata_dict,
-                                                                         shed_config_dict,
-                                                                         relative_install_dir,
-                                                                         tool_shed_repository,
-                                                                         repository_tools_tups )
-        if 'datatypes' in metadata_dict:
-            tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
-            if not tool_shed_repository.includes_datatypes:
-                tool_shed_repository.includes_datatypes = True
-            trans.sa_session.add( tool_shed_repository )
-            trans.sa_session.flush()
-            files_dir = relative_install_dir
-            if shed_config_dict.get( 'tool_path' ):
-                files_dir = os.path.join( shed_config_dict['tool_path'], files_dir )
-            datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir )
-            # Load data types required by tools.
-            converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False )
-            if converter_path or display_path:
-                # Create a dictionary of tool shed repository related information.
-                repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed,
-                                                                                                  name=tool_shed_repository.name,
-                                                                                                  owner=tool_shed_repository.owner,
-                                                                                                  installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
-                                                                                                  tool_dicts=metadata_dict.get( 'tools', [] ),
-                                                                                                  converter_path=converter_path,
-                                                                                                  display_path=display_path )
-            if converter_path:
-                # Load proprietary datatype converters
-                trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict )
-            if display_path:
-                # Load proprietary datatype display applications
-                trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
-
     @web.expose
     @web.require_admin
     def import_workflow( self, trans, workflow_name, repository_id, **kwd ):
@@ -605,96 +536,21 @@ class AdminToolshed( AdminGalaxy ):
         """Install specified tool shed repositories."""
         shed_tool_conf = kwd.get( 'shed_tool_conf', '' )
         tool_path = kwd[ 'tool_path' ]
-        includes_tool_dependencies = util.string_as_bool( kwd[ 'includes_tool_dependencies' ] )
         install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) )
-        # There must be a one-to-one mapping between items in the 3 lists:tool_shed_repositories, tool_panel_section_keys, repo_info_dicts.
+        # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts.
         tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] )
         repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] )
         for index, tool_shed_repository in enumerate( tool_shed_repositories ):
             repo_info_dict = repo_info_dicts[ index ]
             tool_panel_section_key = tool_panel_section_keys[ index ]
-            if tool_panel_section_key:
-                tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
-            else:
-                tool_section = None
-            if isinstance( repo_info_dict, basestring ):
-                repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict )
-            # Clone each repository to the configured location.
-            suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING )
-            repo_info_tuple = repo_info_dict[ tool_shed_repository.name ]
-            description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple
-            relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision )
-            clone_dir = os.path.join( tool_path, relative_clone_dir )
-            relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name )
-            install_dir = os.path.join( tool_path, relative_install_dir )
-            cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev )
-            if cloned_ok:
-                if reinstalling:
-                    # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated.
-                    changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository )
-                    current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None )
-                    current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None )
-                    if current_ctx_rev != ctx_rev:
-                        repo = hg.repository( suc.get_configured_ui(), path=os.path.abspath( install_dir ) )
-                        repository_util.pull_repository( repo, repository_clone_url, current_changeset_revision )
-                        suc.update_repository( repo, ctx_rev=current_ctx_rev )
-                self.handle_repository_contents( trans,
-                                                 tool_shed_repository=tool_shed_repository,
-                                                 tool_path=tool_path,
-                                                 repository_clone_url=repository_clone_url,
-                                                 relative_install_dir=relative_install_dir,
-                                                 tool_shed=tool_shed_repository.tool_shed,
-                                                 tool_section=tool_section,
-                                                 shed_tool_conf=shed_tool_conf,
-                                                 reinstalling=reinstalling )
-                trans.sa_session.refresh( tool_shed_repository )
-                metadata = tool_shed_repository.metadata
-                if 'tools' in metadata:
-                    # Get the tool_versions from the tool shed for each tool in the installed change set.
-                    suc.update_tool_shed_repository_status( trans.app,
-                                                            tool_shed_repository,
-                                                            trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
-                    tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed )
-                    url = suc.url_join( tool_shed_url,
-                                        '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \
-                                        ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) )
-                    response = urllib2.urlopen( url )
-                    text = response.read()
-                    response.close()
-                    if text:
-                        tool_version_dicts = json.from_json_string( text )
-                        tool_util.handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository )
-                    else:
-                        message += "Version information for the tools included in the <b>%s</b> repository is missing.  " % name
-                        message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions "
-                        message += "from the installed repository's <b>Repository Actions</b> menu.  "
-                        status = 'error'
-                if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
-                    work_dir = tempfile.mkdtemp()
-                    # Install tool dependencies.
-                    suc.update_tool_shed_repository_status( trans.app,
-                                                            tool_shed_repository,
-                                                            trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
-                    # Get the tool_dependencies.xml file from the repository.
-                    tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir )
-                    installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app,
-                                                                                                tool_shed_repository=tool_shed_repository,
-                                                                                                tool_dependencies_config=tool_dependencies_config,
-                                                                                                tool_dependencies=tool_shed_repository.tool_dependencies )
-                    try:
-                        shutil.rmtree( work_dir )
-                    except:
-                        pass
-                suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED )
-            else:
-                # An error occurred while cloning the repository, so reset everything necessary to enable another attempt.
-                self.set_repository_attributes( trans,
-                                                tool_shed_repository,
-                                                status=trans.model.ToolShedRepository.installation_status.ERROR,
-                                                error_message=error_message,
-                                                deleted=False,
-                                                uninstalled=False,
-                                                remove_from_disk=True )
+            repository_util.install_tool_shed_repository( trans,
+                                                          tool_shed_repository,
+                                                          repo_info_dict,
+                                                          tool_panel_section_key,
+                                                          shed_tool_conf,
+                                                          tool_path,
+                                                          install_tool_dependencies,
+                                                          reinstalling=reinstalling )
         tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ]
         return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
                                                           action='monitor_repository_installation',
@@ -929,7 +785,7 @@ class AdminToolshed( AdminGalaxy ):
     @web.expose
     @web.require_admin
     def prepare_for_install( self, trans, **kwd ):
-        if not have_shed_tool_conf_for_install( trans ):
+        if not suc.have_shed_tool_conf_for_install( trans ):
             message = 'The <b>tool_config_file</b> setting in <b>universe_wsgi.ini</b> must include at least one shed tool configuration file name with a '
             message += '<b>&lt;toolbox&gt;</b> tag that includes a <b>tool_path</b> attribute value which is a directory relative to the Galaxy installation '
             message += 'directory in order to automatically install tools from a Galaxy tool shed (e.g., the file name <b>shed_tool_conf.xml</b> whose '
@@ -956,7 +812,7 @@ class AdminToolshed( AdminGalaxy ):
         install_tool_dependencies = kwd.get( 'install_tool_dependencies', '' )
         encoded_repo_info_dicts = util.listify( kwd.get( 'encoded_repo_info_dicts', None ) )
         if not encoded_repo_info_dicts:
-            # The request originated in the tool shed.
+            # The request originated in the tool shed via a tool search.
             repository_ids = kwd.get( 'repository_ids', None )
             changeset_revisions = kwd.get( 'changeset_revisions', None )
             # Get the information necessary to install each repository.
@@ -981,78 +837,40 @@ class AdminToolshed( AdminGalaxy ):
             else:
                 install_tool_dependencies = False
             tool_path = suc.get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf )
+            installation_dict = dict( install_repository_dependencies=install_repository_dependencies,
+                                      new_tool_panel_section=new_tool_panel_section,
+                                      no_changes_checked=False,
+                                      reinstalling=False,
+                                      repo_info_dicts=repo_info_dicts,
+                                      tool_panel_section=tool_panel_section,
+                                      tool_path=tool_path,
+                                      tool_shed_url=tool_shed_url )
             created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \
-                repository_dependency_util.create_repository_dependency_objects( trans,
-                                                                                 tool_path,
-                                                                                 tool_shed_url,
-                                                                                 repo_info_dicts,
-                                                                                 reinstalling=False,
-                                                                                 install_repository_dependencies=install_repository_dependencies,
-                                                                                 no_changes_checked=False,
-                                                                                 tool_panel_section=tool_panel_section,
-                                                                                 new_tool_panel_section=new_tool_panel_section )
+                repository_util.handle_tool_shed_repositories( trans, installation_dict, using_api=False )
             if message and len( repo_info_dicts ) == 1:
-                installed_tool_shed_repository = created_or_updated_tool_shed_repositories[ 0 ]
-                message+= 'Click <a href="%s">here</a> to manage the repository.  ' % \
-                    ( web.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( installed_tool_shed_repository.id ) ) )
                 return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
                                                                   action='browse_repositories',
                                                                   message=message,
                                                                   status='error' ) )
+
             if created_or_updated_tool_shed_repositories:
-                # Handle contained tools.
-                if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section or tool_panel_section ):
-                    if new_tool_panel_section:
-                        section_id = new_tool_panel_section.lower().replace( ' ', '_' )
-                        tool_panel_section_key = 'section_%s' % str( section_id )
-                        if tool_panel_section_key in trans.app.toolbox.tool_panel:
-                            # Appending a tool to an existing section in trans.app.toolbox.tool_panel
-                            log.debug( "Appending to tool panel section: %s" % new_tool_panel_section )
-                            tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
-                        else:
-                            # Appending a new section to trans.app.toolbox.tool_panel
-                            log.debug( "Loading new tool panel section: %s" % new_tool_panel_section )
-                            elem = Element( 'section' )
-                            elem.attrib[ 'name' ] = new_tool_panel_section
-                            elem.attrib[ 'id' ] = section_id
-                            elem.attrib[ 'version' ] = ''
-                            tool_section = tools.ToolSection( elem )
-                            trans.app.toolbox.tool_panel[ tool_panel_section_key ] = tool_section
-                    else:
-                        tool_panel_section_key = 'section_%s' % tool_panel_section
-                        tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
-                else:
-                    tool_panel_section_key = None
-                    tool_section = None
-                encoded_repository_ids = [ trans.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ]
-                # Create a one-to-one mapping of tool shed repository id and tool panel section key.  All tools contained in the repositories
-                # being installed will be loaded into the same section in the tool panel.
-                for tsr in created_or_updated_tool_shed_repositories:
-                    tool_panel_section_keys.append( tool_panel_section_key )
-                new_kwd = dict( includes_tools=includes_tools,
-                                includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
-                                has_repository_dependencies=has_repository_dependencies,
-                                install_repository_dependencies=install_repository_dependencies,
-                                includes_tool_dependencies=includes_tool_dependencies,
-                                install_tool_dependencies=install_tool_dependencies,
-                                message=message,
-                                repo_info_dicts=filtered_repo_info_dicts,
-                                shed_tool_conf=shed_tool_conf,
-                                status=status,
-                                tool_path=tool_path,
-                                tool_panel_section_keys=tool_panel_section_keys,
-                                tool_shed_repository_ids=encoded_repository_ids,
-                                tool_shed_url=tool_shed_url )
-                encoded_kwd = encoding_util.tool_shed_encode( new_kwd )
-                tsr_ids = [ r.id  for r in created_or_updated_tool_shed_repositories  ]
-                tool_shed_repositories = []
-                for tsr_id in tsr_ids:
-                    tsr = trans.sa_session.query( trans.model.ToolShedRepository ).get( tsr_id )
-                    tool_shed_repositories.append( tsr )
-                clause_list = []
-                for tsr_id in tsr_ids:
-                    clause_list.append( trans.model.ToolShedRepository.table.c.id == tsr_id )
-                query = trans.sa_session.query( trans.model.ToolShedRepository ).filter( or_( *clause_list ) )
+                installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories,
+                                          filtered_repo_info_dicts=filtered_repo_info_dicts,
+                                          has_repository_dependencies=has_repository_dependencies,
+                                          includes_tool_dependencies=includes_tool_dependencies,
+                                          includes_tools=includes_tools,
+                                          includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
+                                          install_repository_dependencies=install_repository_dependencies,
+                                          install_tool_dependencies=install_tool_dependencies,
+                                          message=message,
+                                          new_tool_panel_section=new_tool_panel_section,
+                                          shed_tool_conf=shed_tool_conf,
+                                          status=status,
+                                          tool_panel_section=tool_panel_section,
+                                          tool_panel_section_keys=tool_panel_section_keys,
+                                          tool_path=tool_path,
+                                          tool_shed_url=tool_shed_url )
+                encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = repository_util.initiate_repository_installation( trans, installation_dict )
                 return trans.fill_template( 'admin/tool_shed_repository/initiate_repository_installation.mako',
                                             encoded_kwd=encoded_kwd,
                                             query=query,
@@ -1065,9 +883,9 @@ class AdminToolshed( AdminGalaxy ):
                 return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
                                                                   action='manage_repositories',
                                                                   **kwd ) )
-        shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans )
+        shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans )
         tool_path = suc.get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf )
-        tool_panel_section_select_field = build_tool_panel_section_select_field( trans )
+        tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field( trans )
         if len( repo_info_dicts ) == 1:
             # If we're installing a single repository, see if it contains a readme or dependencies that we can display.
             repo_info_dict = repo_info_dicts[ 0 ]
@@ -1401,7 +1219,7 @@ class AdminToolshed( AdminGalaxy ):
                     original_section_name = ''
             else:
                 original_section_name = ''
-            tool_panel_section_select_field = build_tool_panel_section_select_field( trans )
+            tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field( trans )
             no_changes_check_box = CheckboxField( 'no_changes', checked=True )
             if original_section_name:
                 message += "The tools contained in your <b>%s</b> repository were last loaded into the tool panel section <b>%s</b>.  " \
@@ -1417,7 +1235,7 @@ class AdminToolshed( AdminGalaxy ):
             no_changes_check_box = None
             original_section_name = ''
             tool_panel_section_select_field = None
-        shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans )
+        shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans )
         containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans,
                                                                                     tool_shed_url=tool_shed_url,
                                                                                     tool_path=tool_path,
@@ -1521,13 +1339,13 @@ class AdminToolshed( AdminGalaxy ):
         """An error occurred while cloning the repository, so reset everything necessary to enable another attempt."""
         repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] )
         if kwd.get( 'reset_repository', False ):
-            self.set_repository_attributes( trans,
-                                            repository,
-                                            status=trans.model.ToolShedRepository.installation_status.NEW,
-                                            error_message=None,
-                                            deleted=False,
-                                            uninstalled=False,
-                                            remove_from_disk=True )
+            suc.set_repository_attributes( trans,
+                                           repository,
+                                           status=trans.model.ToolShedRepository.installation_status.NEW,
+                                           error_message=None,
+                                           deleted=False,
+                                           uninstalled=False,
+                                           remove_from_disk=True )
             new_kwd = {}
             new_kwd[ 'message' ] = "You can now attempt to install the repository named <b>%s</b> again." % repository.name
             new_kwd[ 'status' ] = "done"
@@ -1538,20 +1356,6 @@ class AdminToolshed( AdminGalaxy ):
                                                           action='manage_repository',
                                                           **kwd ) )
 
-    def set_repository_attributes( self, trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ):
-        if remove_from_disk:
-            relative_install_dir = repository.repo_path( trans.app )
-            if relative_install_dir:
-                clone_dir = os.path.abspath( relative_install_dir )
-                shutil.rmtree( clone_dir )
-                log.debug( "Removed repository installation directory: %s" % str( clone_dir ) )
-        repository.error_message = error_message
-        repository.status = status
-        repository.deleted = deleted
-        repository.uninstalled = uninstalled
-        trans.sa_session.add( repository )
-        trans.sa_session.flush()
-
     @web.expose
     @web.require_admin
     def set_tool_versions( self, trans, **kwd ):
@@ -1803,46 +1607,3 @@ class AdminToolshed( AdminGalaxy ):
                                     metadata=metadata,
                                     message=message,
                                     status=status )
-
-## ---- Utility methods -------------------------------------------------------
-
-def build_shed_tool_conf_select_field( trans ):
-    """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs."""
-    options = []
-    for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs:
-        shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ]
-        if shed_tool_conf_filename != trans.app.config.migrated_tools_config:
-            if shed_tool_conf_filename.startswith( './' ):
-                option_label = shed_tool_conf_filename.replace( './', '', 1 )
-            else:
-                option_label = shed_tool_conf_filename
-            options.append( ( option_label, shed_tool_conf_filename ) )
-    select_field = SelectField( name='shed_tool_conf' )
-    for option_tup in options:
-        select_field.add_option( option_tup[0], option_tup[1] )
-    return select_field
-
-def build_tool_panel_section_select_field( trans ):
-    """Build a SelectField whose options are the sections of the current in-memory toolbox."""
-    options = []
-    for k, v in trans.app.toolbox.tool_panel.items():
-        if isinstance( v, tools.ToolSection ):
-            options.append( ( v.name, v.id ) )
-    select_field = SelectField( name='tool_panel_section', display='radio' )
-    for option_tup in options:
-        select_field.add_option( option_tup[0], option_tup[1] )
-    return select_field
-
-def can_select_tool_panel_section():
-    pass
-
-def have_shed_tool_conf_for_install( trans ):
-    if not trans.app.toolbox.shed_tool_confs:
-        return False
-    migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( trans.app.config.migrated_tools_config )
-    for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs:
-        shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
-        shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf )
-        if shed_tool_conf_name != migrated_tools_conf_name:
-            return True
-    return False

lib/galaxy/webapps/galaxy/controllers/biostar.py

@@ -8,26 +8,52 @@ import base64
 from galaxy.util import json
 import hmac
 
+# Slugifying from Armin Ronacher (http://flask.pocoo.org/snippets/5/)
+
+import re
+from unicodedata import normalize
+
+_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')
+
+
+def slugify(text, delim=u'-'):
+    """Generates an slightly worse ASCII-only slug."""
+    result = []
+    for word in _punct_re.split(text.lower()):
+        word = normalize('NFKD', word).encode('ascii', 'ignore')
+        if word:
+            result.append(word)
+    return unicode(delim.join(result))
+
+
 # Biostar requires all keys to be present, so we start with a template
 DEFAULT_PAYLOAD = {
-    'email': "", 
-    'title': "Question about Galaxy", 
+    'email': "",
+    'title': "Question about Galaxy",
     'tags': 'galaxy',
-    'tool_name': '', 
-    'tool_version': '', 
+    'tool_name': '',
+    'tool_version': '',
     'tool_id': ''
 }
 
+
 def encode_data( key, data ):
     """
     Encode data to send a question to Biostar
     """
-    text = json.dumps(data)
+    text = json.to_json_string(data)
     text = base64.urlsafe_b64encode(text)
     digest = hmac.new(key, text).hexdigest()
     return text, digest
 
 
+def tag_for_tool( tool ):
+    """
+    Generate a reasonavle biostar tag for a tool.
+    """
+    return slugify( unicode( tool.name ) )
+
+
 class BiostarController( BaseUIController ):
     """
     Provides integration with Biostar through external authentication, see: http://liondb.com/help/x/
@@ -81,6 +107,10 @@ class BiostarController( BaseUIController ):
         if not tool:
             return error( "No tool found matching '%s'" % tool_id )
         # Tool specific information for payload
-        payload = { 'title': "Question about Galaxy tool '%s'" % tool.name, 'tool_name': tool.name, 'tool_version': tool.version, 'tool_id': tool.id }
+        payload = { 'title': "Question about Galaxy tool '%s'" % tool.name,
+                    'tool_name': tool.name,
+                    'tool_version': tool.version,
+                    'tool_id': tool.id,
+                    'tags': 'galaxy ' + tag_for_tool( tool ) }
         # Pass on to regular question method
         return self.biostar_question_redirect( trans, payload )

lib/galaxy/webapps/tool_shed/api/repositories.py

@@ -1,20 +1,42 @@
 import logging
-import tool_shed.util.shed_util_common as suc
-from galaxy import web, util
+from galaxy.web.framework.helpers import time_ago
+from galaxy import web
+from galaxy import util
 from galaxy.web.base.controller import BaseAPIController
+import tool_shed.util.shed_util_common as suc
+from tool_shed.galaxy_install import repository_util
+
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( 'mercurial' )
+from mercurial import hg, ui, commands
 
 log = logging.getLogger( __name__ )
 
+def default_repository_value_mapper( trans, repository ):
+    value_mapper={ 'id' : trans.security.encode_id( repository.id ),
+                   'user_id' : trans.security.encode_id( repository.user_id ) }
+    return value_mapper
+
+def default_repository_metadata_value_mapper( trans, repository_metadata ):
+    value_mapper = { 'id' : trans.security.encode_id( repository_metadata.id ),
+                     'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) }
+    if repository_metadata.time_last_tested:
+        value_mapper[ 'time_last_tested' ] = time_ago( repository_metadata.time_last_tested )
+    return value_mapper
+
 
 class RepositoriesController( BaseAPIController ):
     """RESTful controller for interactions with repositories in the Tool Shed."""
 
-    @web.expose_api
+    @web.expose_api_anonymous
     def index( self, trans, deleted=False, **kwd ):
         """
         GET /api/repositories
         Displays a collection (list) of repositories.
         """
+        # Example URL: http://localhost:9009/api/repositories
         repository_dicts = []
         deleted = util.string_as_bool( deleted )
         try:
@@ -23,12 +45,10 @@ class RepositoriesController( BaseAPIController ):
                                     .order_by( trans.app.model.Repository.table.c.name ) \
                                     .all()
             for repository in query:
-                value_mapper={ 'id' : trans.security.encode_id( repository.id ),
-                               'user_id' : trans.security.encode_id( repository.user_id ) }
-                repository_dict = repository.get_api_value( view='collection', value_mapper=value_mapper )
-                repository_dict[ 'url' ] = web.url_for( controller='repository_contents',
-                                                        action='index',
-                                                        repository_id=trans.security.encode_id( repository.id ) )
+                repository_dict = repository.get_api_value( view='collection', value_mapper=default_repository_value_mapper( trans, repository ) )
+                repository_dict[ 'url' ] = web.url_for( controller='repositories',
+                                                        action='show',
+                                                        id=trans.security.encode_id( repository.id ) )
                 repository_dicts.append( repository_dict )
             return repository_dicts
         except Exception, e:
@@ -37,25 +57,117 @@ class RepositoriesController( BaseAPIController ):
             trans.response.status = 500
             return message
 
-    @web.expose_api
+    @web.expose_api_anonymous
     def show( self, trans, id, **kwd ):
         """
         GET /api/repositories/{encoded_repository_id}
-        Displays information about a repository in the Tool Shed.
+        Returns information about a repository in the Tool Shed.
         
-        :param id: the encoded id of the `Repository` object
+        :param id: the encoded id of the Repository object
         """
+        # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135
         try:
             repository = suc.get_repository_in_tool_shed( trans, id )
-            value_mapper={ 'id' : trans.security.encode_id( repository.id ),
-                           'user_id' : trans.security.encode_id( repository.user_id ) }
-            repository_dict = repository.get_api_value( view='element', value_mapper=value_mapper )
-            repository_dict[ 'url' ] = web.url_for( controller='repository_contents',
-                                                    action='index',
-                                                    repository_id=trans.security.encode_id( repository.id ) )
+            repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository ) )
+            repository_dict[ 'url' ] = web.url_for( controller='repositories',
+                                                    action='show',
+                                                    id=trans.security.encode_id( repository.id ) )
             return repository_dict
         except Exception, e:
             message = "Error in the Tool Shed repositories API in show: %s" % str( e )
             log.error( message, exc_info=True )
             trans.response.status = 500
             return message
+
+    @web.expose_api_anonymous
+    def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ):
+        """
+        GET /api/repository/get_repository_revision_install_info
+        
+        :param name: the name of the Repository
+        :param owner: the owner of the Repository
+        :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository
+        
+        Returns a list of the following dictionaries::
+        - a dictionary defining the Repository.  For example:
+        {
+            "deleted": false, 
+            "deprecated": false, 
+            "description": "add_column hello", 
+            "id": "f9cad7b01a472135", 
+            "long_description": "add_column hello", 
+            "name": "add_column", 
+            "owner": "test", 
+            "private": false, 
+            "times_downloaded": 6, 
+            "url": "/api/repositories/f9cad7b01a472135", 
+            "user_id": "f9cad7b01a472135"
+        }
+        - a dictionary defining the Repsoitory revision (RepositoryMetadata).  For example:
+        {
+            "changeset_revision": "3a08cc21466f", 
+            "downloadable": true, 
+            "has_repository_dependencies": false, 
+            "id": "f9cad7b01a472135", 
+            "includes_datatypes": false, 
+            "includes_tool_dependencies": false, 
+            "includes_tools": true,
+            "includes_tools_for_display_in_tool_panel": true,
+            "includes_workflows": false, 
+            "malicious": false, 
+            "repository_id": "f9cad7b01a472135", 
+            "url": "/api/repository_revisions/f9cad7b01a472135"
+        }
+        - a dictionary including the additional information required to install the repository.  For example:
+        {
+            "add_column": [
+                "add_column hello", 
+                "http://test@localhost:9009/repos/test/add_column", 
+                "3a08cc21466f", 
+                "1", 
+                "test", 
+                {}, 
+                {}
+            ]
+        }
+        """
+        # Example URL: http://localhost:9009/api/repositories/get_repository_revision_install_info?name=add_column&owner=test&changeset_revision=3a08cc21466f
+        try:
+            # Get the repository information.
+            repository = suc.get_repository_by_name_and_owner( trans.app, name, owner )
+            encoded_repository_id = trans.security.encode_id( repository.id )
+            repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository ) )
+            repository_dict[ 'url' ] = web.url_for( controller='repositories',
+                                                    action='show',
+                                                    id=encoded_repository_id )
+            # Get the repository_metadata information.
+            repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision )
+            if not repository_metadata:
+                # The changeset_revision column in the repository_metadata table has been updated with a new value value, so find the
+                # changeset_revision to which we need to update.
+                repo_dir = repository.repo_path( trans.app )
+                repo = hg.repository( suc.get_configured_ui(), repo_dir )
+                new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision )
+                repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision )
+                changeset_revision = new_changeset_revision
+            if repository_metadata:
+                encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id )
+                repository_metadata_dict = repository_metadata.get_api_value( view='collection',
+                                                                              value_mapper=default_repository_metadata_value_mapper( trans, repository_metadata ) )
+                repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
+                                                                 action='show',
+                                                                 id=encoded_repository_metadata_id )
+                # Get the repo_info_dict for installing the repository.
+                repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies = \
+                    repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision )
+                return repository_dict, repository_metadata_dict, repo_info_dict
+            else:
+                message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision )
+                log.error( message, exc_info=True )
+                trans.response.status = 500
+                return repository_dict, {}, {}
+        except Exception, e:
+            message = "Error in the Tool Shed repositories API in get_repository_revision_install_info: %s" % str( e )
+            log.error( message, exc_info=True )
+            trans.response.status = 500
+            return message

lib/galaxy/webapps/tool_shed/api/repository_contents.py

@@ -1,33 +0,0 @@
-import logging
-import tool_shed.util.shed_util_common as suc
-from galaxy import web
-from galaxy.web.base.controller import BaseAPIController
-
-log = logging.getLogger( __name__ )
-
-
-class RepositoryContentsController( BaseAPIController ):
-
-    @web.expose_api
-    def index( self, trans, **kwd ):
-        """
-        GET /api/repositories/{encoded_repository_id}
-        Displays a collection (dictionary) of repository contents.
-
-        :param repository_id: the encoded id of the `Repository` object
-        """
-        try:
-            repository_id = kwd[ 'repository_id' ]
-            repository = suc.get_repository_in_tool_shed( trans, repository_id )
-            value_mapper={ 'id' : trans.security.encode_id( repository.id ),
-                           'user_id' : trans.security.encode_id( repository.user_id ) }
-            repository_dict = repository.as_dict( value_mapper )
-            repository_dict[ 'url' ] = web.url_for( controller='repository_contents',
-                                                    action='index',
-                                                    repository_id=repository_id )
-            return repository_dict
-        except Exception, e:
-            message = "Error in the Tool Shed  repository_contents API in index: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message

lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py

@@ -1,39 +0,0 @@
-import logging
-from galaxy import web
-from galaxy.web.framework.helpers import time_ago
-from tool_shed.util import metadata_util
-from galaxy.web.base.controller import BaseAPIController
-
-log = logging.getLogger( __name__ )
-
-def default_value_mapper( trans, repository_metadata ):
-    value_mapper = { 'id' : trans.security.encode_id( repository_metadata.id ),
-                     'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) }
-    if repository_metadata.time_last_tested:
-        value_mapper[ 'time_last_tested' ] = time_ago( repository_metadata.time_last_tested )
-    return value_mapper
-
-
-class RepositoryRevisionContentsController( BaseAPIController ):
-
-    @web.expose_api
-    def index( self, trans, **kwd ):
-        """
-        GET /api/repository_revisions/{encoded_repository_metadata_id}
-        Displays a collection (dictionary) of repository_metadata contents.
-
-        :param repository_metadata_id: the encoded id of the `RepositoryMetadata` object
-        """
-        try:
-            repository_metadata_id = kwd.get( 'repository_metadata_id', None )
-            repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id )
-            repository_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) )
-            repository_dict[ 'url' ] = web.url_for( controller='repository_revision_contents',
-                                                    action='index',
-                                                    repository_metadata_id=repository_metadata_id )
-            return repository_dict
-        except Exception, e:
-            message = "Error in the Tool Shed repository_revision_contents API in index: %s" % str( e )
-            log.error( message, exc_info=True )
-            trans.response.status = 500
-            return message

lib/galaxy/webapps/tool_shed/api/repository_revisions.py

@@ -1,7 +1,9 @@
-import datetime, logging
+import datetime
+import logging
 from galaxy.web.framework.helpers import time_ago
 from tool_shed.util import metadata_util
-from galaxy import web, util
+from galaxy import web
+from galaxy import util
 from galaxy.model.orm import and_
 from galaxy.web.base.controller import BaseAPIController
 
@@ -24,6 +26,7 @@ class RepositoryRevisionsController( BaseAPIController ):
         GET /api/repository_revisions
         Displays a collection (list) of repository revisions.
         """
+        # Example URL: http://localhost:9009/api/repository_revisions
         repository_metadata_dicts = []
         # Build up an anded clause list of filters.
         clause_list = []
@@ -55,9 +58,9 @@ class RepositoryRevisionsController( BaseAPIController ):
             for repository_metadata in query:
                 repository_metadata_dict = repository_metadata.get_api_value( view='collection',
                                                                               value_mapper=default_value_mapper( trans, repository_metadata ) )
-                repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revision_contents',
-                                                                 action='index',
-                                                                 repository_metadata_id=trans.security.encode_id( repository_metadata.id ) )
+                repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
+                                                                 action='show',
+                                                                 id=trans.security.encode_id( repository_metadata.id ) )
                 repository_metadata_dicts.append( repository_metadata_dict )
             return repository_metadata_dicts
         except Exception, e:
@@ -74,12 +77,13 @@ class RepositoryRevisionsController( BaseAPIController ):
         
         :param id: the encoded id of the `RepositoryMetadata` object
         """
+        # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620
         try:
             repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id )
             repository_metadata_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) )
-            repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revision_contents',
-                                                             action='index',
-                                                             repository_metadata_id=trans.security.encode_id( repository_metadata.id ) )
+            repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
+                                                             action='show',
+                                                             id=trans.security.encode_id( repository_metadata.id ) )
             return repository_metadata_dict
         except Exception, e:
             message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e )
@@ -114,7 +118,7 @@ class RepositoryRevisionsController( BaseAPIController ):
             trans.response.status = 500
             return message
         repository_metadata_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) )
-        repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revision_contents',
-                                                         action='index',
-                                                         repository_metadata_id=trans.security.encode_id( repository_metadata.id ) )
+        repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions',
+                                                         action='show',
+                                                         id=trans.security.encode_id( repository_metadata.id ) )
         return repository_metadata_dict

lib/galaxy/webapps/tool_shed/buildapp.py

@@ -1,8 +1,11 @@
 """
 Provides factory methods to assemble the Galaxy web application
 """
-
-import logging, atexit, os, os.path, sys, config
+import atexit
+import config
+import logging
+import os
+import sys
 
 from inspect import isclass
 
@@ -69,22 +72,21 @@ def app_factory( global_conf, **kwargs ):
     webapp.add_route( '/:controller/:action', action='index' )
     webapp.add_route( '/:action', controller='repository', action='index' )
     webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' )
-    # Add the web API.
+    # Add the web API.  # A good resource for RESTful services - http://routes.readthedocs.org/en/latest/restful.html
     webapp.add_api_controllers( 'galaxy.webapps.tool_shed.api', app )
-    webapp.api_mapper.resource( 'content',
-                                'contents',
-                                controller='repository_contents',
+    webapp.api_mapper.resource( 'repository',
+                                'repositories',
+                                controller='repositories',
+                                collection={ 'get_repository_revision_install_info' : 'GET' },
                                 name_prefix='repository_',
-                                path_prefix='/api/repositories/:repository_id',
+                                path_prefix='/api',
                                 parent_resources=dict( member_name='repository', collection_name='repositories' ) )
-    webapp.api_mapper.resource( 'content',
-                                'contents',
-                                controller='repository_revision_contents',
+    webapp.api_mapper.resource( 'repository_revision',
+                                'repository_revisions',
+                                controller='repository_revisions',
                                 name_prefix='repository_revision_',
-                                path_prefix='/api/repository_revisions/:repository_metadata_id',
+                                path_prefix='/api',
                                 parent_resources=dict( member_name='repository_revision', collection_name='repository_revisions' ) )
-    webapp.api_mapper.resource( 'repository', 'repositories', path_prefix='/api' )
-    webapp.api_mapper.resource( 'repository_revision', 'repository_revisions', path_prefix='/api' )
     webapp.finalize_config()
     # Wrap the webapp in some useful middleware
     if kwargs.get( 'middleware', True ):

lib/galaxy/webapps/tool_shed/controllers/repository.py

@@ -1,26 +1,47 @@
-import os, logging, re, tempfile, ConfigParser, string
-from time import gmtime, strftime
+import ConfigParser
+import logging
+import os
+import re
+import string
+import tempfile
+from time import gmtime
+from time import strftime
 from datetime import date, datetime
-from galaxy import util, web
+from galaxy import util
+from galaxy import web
 from galaxy.util.odict import odict
 from galaxy.web.base.controller import BaseUIController
-from galaxy.web.form_builder import CheckboxField, build_select_field
+from galaxy.web.form_builder import CheckboxField
+from galaxy.web.form_builder import build_select_field
 from galaxy.webapps.tool_shed import model
 from galaxy.webapps.tool_shed.model import directory_hash_id
 from galaxy.web.framework.helpers import grids
 from galaxy.util import json
-from galaxy.model.orm import and_, or_
+from galaxy.model.orm import and_
+from galaxy.model.orm import or_
 import tool_shed.util.shed_util_common as suc
-from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, review_util, tool_dependency_util, tool_util, workflow_util
+from tool_shed.util import encoding_util
+from tool_shed.util import metadata_util
+from tool_shed.util import readme_util
+from tool_shed.util import repository_dependency_util
+from tool_shed.util import review_util
+from tool_shed.util import tool_dependency_util
+from tool_shed.util import tool_util
+from tool_shed.util import workflow_util
 from tool_shed.galaxy_install import repository_util
-from galaxy.webapps.tool_shed.util import common_util, container_util
+from galaxy.webapps.tool_shed.util import common_util
+from galaxy.webapps.tool_shed.util import container_util
 import galaxy.tools
 import tool_shed.grids.repository_grids as repository_grids
 import tool_shed.grids.util as grids_util
 
 from galaxy import eggs
 eggs.require('mercurial')
-from mercurial import hg, ui, patch, commands
+
+from mercurial import commands
+from mercurial import hg
+from mercurial import patch
+from mercurial import ui
 
 log = logging.getLogger( __name__ )
 
@@ -1124,35 +1145,17 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ):
         includes_tool_dependencies = False
         repo_info_dicts = []
         for tup in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ):
-            repository_id, changeset_revision = tup
-            repository = suc.get_repository_in_tool_shed( trans, repository_id )
-            repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository )
-            repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
-            metadata = repository_metadata.metadata
-            if not includes_tools:
-                if 'tools' in metadata:
-                    includes_tools = True
-            if not includes_tools_for_display_in_tool_panel:
-                includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel
-            if not has_repository_dependencies:
-                if 'repository_dependencies' in metadata:
-                    has_repository_dependencies = True
-            if not includes_tool_dependencies:
-                if 'tool_dependencies' in metadata:
-                    includes_tool_dependencies = True
-            repo_dir = repository.repo_path( trans.app )
-            repo = hg.repository( suc.get_configured_ui(), repo_dir )
-            ctx = suc.get_changectx_for_changeset( repo, changeset_revision )
-            repo_info_dict = repository_util.create_repo_info_dict( trans=trans,
-                                                                    repository_clone_url=repository_clone_url,
-                                                                    changeset_revision=changeset_revision,
-                                                                    ctx_rev=str( ctx.rev() ),
-                                                                    repository_owner=repository.user.username,
-                                                                    repository_name=repository.name,
-                                                                    repository=repository,
-                                                                    repository_metadata=repository_metadata,
-                                                                    tool_dependencies=None,
-                                                                    repository_dependencies=None )
+            repository_id, changeset_revision = tup            
+            repo_info_dict, cur_includes_tools, cur_includes_tool_dependencies, cur_includes_tools_for_display_in_tool_panel, cur_has_repository_dependencies = \
+                repository_util.get_repo_info_dict( trans, repository_id, changeset_revision )
+            if cur_has_repository_dependencies and not has_repository_dependencies:
+                has_repository_dependencies = True
+            if cur_includes_tools and not includes_tools:
+                includes_tools = True
+            if cur_includes_tool_dependencies and not includes_tool_dependencies:
+                includes_tool_dependencies = True
+            if cur_includes_tools_for_display_in_tool_panel and not includes_tools_for_display_in_tool_panel:
+                includes_tools_for_display_in_tool_panel = True
             repo_info_dicts.append( encoding_util.tool_shed_encode( repo_info_dict ) )
         return dict( includes_tools=includes_tools,
                      includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,

lib/galaxy/webapps/tool_shed/model/__init__.py

@@ -192,10 +192,10 @@ class Repository( object, APIItem ):
 
 class RepositoryMetadata( object, APIItem ):
     api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes',
-                                    'includes_tools', 'includes_tool_dependencies', 'includes_workflows' )
+                                    'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' )
     api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct',
                                  'do_not_test', 'time_last_tested', 'tool_test_errors', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools',
-                                 'includes_tool_dependencies', 'includes_workflows' )
+                                 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' )
     def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, 
                   tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None, has_repository_dependencies=False,
                   includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ):

lib/tool_shed/galaxy_install/repository_util.py

@@ -1,14 +1,35 @@
-import os, logging, threading, urllib2
-from galaxy.web import url_for
+import logging
+import os
+import shutil
+import tempfile
+import threading
+import urllib2
+from galaxy import tools
+from galaxy.util import json
+from galaxy import web
+from galaxy.model.orm import or_
 from galaxy.webapps.tool_shed.util import container_util
 import tool_shed.util.shed_util_common as suc
-from tool_shed.util import encoding_util, repository_dependency_util, tool_dependency_util, tool_util
+from tool_shed.util import common_install_util
+from tool_shed.util import data_manager_util
+from tool_shed.util import datatype_util
+from tool_shed.util import encoding_util
+from tool_shed.util import repository_dependency_util
+from tool_shed.util import metadata_util
+from tool_shed.util import tool_dependency_util
+from tool_shed.util import tool_util
 
 from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( 'mercurial' )
-from mercurial import hg, ui, commands
+from mercurial import commands
+from mercurial import hg
+from mercurial import ui
+
+pkg_resources.require( 'elementtree' )
+from elementtree import ElementTree
+from elementtree.ElementTree import Element
 
 log = logging.getLogger( __name__ )
 
@@ -41,12 +62,12 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_
                     repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans,
                                                                                                    repository=repository,
                                                                                                    repository_metadata=repository_metadata,
-                                                                                                   toolshed_base_url=str( url_for( '/', qualified=True ) ).rstrip( '/' ),
+                                                                                                   toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ),
                                                                                                    key_rd_dicts_to_be_processed=None,
                                                                                                    all_repository_dependencies=None,
                                                                                                    handled_key_rd_dicts=None,
                                                                                                    circular_repository_dependencies=None )
-                tool_dependencies = metadata.get( 'tool_dependencies', None )
+                tool_dependencies = metadata.get( 'tool_dependencies', {} )
     if tool_dependencies:
         new_tool_dependencies = {}
         for dependency_key, requirements_dict in tool_dependencies.items():
@@ -74,6 +95,39 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_
                                                  tool_dependencies )
     return repo_info_dict
 
+def get_repo_info_dict( trans, repository_id, changeset_revision ):
+    repository = suc.get_repository_in_tool_shed( trans, repository_id )
+    repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository )
+    repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision )
+    metadata = repository_metadata.metadata
+    if 'tools' in metadata:
+        includes_tools = True
+    else:
+        includes_tools = False
+    includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel
+    if 'repository_dependencies' in metadata:
+        has_repository_dependencies = True
+    else:
+        has_repository_dependencies = False
+    if 'tool_dependencies' in metadata:
+        includes_tool_dependencies = True
+    else:
+        includes_tool_dependencies = False
+    repo_dir = repository.repo_path( trans.app )
+    repo = hg.repository( suc.get_configured_ui(), repo_dir )
+    ctx = suc.get_changectx_for_changeset( repo, changeset_revision )
+    repo_info_dict = create_repo_info_dict( trans=trans,
+                                            repository_clone_url=repository_clone_url,
+                                            changeset_revision=changeset_revision,
+                                            ctx_rev=str( ctx.rev() ),
+                                            repository_owner=repository.user.username,
+                                            repository_name=repository.name,
+                                            repository=repository,
+                                            repository_metadata=repository_metadata,
+                                            tool_dependencies=None,
+                                            repository_dependencies=None )
+    return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies
+
 def get_update_to_changeset_revision_and_ctx_rev( trans, repository ):
     """Return the changeset revision hash to which the repository can be updated."""
     changeset_revision_dict = {}
@@ -117,6 +171,271 @@ def get_update_to_changeset_revision_and_ctx_rev( trans, repository ):
         changeset_revision_dict[ 'ctx_rev' ] = None
     return changeset_revision_dict
 
+def handle_repository_contents( trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None,
+                                reinstalling=False ):
+    """
+    Generate the metadata for the installed tool shed repository, among other things.  This method is called from Galaxy (never the tool shed)
+    when an administrator is installing a new repository or reinstalling an uninstalled repository.
+    """
+    shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf )
+    metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app,
+                                                                                               repository=tool_shed_repository,
+                                                                                               changeset_revision=tool_shed_repository.changeset_revision,
+                                                                                               repository_clone_url=repository_clone_url,
+                                                                                               shed_config_dict=shed_config_dict,
+                                                                                               relative_install_dir=relative_install_dir,
+                                                                                               repository_files_dir=None,
+                                                                                               resetting_all_metadata_on_repository=False,
+                                                                                               updating_installed_repository=False,
+                                                                                               persist=True )
+    tool_shed_repository.metadata = metadata_dict
+    trans.sa_session.add( tool_shed_repository )
+    trans.sa_session.flush()
+    if 'tool_dependencies' in metadata_dict and not reinstalling:
+        tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True )
+    if 'tools' in metadata_dict:
+        tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section )
+        sample_files = metadata_dict.get( 'sample_files', [] )
+        tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files )
+        tool_util.copy_sample_files( trans.app, tool_index_sample_files, tool_path=tool_path )
+        sample_files_copied = [ str( s ) for s in tool_index_sample_files ]
+        repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict )
+        if repository_tools_tups:
+            # Handle missing data table entries for tool parameters that are dynamically generated select lists.
+            repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups )
+            # Handle missing index files for tool parameters that are dynamically generated select lists.
+            repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups, sample_files_copied )
+            # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
+            tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied )
+            tool_util.add_to_tool_panel( app=trans.app,
+                                         repository_name=tool_shed_repository.name,
+                                         repository_clone_url=repository_clone_url,
+                                         changeset_revision=tool_shed_repository.installed_changeset_revision,
+                                         repository_tools_tups=repository_tools_tups,
+                                         owner=tool_shed_repository.owner,
+                                         shed_tool_conf=shed_tool_conf,
+                                         tool_panel_dict=tool_panel_dict,
+                                         new_install=True )
+    if 'data_manager' in metadata_dict:
+        new_data_managers = data_manager_util.install_data_managers( trans.app,
+                                                                     trans.app.config.shed_data_manager_config_file,
+                                                                     metadata_dict,
+                                                                     shed_config_dict,
+                                                                     relative_install_dir,
+                                                                     tool_shed_repository,
+                                                                     repository_tools_tups )
+    if 'datatypes' in metadata_dict:
+        tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
+        if not tool_shed_repository.includes_datatypes:
+            tool_shed_repository.includes_datatypes = True
+        trans.sa_session.add( tool_shed_repository )
+        trans.sa_session.flush()
+        files_dir = relative_install_dir
+        if shed_config_dict.get( 'tool_path' ):
+            files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir )
+        datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir )
+        # Load data types required by tools.
+        converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False )
+        if converter_path or display_path:
+            # Create a dictionary of tool shed repository related information.
+            repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed,
+                                                                                              name=tool_shed_repository.name,
+                                                                                              owner=tool_shed_repository.owner,
+                                                                                              installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
+                                                                                              tool_dicts=metadata_dict.get( 'tools', [] ),
+                                                                                              converter_path=converter_path,
+                                                                                              display_path=display_path )
+        if converter_path:
+            # Load proprietary datatype converters
+            trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict )
+        if display_path:
+            # Load proprietary datatype display applications
+            trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
+
+def handle_tool_shed_repositories( trans, installation_dict, using_api=False ):
+    # The following installation_dict entries are all required.
+    install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ]
+    new_tool_panel_section = installation_dict[ 'new_tool_panel_section' ]
+    no_changes_checked = installation_dict[ 'no_changes_checked' ]
+    reinstalling = installation_dict[ 'reinstalling' ]
+    repo_info_dicts = installation_dict[ 'repo_info_dicts' ]
+    tool_panel_section = installation_dict[ 'tool_panel_section' ]
+    tool_path = installation_dict[ 'tool_path' ]
+    tool_shed_url = installation_dict[ 'tool_shed_url' ]
+    created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \
+        repository_dependency_util.create_repository_dependency_objects( trans=trans,
+                                                                         tool_path=tool_path,
+                                                                         tool_shed_url=tool_shed_url,
+                                                                         repo_info_dicts=repo_info_dicts,
+                                                                         reinstalling=reinstalling,
+                                                                         install_repository_dependencies=install_repository_dependencies,
+                                                                         no_changes_checked=no_changes_checked,
+                                                                         tool_panel_section=tool_panel_section,
+                                                                         new_tool_panel_section=new_tool_panel_section )
+    if message and len( repo_info_dicts ) == 1 and not using_api:
+        installed_tool_shed_repository = created_or_updated_tool_shed_repositories[ 0 ]
+        message += 'Click <a href="%s">here</a> to manage the repository.  ' % \
+            ( web.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( installed_tool_shed_repository.id ) ) )
+    return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message
+
+def initiate_repository_installation( trans, installation_dict ):
+    # The following installation_dict entries are all required.
+    created_or_updated_tool_shed_repositories = installation_dict[ 'created_or_updated_tool_shed_repositories' ]
+    filtered_repo_info_dicts = installation_dict[ 'filtered_repo_info_dicts' ]
+    has_repository_dependencies = installation_dict[ 'has_repository_dependencies' ]
+    includes_tool_dependencies = installation_dict[ 'includes_tool_dependencies' ]
+    includes_tools = installation_dict[ 'includes_tools' ]
+    includes_tools_for_display_in_tool_panel = installation_dict[ 'includes_tools_for_display_in_tool_panel' ]
+    install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ]
+    install_tool_dependencies = installation_dict[ 'install_tool_dependencies' ]
+    message = installation_dict[ 'message' ]
+    new_tool_panel_section = installation_dict[ 'new_tool_panel_section' ]
+    shed_tool_conf = installation_dict[ 'shed_tool_conf' ]
+    status = installation_dict[ 'status' ]
+    tool_panel_section = installation_dict[ 'tool_panel_section' ]
+    tool_panel_section_keys = installation_dict[ 'tool_panel_section_keys' ]
+    tool_path = installation_dict[ 'tool_path' ]
+    tool_shed_url = installation_dict[ 'tool_shed_url' ]
+    # Handle contained tools.
+    if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section or tool_panel_section ):
+        if new_tool_panel_section:
+            section_id = new_tool_panel_section.lower().replace( ' ', '_' )
+            tool_panel_section_key = 'section_%s' % str( section_id )
+            if tool_panel_section_key in trans.app.toolbox.tool_panel:
+                # Appending a tool to an existing section in trans.app.toolbox.tool_panel
+                log.debug( "Appending to tool panel section: %s" % new_tool_panel_section )
+                tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
+            else:
+                # Appending a new section to trans.app.toolbox.tool_panel
+                log.debug( "Loading new tool panel section: %s" % new_tool_panel_section )
+                elem = Element( 'section' )
+                elem.attrib[ 'name' ] = new_tool_panel_section
+                elem.attrib[ 'id' ] = section_id
+                elem.attrib[ 'version' ] = ''
+                tool_section = tools.ToolSection( elem )
+                trans.app.toolbox.tool_panel[ tool_panel_section_key ] = tool_section
+        else:
+            tool_panel_section_key = 'section_%s' % tool_panel_section
+            tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
+    else:
+        tool_panel_section_key = None
+        tool_section = None
+    encoded_repository_ids = [ trans.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ]
+    # Create a one-to-one mapping of tool shed repository id and tool panel section key.  All tools contained in the repositories being installed will be loaded
+    # into the same section in the tool panel.
+    for tsr in created_or_updated_tool_shed_repositories:
+        tool_panel_section_keys.append( tool_panel_section_key )
+    new_kwd = dict( includes_tools=includes_tools,
+                    includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
+                    has_repository_dependencies=has_repository_dependencies,
+                    install_repository_dependencies=install_repository_dependencies,
+                    includes_tool_dependencies=includes_tool_dependencies,
+                    install_tool_dependencies=install_tool_dependencies,
+                    message=message,
+                    repo_info_dicts=filtered_repo_info_dicts,
+                    shed_tool_conf=shed_tool_conf,
+                    status=status,
+                    tool_path=tool_path,
+                    tool_panel_section_keys=tool_panel_section_keys,
+                    tool_shed_repository_ids=encoded_repository_ids,
+                    tool_shed_url=tool_shed_url )
+    encoded_kwd = encoding_util.tool_shed_encode( new_kwd )
+    tsr_ids = [ r.id  for r in created_or_updated_tool_shed_repositories  ]
+    tool_shed_repositories = []
+    for tsr_id in tsr_ids:
+        tsr = trans.sa_session.query( trans.model.ToolShedRepository ).get( tsr_id )
+        tool_shed_repositories.append( tsr )
+    clause_list = []
+    for tsr_id in tsr_ids:
+        clause_list.append( trans.model.ToolShedRepository.table.c.id == tsr_id )
+    query = trans.sa_session.query( trans.model.ToolShedRepository ).filter( or_( *clause_list ) )
+    return encoded_kwd, query, tool_shed_repositories, encoded_repository_ids
+
+def install_tool_shed_repository( trans, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies,
+                                  reinstalling=False ):
+    if tool_panel_section_key:
+        tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ]
+    else:
+        tool_section = None
+    if isinstance( repo_info_dict, basestring ):
+        repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict )
+    # Clone each repository to the configured location.
+    suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING )
+    repo_info_tuple = repo_info_dict[ tool_shed_repository.name ]
+    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple
+    relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision )
+    clone_dir = os.path.join( tool_path, relative_clone_dir )
+    relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name )
+    install_dir = os.path.join( tool_path, relative_install_dir )
+    cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev )
+    if cloned_ok:
+        if reinstalling:
+            # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated.
+            changeset_revision_dict = get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository )
+            current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None )
+            current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None )
+            if current_ctx_rev != ctx_rev:
+                repo = hg.repository( suc.get_configured_ui(), path=os.path.abspath( install_dir ) )
+                pull_repository( repo, repository_clone_url, current_changeset_revision )
+                suc.update_repository( repo, ctx_rev=current_ctx_rev )
+        handle_repository_contents( trans,
+                                    tool_shed_repository=tool_shed_repository,
+                                    tool_path=tool_path,
+                                    repository_clone_url=repository_clone_url,
+                                    relative_install_dir=relative_install_dir,
+                                    tool_shed=tool_shed_repository.tool_shed,
+                                    tool_section=tool_section,
+                                    shed_tool_conf=shed_tool_conf,
+                                    reinstalling=reinstalling )
+        trans.sa_session.refresh( tool_shed_repository )
+        metadata = tool_shed_repository.metadata
+        if 'tools' in metadata:
+            # Get the tool_versions from the tool shed for each tool in the installed change set.
+            suc.update_tool_shed_repository_status( trans.app,
+                                                    tool_shed_repository,
+                                                    trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
+            tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed )
+            url = suc.url_join( tool_shed_url,
+                                '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \
+                                ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) )
+            response = urllib2.urlopen( url )
+            text = response.read()
+            response.close()
+            if text:
+                tool_version_dicts = json.from_json_string( text )
+                tool_util.handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository )
+            else:
+                message += "Version information for the tools included in the <b>%s</b> repository is missing.  " % name
+                message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions "
+                message += "from the installed repository's <b>Repository Actions</b> menu.  "
+                status = 'error'
+        if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
+            work_dir = tempfile.mkdtemp()
+            # Install tool dependencies.
+            suc.update_tool_shed_repository_status( trans.app,
+                                                    tool_shed_repository,
+                                                    trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
+            # Get the tool_dependencies.xml file from the repository.
+            tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir )
+            installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app,
+                                                                                        tool_shed_repository=tool_shed_repository,
+                                                                                        tool_dependencies_config=tool_dependencies_config,
+                                                                                        tool_dependencies=tool_shed_repository.tool_dependencies )
+            try:
+                shutil.rmtree( work_dir )
+            except:
+                pass
+        suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED )
+    else:
+        # An error occurred while cloning the repository, so reset everything necessary to enable another attempt.
+        suc.set_repository_attributes( trans,
+                                       tool_shed_repository,
+                                       status=trans.model.ToolShedRepository.installation_status.ERROR,
+                                       error_message=error_message,
+                                       deleted=False,
+                                       uninstalled=False,
+                                       remove_from_disk=True )
+
 def merge_containers_dicts_for_new_install( containers_dicts ):
     """
     When installing one or more tool shed repositories for the first time, the received list of containers_dicts contains a containers_dict for

lib/tool_shed/scripts/api/common.py

@@ -0,0 +1,165 @@
+import os, sys, urllib, urllib2
+
+new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', 'lib' ) ]
+new_path.extend( sys.path[ 1: ] )
+sys.path = new_path
+
+from galaxy import eggs
+import pkg_resources
+
+pkg_resources.require( "simplejson" )
+import simplejson
+
+pkg_resources.require( "pycrypto" )
+from Crypto.Cipher import Blowfish
+from Crypto.Util.randpool import RandomPool
+from Crypto.Util import number
+
+def encode_id( config_id_secret, obj_id ):
+    # Utility method to encode ID's
+    id_cipher = Blowfish.new( config_id_secret )
+    # Convert to string
+    s = str( obj_id )
+    # Pad to a multiple of 8 with leading "!" 
+    s = ( "!" * ( 8 - len(s) % 8 ) ) + s
+    # Encrypt
+    return id_cipher.encrypt( s ).encode( 'hex' )
+
+def delete( api_key, url, data, return_formatted=True ):
+    # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy.
+    try:
+        url = make_url( api_key, url )
+        req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ))
+        req.get_method = lambda: 'DELETE'
+        r = simplejson.loads( urllib2.urlopen( req ).read() )
+    except urllib2.HTTPError, e:
+        if return_formatted:
+            print e
+            print e.read( 1024 )
+            sys.exit( 1 )
+        else:
+            return 'Error. '+ str( e.read( 1024 ) )
+    if not return_formatted:
+        return r
+    print 'Response'
+    print '--------'
+    print r
+
+def display( api_key, url, return_formatted=True ):
+    # Sends an API GET request and acts as a generic formatter for the JSON response.
+    try:
+        r = get( api_key, url )
+    except urllib2.HTTPError, e:
+        print e
+        print e.read( 1024 ) # Only return the first 1K of errors.
+        sys.exit( 1 )
+    if type( r ) == unicode:
+        print 'error: %s' % r
+        return None
+    if not return_formatted:
+        return r
+    elif type( r ) == list:
+        # Response is a collection as defined in the REST style.
+        print 'Collection Members'
+        print '------------------'
+        for n, i in enumerate(r):
+            # All collection members should have a name and url in the response.
+            print '#%d: %s' % (n+1, i.pop( 'url' ) )
+            if 'name' in i:
+                print '  name: %s' % i.pop( 'name' )
+            for k, v in i.items():
+                print '  %s: %s' % ( k, v )
+        print ''
+        print '%d element(s) in collection' % len( r )
+    elif type( r ) == dict:
+        # Response is an element as defined in the REST style.
+        print 'Member Information'
+        print '------------------'
+        for k, v in r.items():
+            print '%s: %s' % ( k, v )
+    elif type( r ) == str:
+        print r
+    else:
+        print 'response is unknown type: %s' % type( r )
+
+def get( api_key, url ):
+    # Do the actual GET.
+    url = make_url( api_key, url )
+    try:
+        return simplejson.loads( urllib2.urlopen( url ).read() )
+    except simplejson.decoder.JSONDecodeError, e:
+        print "URL did not return JSON data"
+        sys.exit(1)
+
+def make_url( api_key, url, args=None ):
+    # Adds the API Key to the URL if it's not already there.
+    if args is None:
+        args = []
+    argsep = '&'
+    if '?' not in url:
+        argsep = '?'
+    if '?key=' not in url and '&key=' not in url:
+        args.insert( 0, ( 'key', api_key ) )
+    return url + argsep + '&'.join( [ '='.join( t ) for t in args ] )
+
+def post( api_key, url, data ):
+    # Do the actual POST.
+    url = make_url( api_key, url )
+    req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) )
+    return simplejson.loads( urllib2.urlopen( req ).read() )
+
+def put( api_key, url, data ):
+    # Do the actual PUT.
+    url = make_url( api_key, url )
+    req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ))
+    req.get_method = lambda: 'PUT'
+    return simplejson.loads( urllib2.urlopen( req ).read() )
+
+def submit( api_key, url, data, return_formatted=True ):
+    # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy.
+    try:
+        r = post( api_key, url, data )
+    except urllib2.HTTPError, e:
+        if return_formatted:
+            print e
+            print e.read( 1024 )
+            sys.exit( 1 )
+        else:
+            return 'Error. '+ str( e.read( 1024 ) )
+    if not return_formatted:
+        return r
+    print 'Response'
+    print '--------'
+    if type( r ) == list:
+        # Currently the only implemented responses are lists of dicts, because submission creates some number of collection elements.
+        for i in r:
+            if type( i ) == dict:
+                if 'url' in i:
+                    print i.pop( 'url' )
+                else:
+                    print '----'
+                if 'name' in i:
+                    print '  name: %s' % i.pop( 'name' )
+                for k, v in i.items():
+                    print '  %s: %s' % ( k, v )
+            else:
+                print i
+    else:
+        print r
+
+def update( api_key, url, data, return_formatted=True ):
+    # Sends an API PUT request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy.
+    try:
+        r = put( api_key, url, data )
+    except urllib2.HTTPError, e:
+        if return_formatted:
+            print e
+            print e.read( 1024 )
+            sys.exit( 1 )
+        else:
+            return 'Error. '+ str( e.read( 1024 ) )
+    if not return_formatted:
+        return r
+    print 'Response'
+    print '--------'
+    print r

lib/tool_shed/util/common_install_util.py

@@ -1,17 +1,25 @@
-import os, logging, urllib2
-import tool_shed.util.shed_util_common as suc
-from tool_shed.util import encoding_util, data_manager_util, datatype_util, tool_util
-from tool_shed.galaxy_install.tool_dependencies.install_util import install_package, set_environment
+import logging
+import os
+import urllib2
+from galaxy import eggs
 from galaxy import util
 from galaxy.util import json
 from galaxy.webapps.tool_shed.util import container_util
+import tool_shed.util.shed_util_common as suc
+from tool_shed.util import encoding_util
+from tool_shed.util import data_manager_util
+from tool_shed.util import datatype_util
+from tool_shed.util import tool_util
+from tool_shed.galaxy_install.tool_dependencies.install_util import install_package
+from tool_shed.galaxy_install.tool_dependencies.install_util import set_environment
 
-from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( 'elementtree' )
-from elementtree import ElementTree, ElementInclude
-from elementtree.ElementTree import Element, SubElement
+from elementtree import ElementTree
+from elementtree import ElementInclude
+from elementtree.ElementTree import Element
+from elementtree.ElementTree import SubElement
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/common_util.py

@@ -1,4 +1,5 @@
-import os, urllib2
+import os
+import urllib2
 from galaxy import util
 from galaxy.util.odict import odict
 from tool_shed.util import encoding_util
@@ -63,6 +64,7 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri
         exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n'
         raise Exception( exception_msg )  
     return tool_shed_accessible, missing_tool_configs_dict
+
 def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ):
     file_path = elem.get( 'file', None )
     if file_path:
@@ -71,6 +73,7 @@ def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_d
             tool_dependencies = migrated_tool_configs_dict[ name ]
             missing_tool_configs_dict[ name ] = tool_dependencies
     return missing_tool_configs_dict
+
 def get_non_shed_tool_panel_configs( app ):
     # Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.
     config_filenames = []
@@ -83,6 +86,7 @@ def get_non_shed_tool_panel_configs( app ):
         if tool_path is None:
             config_filenames.append( config_filename )
     return config_filenames
+
 def get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ):
     search_str = '://%s' % tool_shed
     for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items():

lib/tool_shed/util/data_manager_util.py

@@ -1,4 +1,5 @@
-import os, logging
+import logging
+import os
 from galaxy import util
 import tool_shed.util.shed_util_common as suc
 

lib/tool_shed/util/datatype_util.py

@@ -1,13 +1,17 @@
-import os, logging, tempfile
-import tool_shed.util.shed_util_common as suc
-from galaxy import util
-
+import logging
+import os
+import tempfile
 from galaxy import eggs
+from galaxy import util
+import tool_shed.util.shed_util_common as suc
+
 import pkg_resources
 
 pkg_resources.require( 'elementtree' )
-from elementtree import ElementTree, ElementInclude
-from elementtree.ElementTree import Element, SubElement
+from elementtree import ElementTree
+from elementtree import ElementInclude
+from elementtree.ElementTree import Element
+from elementtree.ElementTree import SubElement
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/encoding_util.py

@@ -1,8 +1,9 @@
-import binascii, logging
+import binascii
+import logging
+from galaxy import eggs
 from galaxy.util.hash_util import hmac_new
 from galaxy.util.json import json_fix
 
-from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( "simplejson" )
@@ -34,6 +35,7 @@ def tool_shed_decode( value ):
     if values is None:
         values = value
     return values
+
 def tool_shed_encode( val ):
     if isinstance( val, dict ):
         value = simplejson.dumps( val )

lib/tool_shed/util/metadata_util.py

@@ -1,21 +1,32 @@
-import os, tempfile, logging, urllib2
-from galaxy import web, util
-from galaxy.util import inflector, json
-from galaxy.webapps.tool_shed.util import container_util
-from galaxy.datatypes import checkers
-from galaxy.tools.data_manager.manager import DataManager
-import tool_shed.util.shed_util_common as suc
-from tool_shed.util import common_install_util, readme_util, tool_dependency_util, tool_util
-from galaxy.model.orm import and_
-
+import logging
+import os
+import tempfile
+import urllib2
 from galaxy import eggs
+from galaxy import util
+from galaxy import web
+from galaxy.datatypes import checkers
+from galaxy.model.orm import and_
+from galaxy.tools.data_manager.manager import DataManager
+from galaxy.util import inflector
+from galaxy.util import json
+from galaxy.webapps.tool_shed.util import container_util
+import tool_shed.util.shed_util_common as suc
+from tool_shed.util import common_install_util
+from tool_shed.util import readme_util
+from tool_shed.util import tool_dependency_util
+from tool_shed.util import tool_util
+
 import pkg_resources
 
 pkg_resources.require( 'mercurial' )
-from mercurial import hg, ui, commands
+from mercurial import commands
+from mercurial import hg
+from mercurial import ui
 
 pkg_resources.require( 'elementtree' )
-from elementtree import ElementTree, ElementInclude
+from elementtree import ElementTree
+from elementtree import ElementInclude
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/readme_util.py

@@ -1,6 +1,8 @@
-import os, logging, urllib2
-import tool_shed.util.shed_util_common as suc
+import logging
+import os
+import urllib2
 from galaxy.util import json
+import tool_shed.util.shed_util_common as suc
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/repository_dependency_util.py

@@ -1,14 +1,20 @@
-import os, logging
-from galaxy.util import json
-import tool_shed.util.shed_util_common as suc
-from tool_shed.util import common_install_util, encoding_util, metadata_util, tool_util
-from galaxy.webapps.tool_shed.util import container_util
-
+import logging
+import os
 from galaxy import eggs
+from galaxy.util import json
+from galaxy.webapps.tool_shed.util import container_util
+import tool_shed.util.shed_util_common as suc
+from tool_shed.util import common_install_util
+from tool_shed.util import encoding_util
+from tool_shed.util import metadata_util
+from tool_shed.util import tool_util
+
 import pkg_resources
 
 pkg_resources.require( 'mercurial' )
-from mercurial import hg, ui, commands
+from mercurial import commands
+from mercurial import hg
+from mercurial import ui
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/review_util.py

@@ -1,13 +1,16 @@
-import os, logging
+import logging
+import os
+from galaxy import eggs
+from galaxy.model.orm import and_
 from galaxy.util.odict import odict
 import tool_shed.util.shed_util_common as suc
-from galaxy.model.orm import and_
 
-from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( 'mercurial' )
-from mercurial import hg, ui, commands
+from mercurial import commands
+from mercurial import hg
+from mercurial import ui
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/shed_util_common.py

@@ -1,6 +1,12 @@
-import os, shutil, tempfile, logging, string, urllib2
+import logging
+import os
+import shutil
+import string
+import tempfile
+import urllib2
 from datetime import datetime
-from time import gmtime, strftime
+from time import gmtime
+from time import strftime
 from galaxy import util
 from galaxy.util import json
 from galaxy.web import url_for
@@ -16,8 +22,10 @@ pkg_resources.require( 'mercurial' )
 from mercurial import hg, ui, commands
 
 pkg_resources.require( 'elementtree' )
-from elementtree import ElementTree, ElementInclude
-from elementtree.ElementTree import Element, SubElement
+from elementtree import ElementTree
+from elementtree import ElementInclude
+from elementtree.ElementTree import Element
+from elementtree.ElementTree import SubElement
 
 eggs.require( 'markupsafe' )
 import markupsafe
@@ -522,7 +530,7 @@ def get_next_downloadable_changeset_revision( repository, repo, after_changeset_
             if changeset_revision in changeset_revisions:
                 return changeset_revision
         elif not found_after_changeset_revision and changeset_revision == after_changeset_revision:
-            # We've found the changeset in the changelog for which we need to get the next downloadable changset.
+            # We've found the changeset in the changelog for which we need to get the next downloadable changeset.
             found_after_changeset_revision = True
     return None
 
@@ -977,6 +985,17 @@ def handle_galaxy_url( trans, **kwd ):
         galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' )
     return galaxy_url
 
+def have_shed_tool_conf_for_install( trans ):
+    if not trans.app.toolbox.shed_tool_confs:
+        return False
+    migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( trans.app.config.migrated_tools_config )
+    for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs:
+        shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ]
+        shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf )
+        if shed_tool_conf_name != migrated_tools_conf_name:
+            return True
+    return False
+
 def open_repository_files_folder( trans, folder_path ):
     """Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy."""
     try:
@@ -1064,6 +1083,20 @@ def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revi
     """Return a reversed list of changesets in the repository changelog up to and including the included_upper_bounds_changeset_revision."""
     return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision )
 
+def set_repository_attributes( trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ):
+    if remove_from_disk:
+        relative_install_dir = repository.repo_path( trans.app )
+        if relative_install_dir:
+            clone_dir = os.path.abspath( relative_install_dir )
+            shutil.rmtree( clone_dir )
+            log.debug( "Removed repository installation directory: %s" % str( clone_dir ) )
+    repository.error_message = error_message
+    repository.status = status
+    repository.deleted = deleted
+    repository.uninstalled = uninstalled
+    trans.sa_session.add( repository )
+    trans.sa_session.flush()
+
 def strip_path( fpath ):
     """Attempt to strip the path from a file name."""
     if not fpath:

lib/tool_shed/util/tool_dependency_util.py

@@ -1,15 +1,14 @@
 import logging
 import os
 import shutil
-
 from galaxy import eggs
-eggs.require( 'elementtree' )
-from elementtree import ElementTree, ElementInclude
-
+from galaxy import util
 from galaxy.model.orm import and_
 import tool_shed.util.shed_util_common as suc
 
-from galaxy import util
+eggs.require( 'elementtree' )
+from elementtree import ElementTree
+from elementtree import ElementInclude
 
 log = logging.getLogger( __name__ )
 

lib/tool_shed/util/tool_util.py

@@ -1,22 +1,31 @@
-import filecmp, logging, os, shutil, tempfile
-import tool_shed.util.shed_util_common as suc
-from galaxy import util
+import filecmp
+import logging
+import os
+import shutil
+import tempfile
 import galaxy.tools
-from galaxy.tools.search import ToolBoxSearch
-from galaxy.model.orm import and_
+from galaxy import eggs
+from galaxy import util
 from galaxy.datatypes import checkers
+from galaxy.model.orm import and_
 from galaxy.tools import parameters
 from galaxy.tools.parameters import dynamic_options
+from galaxy.tools.search import ToolBoxSearch
+from galaxy.web.form_builder import SelectField
+import tool_shed.util.shed_util_common as suc
 
-from galaxy import eggs
 import pkg_resources
 
 pkg_resources.require( 'mercurial' )
-from mercurial import hg, ui, commands
+from mercurial import commands
+from mercurial import hg
+from mercurial import ui
 
 pkg_resources.require( 'elementtree' )
-from elementtree import ElementTree, ElementInclude
-from elementtree.ElementTree import Element, SubElement
+from elementtree import ElementTree
+from elementtree import ElementInclude
+from elementtree.ElementTree import Element
+from elementtree.ElementTree import SubElement
 
 log = logging.getLogger( __name__ )
 
@@ -79,6 +88,33 @@ def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_rev
         app.toolbox.write_integrated_tool_panel_config_file()
     app.toolbox_search = ToolBoxSearch( app.toolbox )
 
+def build_shed_tool_conf_select_field( trans ):
+    """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs."""
+    options = []
+    for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs:
+        shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ]
+        if shed_tool_conf_filename != trans.app.config.migrated_tools_config:
+            if shed_tool_conf_filename.startswith( './' ):
+                option_label = shed_tool_conf_filename.replace( './', '', 1 )
+            else:
+                option_label = shed_tool_conf_filename
+            options.append( ( option_label, shed_tool_conf_filename ) )
+    select_field = SelectField( name='shed_tool_conf' )
+    for option_tup in options:
+        select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] )
+    return select_field
+
+def build_tool_panel_section_select_field( trans ):
+    """Build a SelectField whose options are the sections of the current in-memory toolbox."""
+    options = []
+    for k, v in trans.app.toolbox.tool_panel.items():
+        if isinstance( v, galaxy.tools.ToolSection ):
+            options.append( ( v.name, v.id ) )
+    select_field = SelectField( name='tool_panel_section', display='radio' )
+    for option_tup in options:
+        select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] )
+    return select_field
+
 def can_use_tool_config_disk_file( trans, repository, repo, file_path, changeset_revision ):
     """
     Determine if repository's tool config file on disk can be used.  This method is restricted to tool config files since, with the

lib/tool_shed/util/workflow_util.py

@@ -1,19 +1,25 @@
+import logging
+import galaxy.tools
+import galaxy.tools.parameters
+import galaxy.webapps.galaxy.controllers.workflow
 from galaxy import eggs
+from galaxy.util import json
+from galaxy.workflow.modules import InputDataModule
+from galaxy.workflow.modules import ToolModule
+from galaxy.workflow.modules import WorkflowModuleFactory
+import tool_shed.util.shed_util_common as suc
+from tool_shed.util import encoding_util
+from tool_shed.util import metadata_util
+from tool_shed.util import tool_util
+
 import pkg_resources
 
 pkg_resources.require( "SVGFig" )
-
-import logging, svgfig
-from galaxy.util import json
-import tool_shed.util.shed_util_common as suc
-from tool_shed.util import encoding_util, metadata_util, tool_util
-from galaxy.workflow.modules import InputDataModule, ToolModule, WorkflowModuleFactory
-import galaxy.webapps.galaxy.controllers.workflow
-import galaxy.tools
-import galaxy.tools.parameters
+import svgfig
 
 log = logging.getLogger( __name__ )
 
+
 class RepoInputDataModule( InputDataModule ):
 
     type = "data_input"
@@ -24,12 +30,14 @@ class RepoInputDataModule( InputDataModule ):
         module = Class( trans )
         module.state = dict( name="Input Dataset" )
         return module
+
     @classmethod
     def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata=None, secure=True ):
         module = Class( trans )
         state = json.from_json_string( step_dict[ "tool_state" ] )
         module.state = dict( name=state.get( "name", "Input Dataset" ) )
         return module
+
     @classmethod
     def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ):
         module = Class( trans )
@@ -38,6 +46,7 @@ class RepoInputDataModule( InputDataModule ):
             module.state[ 'name' ] = step.tool_inputs[ 'name' ]
         return module
 
+
 class RepoToolModule( ToolModule ):
     
     type = "tool"
@@ -64,11 +73,13 @@ class RepoToolModule( ToolModule ):
         self.post_job_actions = {}
         self.workflow_outputs = []
         self.state = None
+
     @classmethod
     def new( Class, trans, repository_id, changeset_revision, tools_metadata, tool_id=None ):
         module = Class( trans, repository_id, changeset_revision, tools_metadata, tool_id )
         module.state = module.tool.new_state( trans, all_pages=True )
         return module
+
     @classmethod
     def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata, secure=True ):
         tool_id = step_dict[ 'tool_id' ]
@@ -78,6 +89,7 @@ class RepoToolModule( ToolModule ):
             module.state.decode( step_dict[ "tool_state" ], module.tool, module.trans.app, secure=secure )
         module.errors = step_dict.get( "tool_errors", None )
         return module
+
     @classmethod
     def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ):
         module = Class( trans, repository_id, changeset_revision, tools_metadata, step.tool_id )
@@ -88,6 +100,7 @@ class RepoToolModule( ToolModule ):
             module.state.inputs = {}
         module.errors = step.tool_errors
         return module
+
     def get_data_inputs( self ):
         data_inputs = []
         def callback( input, value, prefixed_name, prefixed_label ):
@@ -98,6 +111,7 @@ class RepoToolModule( ToolModule ):
         if self.tool:
             galaxy.tools.parameters.visit_input_values( self.tool.inputs, self.state.inputs, callback )
         return data_inputs
+
     def get_data_outputs( self ):
         data_outputs = []
         if self.tool:
@@ -124,18 +138,23 @@ class RepoToolModule( ToolModule ):
                 data_outputs.append( dict( name=name, extensions=formats ) )
         return data_outputs
 
+
 class RepoWorkflowModuleFactory( WorkflowModuleFactory ):
+
     def __init__( self, module_types ):
         self.module_types = module_types
+
     def new( self, trans, type, tools_metadata=None, tool_id=None ):
         """Return module for type and (optional) tool_id initialized with new / default state."""
         assert type in self.module_types
         return self.module_types[type].new( trans, tool_id )
+
     def from_dict( self, trans, repository_id, changeset_revision, step_dict, **kwd ):
         """Return module initialized from the data in dictionary `step_dict`."""
         type = step_dict[ 'type' ]
         assert type in self.module_types
         return self.module_types[ type ].from_dict( trans, repository_id, changeset_revision, step_dict, **kwd )
+
     def from_workflow_step( self, trans, repository_id, changeset_revision, tools_metadata, step ):
         """Return module initialized from the WorkflowStep object `step`."""
         type = step.type
@@ -292,6 +311,7 @@ def generate_workflow_image( trans, workflow_name, repository_metadata_id=None,
     canvas[ 'viewBox' ] = "0 0 %s %s" % ( width, height )
     trans.response.set_content_type( "image/svg+xml" )
     return canvas.standalone_xml()
+
 def get_workflow_data_inputs( step, module ):
     if module.type == 'tool':
         if module.tool:
@@ -306,6 +326,7 @@ def get_workflow_data_inputs( step, module ):
                 data_inputs.append( data_inputs_dict )
             return data_inputs
     return module.get_data_inputs()
+
 def get_workflow_data_outputs( step, module, steps ):
     if module.type == 'tool':
         if module.tool:
@@ -329,6 +350,7 @@ def get_workflow_data_outputs( step, module, steps ):
             data_outputs.append( data_outputs_dict )       
             return data_outputs
     return module.get_data_outputs()
+
 def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, changeset_revision ):
     """
     Return an in-memory Workflow object from the dictionary object created when it was exported.  This method is called from
@@ -383,6 +405,7 @@ def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id,
     galaxy.webapps.galaxy.controllers.workflow.attach_ordered_steps( workflow, steps )
     # Return the in-memory Workflow object for display or later persistence to the Galaxy database.
     return workflow, missing_tool_tups
+
 def get_workflow_module_name( module, missing_tool_tups ):
     module_name = module.get_name()
     if module.type == 'tool' and module_name == 'unavailable':
@@ -392,6 +415,7 @@ def get_workflow_module_name( module, missing_tool_tups ):
                 module_name = '%s' % missing_tool_name
                 break
     return module_name
+
 def save_workflow( trans, workflow ):
     """Use the received in-memory Workflow object for saving to the Galaxy database."""
     stored = trans.model.StoredWorkflow()

scripts/api/common.py

@@ -1,4 +1,7 @@
-import os, sys, urllib, urllib2
+import logging
+import os
+import sys
+import urllib2
 
 new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', 'lib' ) ]
 new_path.extend( sys.path[1:] )
@@ -15,6 +18,8 @@ from Crypto.Cipher import Blowfish
 from Crypto.Util.randpool import RandomPool
 from Crypto.Util import number
 
+log = logging.getLogger( __name__ )
+
 def make_url( api_key, url, args=None ):
     # Adds the API Key to the URL if it's not already there.
     if args is None:

scripts/api/install_repository_tools_into_existing_tool_panel_section.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Install a specified repository revision from a specified tool shed into Galaxy.  This example demonstrates installation of a repository that contains
+valid tools, loading them into an existing section of the Galaxy tool panel.  The repository has no tool dependencies or repository dependencies, so only
+a single repository will be installed.
+
+This example requires a tool panel config file (e.g., tool_conf.xml, shed_tool_conf.xml, etc) to contain a tool panel section like the following:
+
+<section id="from_test_tool_shed" name="From Test Tool Shed" version="">
+</section>
+
+usage: ./install_repository_tools_into_existing_tool_panel_section.py <api_key <galaxy base url> tool_shed_url name owner changeset_revision tool_panel_section_id
+
+Here is a working example of how to use this script to install a repository from the test tool shed.
+./install_repository_tools_into_existing_tool_panel_section.py <api key> <galaxy base url>/api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb from_test_tool_shed
+"""
+
+import os
+import sys
+sys.path.insert( 0, os.path.dirname( __file__ ) )
+from common import submit
+
+try:
+    assert sys.argv[ 7 ]
+except IndexError:
+    print 'usage: %s key url tool_shed_url name owner changeset_revision tool_panel_section_id' % os.path.basename( sys.argv[ 0 ] )
+    sys.exit( 1 )
+
+try:
+    data = {}
+    data[ 'tool_shed_url' ] = sys.argv[ 3 ]
+    data[ 'name' ] = sys.argv[ 4 ]
+    data[ 'owner' ] = sys.argv[ 5 ]
+    data[ 'changeset_revision' ] = sys.argv[ 6 ]
+    data[ 'tool_panel_section_id' ] = sys.argv[ 7 ]
+except IndexError:
+    print 'usage: %s key url tool_shed_url name owner changeset_revision tool_panel_section_id' % os.path.basename( sys.argv[ 0 ] )
+    sys.exit( 1 )
+
+submit( sys.argv[ 1 ], sys.argv[ 2 ], data )

scripts/api/install_repository_tools_into_new_tool_panel_section.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""
+Install a specified repository revision from a specified tool shed into Galaxy.  This example demonstrates installation of a repository that contains
+valid tools, loading them into a new section of the Galaxy tool panel.  The repository has no tool dependencies or repository dependencies, so only
+a single repository will be installed.
+
+usage: ./install_repository_tools_into_new_tool_panel_section.py <api_key <galaxy base url> tool_shed_url name owner changeset_revision new_tool_panel_section_label
+
+Here is a working example of how to use this script to install a repository from the test tool shed.
+./install_repository_tools_into_new_tool_panel_section.py <api key> <galaxy base url>/api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb 'From Test Tool Shed'
+"""
+
+import os
+import sys
+sys.path.insert( 0, os.path.dirname( __file__ ) )
+from common import submit
+
+try:
+    assert sys.argv[ 7 ]
+except IndexError:
+    print 'usage: %s key url tool_shed_url name owner changeset_revision tool_panel_section_id' % os.path.basename( sys.argv[ 0 ] )
+    sys.exit( 1 )
+
+try:
+    data = {}
+    data[ 'tool_shed_url' ] = sys.argv[ 3 ]
+    data[ 'name' ] = sys.argv[ 4 ]
+    data[ 'owner' ] = sys.argv[ 5 ]
+    data[ 'changeset_revision' ] = sys.argv[ 6 ]
+    data[ 'new_tool_panel_section_label' ] = sys.argv[ 7 ]
+except IndexError:
+    print 'usage: %s key url tool_shed_url name owner changeset_revision new_tool_panel_section_label' % os.path.basename( sys.argv[ 0 ] )
+    sys.exit( 1 )
+
+submit( sys.argv[ 1 ], sys.argv[ 2 ], data )

scripts/api/install_repository_with_repository_dependencies.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+"""
+Install a specified repository revision from a specified tool shed into Galaxy.  This example demonstrates installation of a repository that has
+repository dependencies, so multiple repositories will ultimately be installed.  Since no Galaxy tool panel section information is used, all tools
+contained in the installed repositories will be loaded into the Galaxy tool panel outside of any sections.
+
+usage: ./install_repository_with_repository_dependencies.py <api_key <galaxy base url> tool_shed_url name owner changeset_revision True
+
+Here is a working example of how to use this script to install a repository from the test tool shed.
+./install_repository_with_repository_dependencies.py <api key> <galaxy base url>/api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu emboss_5 devteam 8ddad0c9a75a True
+"""
+
+import os
+import sys
+sys.path.insert( 0, os.path.dirname( __file__ ) )
+from common import submit
+
+try:
+    assert sys.argv[ 7 ]
+except IndexError:
+    print 'usage: %s key url tool_shed_url name owner changeset_revision install_repository_dependencies' % os.path.basename( sys.argv[ 0 ] )
+    sys.exit( 1 )
+
+try:
+    data = {}
+    data[ 'tool_shed_url' ] = sys.argv[ 3 ]
+    data[ 'name' ] = sys.argv[ 4 ]
+    data[ 'owner' ] = sys.argv[ 5 ]
+    data[ 'changeset_revision' ] = sys.argv[ 6 ]
+    data[ 'install_repository_dependencies' ] = sys.argv[ 7 ]
+except IndexError:
+    print 'usage: %s key url tool_shed_url name owner changeset_revision install_repository_dependencies' % os.path.basename( sys.argv[ 0 ] )
+    sys.exit( 1 )
+
+submit( sys.argv[ 1 ], sys.argv[ 2 ], data )

scripts/api/tool_shed_repository_revision_update.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-"""
-PUT/update script to update appropriate values in a repository_metadata table record in the Tool Shed.
-
-usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... 
-"""
-
-import os, sys
-sys.path.insert( 0, os.path.dirname( __file__ ) )
-from common import update
-
-import pkg_resources
-pkg_resources.require( "simplejson" )
-
-import simplejson
-
-to_json_string = simplejson.dumps
-from_json_string = simplejson.loads
-
-data = {}
-for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]:      
-    """
-    This example script will properly handle updating the value of one or more of the following RepositoryMetadata attributes:
-    tools_functionally_correct, do_not_test, tool_test_errors
-    """
-    if key in [ 'tools_functionally_correct', 'do_not_test' ]:
-        if str( value ).lower() in [ 'true', 'yes', 'on' ]:
-            new_value = True
-        else:
-            new_value = False
-    elif key in [ 'tool_test_errors' ]:
-        new_value = from_json_string( value )
-    else:
-        new_value = str( value )
-    data[ key ] = new_value
-
-update( sys.argv[ 1 ], sys.argv[ 2 ], data )

static/scripts/viz/trackster/tracks.js

@@ -1276,7 +1276,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, {
                     view.reference_track.init();
                 }
             }
-            if (low !== undefined && high !== undefined) {
+            if (low && high) {
                 view.low = Math.max(low, 0);
                 view.high = Math.min(high, view.max_high);
             }
@@ -1289,34 +1289,49 @@ extend( TracksterView.prototype, DrawableCollection.prototype, {
             view.request_redraw();
         }
     },
+
+    /**
+     * Change viewing region to that denoted by string. General format of string is:
+     *
+     * <chrom>[ {separator}<start>[-<end>] ]
+     *
+     * where separator can be whitespace or a colon. Examples:
+     *
+     * chr22
+     * chr1:100-200
+     * chr7 89999
+     * chr8 90000 990000
+     */
     go_to: function(str) {
-        // Preprocess str to remove spaces and commas.
-        str = str.replace(/ |,/g, "");
-        
-        // Go to new location.
-        var view = this,
-            new_low, 
-            new_high,
-            chrom_pos = str.split(":"),
+        // Remove commas.
+        str = str.replace(/,/g, '');
+
+        // Replace colons and hyphens with space for easy parsing.
+        str = str.replace(/:|\-/g, ' ');
+
+        // Parse new location.
+        var chrom_pos = str.split(/\s+/),
             chrom = chrom_pos[0],
-            pos = chrom_pos[1];
-        
-        if (pos !== undefined) {
-            try {
-                var pos_split = pos.split("-");
-                new_low = parseInt(pos_split[0], 10);
-                new_high = parseInt(pos_split[1], 10);
-            } catch (e) {
-                return false;
-            }
+            new_low = (chrom_pos[1] ? parseInt(chrom_pos[1], 10) : null),
+            new_high = (chrom_pos[2] ? parseInt(chrom_pos[2], 10) : null);
+
+        // If no new high, new_low is the position of focus, so adjust low, high
+        // accordingly.
+        if (!new_high) {
+            // HACK: max resolution is 30 bases,so adjust low, high accordingly.
+            new_low = new_low - 15;
+            new_high = new_low + 15;
         }
-        view.change_chrom(chrom, new_low, new_high);
+
+        this.change_chrom(chrom, new_low, new_high);
     },
+
     move_fraction: function(fraction) {
         var view = this;
         var span = view.high - view.low;
         this.move_delta(fraction * span);
     },
+
     move_delta: function(delta_chrom) {
         // Update low, high.
         var view = this;

test/install_and_test_tool_shed_repositories/functional_tests.py

@@ -107,12 +107,21 @@ if os.path.exists( 'tool_data_table_conf.test.xml' ):
 else:
     additional_tool_data_tables = None
     additional_tool_data_path = None
-# And set up a blank tool_data_table_conf.xml and shed_tool_data_table_conf.xml.
+
+# Also set up default tool data tables.
+if os.path.exists( 'tool_data_table_conf.xml' ):
+    tool_data_table_conf = 'tool_data_table_conf.xml'
+elif os.path.exists( 'tool_data_table_conf.xml.sample' ):
+    tool_data_table_conf = 'tool_data_table_conf.xml.sample'
+else:
+    tool_data_table_conf = None
+
+# And set up a blank shed_tool_data_table_conf.xml.
 tool_data_table_conf_xml_template = '''<?xml version="1.0"?>
 <tables>
 </tables>
 '''
-
+    
 # The tool shed url and api key must be set for this script to work correctly. Additionally, if the tool shed url does not
 # point to one of the defaults, the GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF needs to point to a tool sheds configuration file
 # that contains a definition for that tool shed.
@@ -232,8 +241,7 @@ def get_repositories_to_install( location, source='file', format='json' ):
         listing = file( location, 'r' ).read()
     elif source == 'url':
         assert tool_shed_api_key is not None, 'Cannot proceed without tool shed API key.'
-        params = urllib.urlencode( dict( tools_functionally_correct='false', 
-                                         do_not_test='false', 
+        params = urllib.urlencode( dict( do_not_test='false', 
                                          downloadable='true', 
                                          malicious='false',
                                          includes_tools='true' ) )
@@ -272,10 +280,15 @@ def json_from_url( url ):
     return from_json_string( url_contents )
 
 def register_test_result( url, metadata_id, test_results_dict, tests_passed=False ):
+    '''
+    Set do_not_test = True if the repository fails functional tests. Set do_not_test = False
+    if the repository passes functional tests, so that the repository will always be re-tested
+    against the most recent code.
+    '''
     params = {}
     if tests_passed:
         params[ 'tools_functionally_correct' ] = 'true'
-        params[ 'do_not_test' ] = 'true'
+        params[ 'do_not_test' ] = 'false'
     else:
         params[ 'tools_functionally_correct' ] = 'false'
         params[ 'do_not_test' ] = 'true'
@@ -328,7 +341,7 @@ def main():
     galaxy_test_proxy_port = None
     # Set up the configuration files for the Galaxy instance.
     shed_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_data_table_conf.xml' ) )
-    galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_data_table_conf.xml' ) )
+    galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', tool_data_table_conf )
     galaxy_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_conf.xml' ) )
     galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_conf.xml' ) )
     galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml' ) )
@@ -350,7 +363,7 @@ def main():
     galaxy_file_path = os.path.join( galaxy_db_path, 'files' )
     new_repos_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir )
     galaxy_tempfiles = tempfile.mkdtemp( dir=galaxy_test_tmp_dir )
-    galaxy_shed_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) 
+    galaxy_shed_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir, prefix='shed_tools' ) 
     galaxy_migrated_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) 
     galaxy_tool_dependency_dir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) 
     os.environ[ 'GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR' ] = galaxy_tool_dependency_dir
@@ -367,9 +380,6 @@ def main():
 
     print "Database connection: ", database_connection
 
-    # Generate the tool_data_table_conf.xml file.
-    file( galaxy_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
-    os.environ[ 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF' ] = galaxy_tool_data_table_conf_file
     # Generate the shed_tool_data_table_conf.xml file.
     file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template )
     os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file
@@ -728,7 +738,7 @@ def main():
                                    ( changeset_revision, name ) ) 
                     # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the
                     # repository using Twill.
-                    execute_uninstall_method( repository_info_dict )
+                    success = execute_uninstall_method( repository_info_dict )
                     # Set the test_toolbox.toolbox module-level variable to the new app.toolbox.
                     test_toolbox.toolbox = app.toolbox
             else:
@@ -776,11 +786,20 @@ def main():
             print "# %d repositories failed:" % len( repositories_failed )
             show_summary_output( repositories_failed )
         if repositories_failed_install:
+            # Set success to False so that the return code will not be 0.
+            success = False
             print '# ----------------------------------------------------------------------------------'
             print "# %d repositories not installed correctly:" % len( repositories_failed_install )
             show_summary_output( repositories_failed_install )
+        else:
+            success = True
+    else:
+        success = True
     print "####################################################################################"
-    
+    # Normally, the value of 'success' would determine whether this test suite is marked as passed or failed
+    # in the automated buildbot framework. However, due to the procedure used here, we only want to report
+    # failure if a repository fails to install correctly. Therefore, we have overriden the value of 'success'
+    # here based on what actions the script has executed. 
     if success:
         return 0
     else:
@@ -791,5 +810,4 @@ if __name__ == "__main__":
     print "####################################################################################"
     print "# %s - running repository installation and testing script." % now
     print "####################################################################################"
-    return_code = main()
-    sys.exit( return_code )
+    sys.exit( main() )

test/tool_shed/base/twilltestcase.py

@@ -454,9 +454,10 @@ class ShedTwillTestCase( TwillTestCase ):
         
     def fill_review_form( self, review_contents_dict, strings_displayed=[], strings_not_displayed=[] ):
         kwd = dict()
+        changed = False
         for label, contents in review_contents_dict.items():
-            strings_displayed.append( label )
             if contents:
+                changed = True
                 kwd[ '%s__ESEP__comment' % label ] = contents[ 'comment' ]
                 kwd[ '%s__ESEP__rating' % label ] = contents[ 'rating' ]
                 if 'private' in contents:
@@ -464,8 +465,10 @@ class ShedTwillTestCase( TwillTestCase ):
                 kwd[ '%s__ESEP__approved' % label ] = contents[ 'approved' ]
             else:
                 kwd[ '%s__ESEP__approved' % label ] = 'not_applicable'
+        self.check_for_strings( strings_displayed, strings_not_displayed )
         self.submit_form( 1, 'Workflows__ESEP__review_button', **kwd )
-        strings_displayed.append( 'Reviews were saved' )
+        if changed:
+            strings_displayed.append( 'Reviews were saved' )
         self.check_for_strings( strings_displayed, strings_not_displayed )
         
     def galaxy_login( self, email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='' ):