Add enhancements to the Galaxy repository install process

to cleanly handle the case described here:
https://github.com/galaxyproject/galaxy/issues/667.  Specifically, the
following scenario is now cleanly handled:

Example use case: In a toolshed I have a repository of a tool, say

package_new_gene_db_1_2_3, and in revision 1 it depends on
package_sqlite_1_0_0:

package_new_gene_db_1_2_3
package_sqlite_1_0_0

At some point in time I discover that postgres is a better solution for
my gene database, and I upload revision 2:

package_new_gene_db_1_2_3
package_postgres__2_0_0

And I remove the dependecy of sqlite.  If I go to my galaxy instance and
I remove revision 1 and install revision 2, the installation is cleanly
handled with theis PR.

It should be noted that this enhancement could adversely impact
reproducibility if best practices are not followed with regard to
defining package dependencies.  Since it is now cleanly possible to
eliminate dependencies over time, best practices must be followed to
ensure that elimination of a dependency does not affect the output of a
tool that uses the underlying hierarchy of packages.

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -1296,6 +1296,21 @@ class AdminToolshed( AdminGalaxy ):
                 # Just in case the tool_section.id differs from tool_panel_section_id, which it shouldn't...
                 tool_panel_section_id = str( tool_section.id )
         if tool_shed_repository.status == trans.install_model.ToolShedRepository.installation_status.UNINSTALLED:
+            repository_type = suc.get_repository_type_from_tool_shed(trans.app,
+                                                                     tool_shed_url,
+                                                                     tool_shed_repository.name,
+                                                                     tool_shed_repository.owner)
+            if repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION:
+                # Repositories of type tool_dependency_definition must get the latest
+                # metadata from the Tool Shed since they have only a single installable
+                # revision.
+                raw_text = suc.get_tool_dependency_definition_metadata_from_tool_shed(trans.app,
+                                                                                      tool_shed_url,
+                                                                                      tool_shed_repository.name,
+                                                                                      tool_shed_repository.owner)
+                new_meta = json.loads(raw_text)
+                # Clean up old repository dependency and tool dependency relationships.
+                suc.clean_dependency_relationships(trans, new_meta, tool_shed_repository, tool_shed_url)
             # The repository's status must be updated from 'Uninstalled' to 'New' when initiating reinstall
             # so the repository_installation_updater will function.
             tool_shed_repository = suc.create_or_update_tool_shed_repository( trans.app,

lib/galaxy/webapps/tool_shed/controllers/repository.py

@@ -1684,6 +1684,14 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
                      includes_tool_dependencies=includes_tool_dependencies,
                      repo_info_dicts=repo_info_dicts )
 
+    @web.expose
+    def get_repository_type( self, trans, **kwd ):
+        """Given a repository name and owner, return the type."""
+        repository_name = kwd[ 'name' ]
+        repository_owner = kwd[ 'owner' ]
+        repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner )
+        return str( repository.type )
+
     @web.json
     def get_required_repo_info_dict( self, trans, encoded_str=None ):
         """
@@ -1751,6 +1759,22 @@ class RepositoryController( BaseUIController, ratings_util.ItemRatings ):
             return contents
         return ''
 
+    @web.json
+    def get_tool_dependency_definition_metadata( self, trans, **kwd ):
+        """
+        Given a repository name and ownerof a repository whose type is
+        tool_dependency_definition, return the current metadata.
+        """
+        repository_name = kwd[ 'name' ]
+        repository_owner = kwd[ 'owner' ]
+        repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner )
+        encoded_id = trans.app.security.encode_id( repository.id )
+        repository_tip = repository.tip( trans.app )
+        repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans.app,
+                                                                                 encoded_id,
+                                                                                 repository_tip )
+        return repository_metadata.metadata
+
     @web.expose
     def get_tool_versions( self, trans, **kwd ):
         """

lib/tool_shed/util/shed_util_common.py

@@ -83,6 +83,59 @@ This message was sent from the Galaxy Tool Shed instance hosted on the server
 """
 
 
+def can_eliminate_repository_dependency(metadata_dict, tool_shed_url, name, owner):
+    """
+    Determine if the relationship between a repository_dependency record
+    associated with a tool_shed_repository record on the Galaxy side
+    can be eliminated.
+    """
+    rd_dict = metadata_dict.get('repository_dependencies', {})
+    rd_tups = rd_dict.get( 'repository_dependencies', [] )
+    for rd_tup in rd_tups:
+        tsu, n, o, none1, none2, none3 = common_util.parse_repository_dependency_tuple(rd_tup)
+        if tsu == tool_shed_url and n == name and o == owner:
+            # The repository dependency is current, so keep it.
+            return False
+    return True
+
+
+def can_eliminate_tool_dependency(metadata_dict, name, type, version):
+    """
+    Determine if the relationship between a tool_dependency record
+    associated with a tool_shed_repository record on the Galaxy side
+    can be eliminated.
+    """
+    td_dict = metadata_dict.get('tool_dependencies', {})
+    for td_key, td_val in td_dict.items():
+        n = td_val.get('name', None)
+        t = td_val.get('type', None)
+        v = td_val.get('version', None)
+        if n == name and t == type and v == version:
+            # The tool dependency is current, so keep it.
+            return False
+    return True
+
+
+def clean_dependency_relationships(trans, metadata_dict, tool_shed_repository, tool_shed_url):
+    """
+    Repositories of type tool_dependency_definition allow for defining a
+    package dependency at some point in the change log and then removing the
+    dependency later in the change log.  This function keeps the dependency
+    relationships on the Galaxy side current by deleting database records
+    that defined the now-broken relationships.
+    """
+    for rrda in tool_shed_repository.required_repositories:
+        rd = rrda.repository_dependency
+        r = rd.repository
+        if can_eliminate_repository_dependency(metadata_dict, tool_shed_url, r.name, r.owner):
+            trans.install_model.context.delete(rrda)
+            trans.install_model.context.flush()
+    for td in tool_shed_repository.tool_dependencies:
+        if can_eliminate_tool_dependency(metadata_dict, td.name, td.type, td.version):
+            trans.install_model.context.delete(td)
+            trans.install_model.context.flush()
+
+
 def create_or_update_tool_shed_repository( app, name, description, installed_changeset_revision, ctx_rev, repository_clone_url,
                                            metadata_dict, status, current_changeset_revision=None, owner='', dist_to_shed=False ):
     """
@@ -300,6 +353,19 @@ def get_latest_downloadable_changeset_revision( app, repository, repo ):
     return hg_util.INITIAL_CHANGELOG_HASH
 
 
+def get_tool_dependency_definition_metadata_from_tool_shed( app, tool_shed_url, name, owner ):
+    """
+    Send a request to the tool shed to retrieve the current metadata for a
+    repository of type tool_dependency_definition defined by the combination
+    of a name and owner.
+    """
+    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
+    params = dict( name=name, owner=owner )
+    pathspec = [ 'repository', 'get_tool_dependency_definition_metadata' ]
+    metadata = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+    return metadata
+
+
 def get_next_downloadable_changeset_revision( repository, repo, after_changeset_revision ):
     """
     Return the installable changeset_revision in the repository changelog after the changeset to which
@@ -708,6 +774,18 @@ def get_repository_query( app ):
     return query
 
 
+def get_repository_type_from_tool_shed( app, tool_shed_url, name, owner ):
+    """
+    Send a request to the tool shed to retrieve the type for a repository defined by the
+    combination of a name and owner.
+    """
+    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
+    params = dict( name=name, owner=owner )
+    pathspec = [ 'repository', 'get_repository_type' ]
+    repository_type = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
+    return repository_type
+
+
 def get_tool_panel_config_tool_path_install_dir( app, repository ):
     """
     Return shed-related tool panel config, the tool_path configured in it, and the relative path to