From 94edc94b73e047612ae51ca2e075f17ac8e238bb Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 28 Oct 2016 11:48:17 +0200
Subject: [PATCH 01/13] Add cached dependency manager

Similar to https://github.com/galaxyproject/galaxy/pull/2986, implement
a mechanism that allows tool dependencies to be cached.

If the `use_cached_dependency_manager` option is set to True in
galaxy.ini, we build a hash of the combination of a tools' requirements,
and store the resulting environment in a directory specified by the
`tool_dependency_cache_dir` option in galaxy.ini.
---
 config/galaxy.ini.sample                 | 13 +++++
 lib/galaxy/config.py                     |  2 +
 lib/galaxy/tools/__init__.py             | 11 ++--
 lib/galaxy/tools/deps/__init__.py        | 66 +++++++++++++++++++++++-
 lib/galaxy/tools/deps/resolvers/conda.py | 12 +++--
 5 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index 88799042671..24398025f1f 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -217,6 +217,19 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # of extra disk space usage and extra time spent copying packages.
 #conda_copy_dependencies = False
 
+# Certain dependency resolvers (namely conda) take a considerable amount of
+# time to build an isolated job environment in the job_working_directory.  Set
+# the following option to true to cache the dependencies in a folder. This
+# option is beta and should only be used if you experience long waiting times
+# before a job is actually submitted to your cluster. If you activate this
+# option, and you install new dependencies you may need to clear out old cached
+# environments
+#use_cached_dependency_manager = False
+
+# By default the tool_dependency_cache_dir is the _cache directory
+# of the tool dependency directory
+#tool_dependency_cache_dir = tool_dependency_dir/_cache
+
 # File containing the Galaxy Tool Sheds that should be made available to
 # install from in the admin interface (.sample used if default does not exist).
 #tool_sheds_config_file = config/tool_sheds_conf.xml
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index be422486e5d..49f49bd3c36 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -325,6 +325,8 @@ class Configuration( object ):
         else:
             self.tool_dependency_dir = None
             self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
+        self.use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False'))
+        self.tool_dependency_cache_dir = kwargs.get( 'tool_dependency_cache_dir', os.path.join(self.tool_dependency_dir, '_cache'))
 
         self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) )
         containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None )
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index f11d1d5d5c3..04f4117dced 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1306,16 +1306,17 @@ class Tool( object, Dictifiable ):
         return messages
 
     def build_dependency_shell_commands( self, job_directory=None, metadata=False ):
-        """Return a list of commands to be run to populate the current environment to include this tools requirements."""
-        requirements_to_dependencies = self.app.toolbox.dependency_manager.requirements_to_dependencies(
-            self.requirements,
+        """
+        Return a list of commands to be run to populate the current environment to include this tools requirements.
+        """
+        return self.app.toolbox.dependency_manager.dependency_shell_commands(
+            requirements=self.requirements,
             installed_tool_dependencies=self.installed_tool_dependencies,
             tool_dir=self.tool_dir,
             job_directory=job_directory,
             metadata=metadata,
+            tool_instance=self
         )
-        self.dependencies = [dep.to_dict() for dep in requirements_to_dependencies.values()]
-        return [dep.shell_commands(req) for req, dep in requirements_to_dependencies.items()]
 
     @property
     def installed_tool_dependencies(self):
diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 4ec3e42a1a4..85937d5b6da 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -2,12 +2,16 @@
 Dependency management for tools.
 """
 
+import json
 import logging
 import os.path
 
 from collections import OrderedDict
 
-from galaxy.util import plugin_config
+from galaxy.util import (
+    hash_util,
+    plugin_config
+)
 
 from .resolvers import NullDependency
 from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
@@ -45,7 +49,11 @@ def build_dependency_manager( config ):
             if value is CONFIG_VAL_NOT_FOUND:
                 value = default_value
             dependency_manager_kwds[key] = value
-        dependency_manager = DependencyManager( **dependency_manager_kwds )
+        if config.use_cached_dependency_manager:
+            dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir
+            dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
+        else:
+            dependency_manager = DependencyManager( **dependency_manager_kwds )
     else:
         dependency_manager = NullDependencyManager()
 
@@ -109,6 +117,8 @@ class DependencyManager( object ):
                 log.debug(dependency.resolver_msg)
                 if dependency.dependency_type:
                     requirement_to_dependency[requirement] = dependency
+        if 'tool_instance' in kwds:
+            kwds['tool_instance'].dependencies = [dep.to_dict() for dep in requirement_to_dependency.values()]
         return requirement_to_dependency
 
     def uses_tool_shed_dependencies(self):
@@ -155,3 +165,55 @@ class DependencyManager( object ):
     def __resolvers_dict( self ):
         import galaxy.tools.deps.resolvers
         return plugin_config.plugins_dict( galaxy.tools.deps.resolvers, 'resolver_type' )
+
+
+class CachedDependencyManager(DependencyManager):
+    def __init__(self, default_base_path, conf_file=None, **extra_config):
+        super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
+
+    def dependency_shell_commands( self, requirements, **kwds ):
+        commands = self.get_cached_commands(requirements, **kwds)
+        if not commands:
+            hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
+            kwds['conda_env'] = hashed_requirements_dir
+            requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds)
+            commands = [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()]
+            if not os.path.exists(hashed_requirements_dir):
+                # conda will create the hashed_requirements_dir, and fail if it already exists,
+                # while other resolvers may not create the hashed_requirements_dir
+                os.mkdir(hashed_requirements_dir)
+            with open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'), 'w') as cmds_f:
+                [cmds_f.write("%s\n" % line) for line in commands]
+            with open(os.path.join(hashed_requirements_dir, 'packages.json'), 'w') as packages:
+                # Keep a list of dependencies, so that on re-use we can log the resolved packages
+                packages.write(json.dumps([dep.to_dict() for dep in requirement_to_dependency.values()], sort_keys=True))
+        return commands
+
+    def hash_requirements(self, requirements):
+        """Return hash for requirements"""
+        r_string = ':'.join(['_'.join((r.type, r.name, str(r.version))) for r in requirements])
+        return hash_util.new_secure_hash(r_string)
+
+    def get_hashed_requirements_path(self, requirements):
+        """
+        Returns the path to the hashed requirements directory (but does not evaluate whether the path exists)
+        :param requirements:
+        :return:
+        """
+        req_hashes = self.hash_requirements(requirements)
+        return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes)
+
+    def get_cached_commands(self, requirements, **kwargs):
+        """
+        Return commands for activating cached env if it exists
+        :param requirements_hash:
+        :return: list of commands
+        """
+        hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
+        if not os.path.exists(os.path.join(hashed_requirements_dir)):
+            return []
+        else:
+            if 'tool_instance' in kwargs:
+                dependencies = json.load(open(os.path.join(hashed_requirements_dir, 'packages.json')))
+                kwargs['tool_instance'].dependencies = dependencies
+            return [line.strip() for line in open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'))]
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index 2a1c13f5116..a45cf717952 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -136,11 +136,15 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
 
         # Have installed conda_target and job_directory to send it to.
         # If dependency is for metadata generation, store environment in conda-metadata-env
-        if kwds.get("metadata", False):
-            conda_env = "conda-metadata-env"
+
+        if kwds.get('conda_env', False):
+            conda_environment = kwds.get('conda_env')
         else:
-            conda_env = "conda-env"
-        conda_environment = os.path.join(job_directory, conda_env)
+            if kwds.get("metadata", False):
+                conda_env = "conda-metadata-env"
+            else:
+                conda_env = "conda-env"
+            conda_environment = os.path.join(job_directory, conda_env)
         env_path, exit_code = build_isolated_environment(
             conda_target,
             path=conda_environment,

From 0a9b6d8755dee530232ab893a3bad199407aa0bc Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 28 Oct 2016 18:22:36 +0200
Subject: [PATCH 02/13] Better phrasing of cache options in galaxy.ini.sample
 (Thanks @nsoranzo)

---
 config/galaxy.ini.sample | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index 24398025f1f..af4799c7f70 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -217,18 +217,18 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # of extra disk space usage and extra time spent copying packages.
 #conda_copy_dependencies = False
 
-# Certain dependency resolvers (namely conda) take a considerable amount of
-# time to build an isolated job environment in the job_working_directory.  Set
-# the following option to true to cache the dependencies in a folder. This
-# option is beta and should only be used if you experience long waiting times
-# before a job is actually submitted to your cluster. If you activate this
-# option, and you install new dependencies you may need to clear out old cached
-# environments
+# Certain dependency resolvers (namely Conda) take a considerable amount of
+# time to build an isolated job environment in the job_working_directory if the
+# job working diretory is on a network share.  Set the following option to True
+# to cache the dependencies in a folder. This option is beta and should only be
+# used if you experience long waiting times before a job is actually submitted
+# to your cluster. If you activate this option and install or remove dependencies,
+# you may need to clear out old cached environments
 #use_cached_dependency_manager = False
 
 # By default the tool_dependency_cache_dir is the _cache directory
 # of the tool dependency directory
-#tool_dependency_cache_dir = tool_dependency_dir/_cache
+#tool_dependency_cache_dir = <tool_dependency_dir>/_cache
 
 # File containing the Galaxy Tool Sheds that should be made available to
 # install from in the admin interface (.sample used if default does not exist).

From 2b1765751b29008aecbd782f28269ac4e63db239 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 28 Oct 2016 18:30:47 +0200
Subject: [PATCH 03/13] Rework cached dependencies to be more robust

to updates or changes in depedencies, folder structure and resolver
configuration. Instead of hashing name, type and version of a
dependency, hash the json representation of the dependencies returned by
the dependency resolver, which include the path to the environment and
the depedency type. This is only applied to resolvers whose cacheable
attribute is set to True (conda-only, currently).
---
 config/galaxy.ini.sample                    |  5 +-
 lib/galaxy/tools/deps/__init__.py           | 40 +++++------
 lib/galaxy/tools/deps/conda_util.py         |  3 +-
 lib/galaxy/tools/deps/resolvers/__init__.py |  3 +-
 lib/galaxy/tools/deps/resolvers/conda.py    | 79 ++++++++++-----------
 5 files changed, 64 insertions(+), 66 deletions(-)

diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index af4799c7f70..2e4a325377f 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -219,11 +219,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 
 # Certain dependency resolvers (namely Conda) take a considerable amount of
 # time to build an isolated job environment in the job_working_directory if the
-# job working diretory is on a network share.  Set the following option to True
+# job working directory is on a network share.  Set the following option to True
 # to cache the dependencies in a folder. This option is beta and should only be
 # used if you experience long waiting times before a job is actually submitted
-# to your cluster. If you activate this option and install or remove dependencies,
-# you may need to clear out old cached environments
+# to your cluster.
 #use_cached_dependency_manager = False
 
 # By default the tool_dependency_cache_dir is the _cache directory
diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 85937d5b6da..13154cd65d8 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -172,35 +172,33 @@ class CachedDependencyManager(DependencyManager):
         super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
 
     def dependency_shell_commands( self, requirements, **kwds ):
-        commands = self.get_cached_commands(requirements, **kwds)
-        if not commands:
-            hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
-            kwds['conda_env'] = hashed_requirements_dir
-            requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds)
-            commands = [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()]
-            if not os.path.exists(hashed_requirements_dir):
-                # conda will create the hashed_requirements_dir, and fail if it already exists,
-                # while other resolvers may not create the hashed_requirements_dir
-                os.mkdir(hashed_requirements_dir)
-            with open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'), 'w') as cmds_f:
-                [cmds_f.write("%s\n" % line) for line in commands]
-            with open(os.path.join(hashed_requirements_dir, 'packages.json'), 'w') as packages:
-                # Keep a list of dependencies, so that on re-use we can log the resolved packages
-                packages.write(json.dumps([dep.to_dict() for dep in requirement_to_dependency.values()], sort_keys=True))
+        """
+        Runs a set of requirements through the dependency resolvers and returns
+        a list of commands required to activate the dependencies. For dependencies
+        that are cacheable (currently only conda), calculates a hash based on the name,
+        version, exact and dependency_type attributes for all dependencies. The hash
+        will be used as name for the folder where the environment will be created,
+        which allows re-using these environments.
+        """
+        resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
+        cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
+        hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
+        [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
+        commands = [dep.shell_commands(req) for req, dep in resolved_dependencies.items()]
         return commands
 
-    def hash_requirements(self, requirements):
+    def hash_requirements(self, resolved_dependencies):
         """Return hash for requirements"""
-        r_string = ':'.join(['_'.join((r.type, r.name, str(r.version))) for r in requirements])
-        return hash_util.new_secure_hash(r_string)
+        hashable_str = json.dumps([(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies])
+        return hash_util.new_secure_hash(hashable_str)[:8]  # short hash
 
-    def get_hashed_requirements_path(self, requirements):
+    def get_hashed_requirements_path(self, resolved_dependencies):
         """
         Returns the path to the hashed requirements directory (but does not evaluate whether the path exists)
         :param requirements:
         :return:
         """
-        req_hashes = self.hash_requirements(requirements)
+        req_hashes = self.hash_requirements(resolved_dependencies)
         return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes)
 
     def get_cached_commands(self, requirements, **kwargs):
@@ -209,6 +207,8 @@ class CachedDependencyManager(DependencyManager):
         :param requirements_hash:
         :return: list of commands
         """
+        if not requirements:  # if tool has no requirements
+            return []
         hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
         if not os.path.exists(os.path.join(hashed_requirements_dir)):
             return []
diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py
index 9914fdd016f..42ddc44f77d 100644
--- a/lib/galaxy/tools/deps/conda_util.py
+++ b/lib/galaxy/tools/deps/conda_util.py
@@ -54,7 +54,7 @@ class CondaContext(installable.InstallableContext):
 
     def __init__(self, conda_prefix=None, conda_exec=None,
                  shell_exec=None, debug=False, ensure_channels='',
-                 condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT):
+                 condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT, copy_dependencies=False):
         self.condarc_override = condarc_override
         if not conda_exec and use_path_exec:
             conda_exec = commands.which("conda")
@@ -63,6 +63,7 @@ class CondaContext(installable.InstallableContext):
         self.conda_exec = conda_exec
         self.debug = debug
         self.shell_exec = shell_exec or commands.shell
+        self.copy_dependencies = copy_dependencies
 
         if conda_prefix is None:
             info = self.conda_info()
diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py
index 6165fdb6af8..226b7a5f063 100644
--- a/lib/galaxy/tools/deps/resolvers/__init__.py
+++ b/lib/galaxy/tools/deps/resolvers/__init__.py
@@ -81,8 +81,9 @@ class InstallableDependencyResolver:
 
 
 class Dependency(Dictifiable, object):
-    dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version']
+    dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version', 'cacheable']
     __metaclass__ = ABCMeta
+    cacheable = False
 
     @abstractmethod
     def shell_commands( self, requirement ):
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index a45cf717952..0443093e4df 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -84,12 +84,12 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
             ensure_channels=ensure_channels,
             condarc_override=condarc_override,
             use_path_exec=use_path_exec,
+            copy_dependencies=_string_as_bool(get_option("copy_dependencies"))
         )
         self.ensure_channels = ensure_channels
 
         # Conda operations options (these define how resolution will occur)
         auto_install = _string_as_bool(get_option("auto_install"))
-        copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
         self.auto_init = _string_as_bool(get_option("auto_init"))
         self.conda_context = conda_context
         self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
@@ -115,20 +115,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         )
 
         job_directory = kwds.get("job_directory", None)
-        if job_directory is None:  # Job directory is None when resolve() called by find_dep()
-            if is_installed:
-                return CondaDependency(
-                    False,
-                    os.path.join(self.conda_context.envs_path, conda_target.install_environment),
-                    exact,
-                    name=name,
-                    version=version
-                )
-            else:
-                log.warning("Conda dependency resolver not sent job directory.")
-                return NullDependency(version=version, name=name)
-
-        if not is_installed and self.auto_install:
+        if not is_installed and self.auto_install and job_directory:
             is_installed = self.install_dependency(name=name, version=version, type=type)
 
         if not is_installed:
@@ -136,31 +123,23 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
 
         # Have installed conda_target and job_directory to send it to.
         # If dependency is for metadata generation, store environment in conda-metadata-env
+        if kwds.get("metadata", False):
+            conda_env = "conda-metadata-env"
+        else:
+            conda_env = "conda-env"
 
-        if kwds.get('conda_env', False):
-            conda_environment = kwds.get('conda_env')
-        else:
-            if kwds.get("metadata", False):
-                conda_env = "conda-metadata-env"
-            else:
-                conda_env = "conda-env"
+        if job_directory:
             conda_environment = os.path.join(job_directory, conda_env)
-        env_path, exit_code = build_isolated_environment(
-            conda_target,
-            path=conda_environment,
-            copy=self.copy_dependencies,
-            conda_context=self.conda_context,
-        )
-        if not exit_code:
-            return CondaDependency(
-                self.conda_context.activate,
-                conda_environment,
-                exact,
-                name,
-                version
-            )
         else:
-            return NullDependency(version=version, name=name)
+            conda_environment = None
+
+        return CondaDependency(
+            self.conda_context,
+            conda_environment,
+            exact,
+            name,
+            version
+        )
 
     def list_dependencies(self):
         for install_target in installed_conda_targets(self.conda_context):
@@ -208,9 +187,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
 class CondaDependency(Dependency):
     dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
     dependency_type = 'conda'
+    cacheable = True
 
-    def __init__(self, activate, environment_path, exact, name=None, version=None):
-        self.activate = activate
+    def __init__(self, conda_context, environment_path, exact, name=None, version=None):
+        self.activate = conda_context.activate
+        self.conda_context = conda_context
         self.environment_path = environment_path
         self._exact = exact
         self._name = name
@@ -228,8 +209,24 @@ class CondaDependency(Dependency):
     def version(self):
         return self._version
 
-    def shell_commands(self, requirement):
-        return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' 2>&1 """ % (
+    def set_cache_path(self, cache_path):
+        self.environment_path = cache_path
+
+    def shell_commands(self, requirement, cache_path=None):
+        env_path, exit_code = build_isolated_environment(
+            CondaTarget(self.name, self.version),
+            path=self.environment_path,
+            copy=self.conda_context.copy_dependencies,
+            conda_context=self.conda_context,
+        )
+        if exit_code:
+            if len(self.conda_environment) > 79:
+                # TODO: remove this once conda_build version 2 is released and packages have been rebuilt.
+                raise Exception("Conda dependency failed to build job environment. "
+                                "This is most likely a limitation in conda. "
+                                "You can try to shorten the path to the job_working_directory.")
+            raise Exception("Conda dependency seemingly installed but failed to build job environment.")
+        return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
             self.environment_path,
             self.activate,
             self.environment_path

From f919a98601252ab8f87bffff6e449105e770aadf Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 2 Nov 2016 11:59:10 +0100
Subject: [PATCH 04/13] Build dependency cache at install time

and only activate cached environments if they exist.
---
 lib/galaxy/tools/__init__.py                  | 12 ++++++
 lib/galaxy/tools/deps/__init__.py             | 39 +++++++------------
 lib/galaxy/tools/deps/resolvers/__init__.py   |  4 ++
 lib/galaxy/tools/deps/resolvers/conda.py      | 24 +++++++++---
 .../galaxy_install/install_manager.py         | 10 +++--
 5 files changed, 55 insertions(+), 34 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 04f4117dced..34a34b37f31 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -34,6 +34,7 @@ from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
 from galaxy.tools.deps import views
+from galaxy.tools.deps import CachedDependencyManager
 from galaxy.tools.parameters import params_to_incoming, check_param, params_from_strings, params_to_strings, visit_input_values
 from galaxy.tools.parameters import output_collect
 from galaxy.tools.parameters.basic import (BaseURLToolParameter,
@@ -1305,6 +1306,17 @@ class Tool( object, Dictifiable ):
         visit_input_values( self.inputs, values, validate_inputs )
         return messages
 
+    def build_dependency_cache(self):
+        if isinstance(self.app.toolbox.dependency_manager, CachedDependencyManager):
+            self.app.toolbox.dependency_manager.build_cache(
+                requirements=self.requirements,
+                installed_tool_dependencies=self.installed_tool_dependencies,
+                tool_dir=self.tool_dir,
+                job_directory=None,
+                metadata=False,
+                tool_instance=self
+            )
+
     def build_dependency_shell_commands( self, job_directory=None, metadata=False ):
         """
         Return a list of commands to be run to populate the current environment to include this tools requirements.
diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 13154cd65d8..4f0a6d2a7d4 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -171,26 +171,32 @@ class CachedDependencyManager(DependencyManager):
     def __init__(self, default_base_path, conf_file=None, **extra_config):
         super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
 
+    def build_cache(self, requirements, **kwds):
+        resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
+        cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
+        hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
+        [dep.build_cache(hashed_requirements_dir) for dep in cacheable_dependencies]
+
     def dependency_shell_commands( self, requirements, **kwds ):
         """
         Runs a set of requirements through the dependency resolvers and returns
-        a list of commands required to activate the dependencies. For dependencies
-        that are cacheable (currently only conda), calculates a hash based on the name,
-        version, exact and dependency_type attributes for all dependencies. The hash
-        will be used as name for the folder where the environment will be created,
-        which allows re-using these environments.
+        a list of commands required to activate the dependencies. If dependencies
+        are cacheable and the cache exists, will generate commands to activate
+        cached environments.
         """
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
         cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
         hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
-        [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
+        if os.path.exists(hashed_requirements_dir):
+            [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
         commands = [dep.shell_commands(req) for req, dep in resolved_dependencies.items()]
         return commands
 
     def hash_requirements(self, resolved_dependencies):
         """Return hash for requirements"""
-        hashable_str = json.dumps([(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies])
-        return hash_util.new_secure_hash(hashable_str)[:8]  # short hash
+        resolved_dependencies = [[(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies]]
+        hash_str = json.dumps(sorted([resolved_dependencies]))
+        return hash_util.new_secure_hash(hash_str)[:8]  # short hash
 
     def get_hashed_requirements_path(self, resolved_dependencies):
         """
@@ -200,20 +206,3 @@ class CachedDependencyManager(DependencyManager):
         """
         req_hashes = self.hash_requirements(resolved_dependencies)
         return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes)
-
-    def get_cached_commands(self, requirements, **kwargs):
-        """
-        Return commands for activating cached env if it exists
-        :param requirements_hash:
-        :return: list of commands
-        """
-        if not requirements:  # if tool has no requirements
-            return []
-        hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
-        if not os.path.exists(os.path.join(hashed_requirements_dir)):
-            return []
-        else:
-            if 'tool_instance' in kwargs:
-                dependencies = json.load(open(os.path.join(hashed_requirements_dir, 'packages.json')))
-                kwargs['tool_instance'].dependencies = dependencies
-            return [line.strip() for line in open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'))]
diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py
index 226b7a5f063..c24c107887a 100644
--- a/lib/galaxy/tools/deps/resolvers/__init__.py
+++ b/lib/galaxy/tools/deps/resolvers/__init__.py
@@ -122,3 +122,7 @@ class NullDependency( Dependency ):
 
     def shell_commands( self, requirement ):
         return None
+
+
+class DependencyException(Exception):
+    pass
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index 0443093e4df..935895a1ce9 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -21,6 +21,7 @@ from ..conda_util import (
 )
 from ..resolvers import (
     Dependency,
+    DependencyException,
     DependencyResolver,
     InstallableDependencyResolver,
     ListableDependencyResolver,
@@ -196,6 +197,7 @@ class CondaDependency(Dependency):
         self._exact = exact
         self._name = name
         self._version = version
+        self.cache_path = None
 
     @property
     def exact(self):
@@ -209,10 +211,15 @@ class CondaDependency(Dependency):
     def version(self):
         return self._version
 
+    def build_cache(self, cache_path):
+        self.set_cache_path(cache_path)
+        self.build_environment()
+
     def set_cache_path(self, cache_path):
+        self.cache_path = cache_path
         self.environment_path = cache_path
 
-    def shell_commands(self, requirement, cache_path=None):
+    def build_environment(self):
         env_path, exit_code = build_isolated_environment(
             CondaTarget(self.name, self.version),
             path=self.environment_path,
@@ -220,12 +227,17 @@ class CondaDependency(Dependency):
             conda_context=self.conda_context,
         )
         if exit_code:
-            if len(self.conda_environment) > 79:
+            if len(os.path.abspath(self.environment_path)) > 79:
                 # TODO: remove this once conda_build version 2 is released and packages have been rebuilt.
-                raise Exception("Conda dependency failed to build job environment. "
-                                "This is most likely a limitation in conda. "
-                                "You can try to shorten the path to the job_working_directory.")
-            raise Exception("Conda dependency seemingly installed but failed to build job environment.")
+                raise DependencyException("Conda dependency failed to build job environment. "
+                                          "This is most likely a limitation in conda. "
+                                          "You can try to shorten the path to the job_working_directory.")
+            raise DependencyException("Conda dependency seemingly installed but failed to build job environment.")
+
+    def shell_commands(self, requirement):
+        if not self.cache_path:
+            # Build an isolated environment if not using a cached dependency manager
+            self.build_environment()
         return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
             self.environment_path,
             self.activate,
diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 699797dcbd9..06c4ff7a4de 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -904,6 +904,13 @@ class InstallRepositoryManager( object ):
             self.install_model.context.refresh( tool_shed_repository )
             metadata = tool_shed_repository.metadata
             if 'tools' in metadata:
+                if install_resolver_dependencies:
+                    requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
+                    [self._view.install_dependency(id=None, **req) for req in requirements]
+                    for tool_d in metadata['tools']:
+                        tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
+                        if tool:
+                            tool.build_dependency_cache()
                 # Get the tool_versions from the tool shed for each tool in the installed change set.
                 self.update_tool_shed_repository_status( tool_shed_repository,
                                                          self.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
@@ -913,9 +920,6 @@ class InstallRepositoryManager( object ):
                     error_message += "Version information for the tools included in the <b>%s</b> repository is missing.  " % tool_shed_repository.name
                     error_message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions "
                     error_message += "from the installed repository's <b>Repository Actions</b> menu.  "
-                if install_resolver_dependencies:
-                    requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
-                    [self._view.install_dependency(id=None, **req) for req in requirements]
             if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
                 work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" )
                 # Install tool dependencies.

From 82f30e7645156b5f35c2b4cd77314e6275663979 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 4 Nov 2016 15:21:45 +0100
Subject: [PATCH 05/13] Only attempt to build cache once per install

and override __eq__ for ToolRequirement, to simplify
checking if ToolRequirements are already installed/cached.
---
 lib/galaxy/tools/deps/requirements.py           | 3 +++
 lib/tool_shed/galaxy_install/install_manager.py | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py
index 2b209f63362..0414ea2526a 100644
--- a/lib/galaxy/tools/deps/requirements.py
+++ b/lib/galaxy/tools/deps/requirements.py
@@ -25,6 +25,9 @@ class ToolRequirement( object ):
         type = dict.get("type", None)
         return ToolRequirement( name=name, type=type, version=version )
 
+    def __eq__(self, other):
+        return self.name == other.name and self.type == other.type and self.version == other.version
+
 
 DEFAULT_CONTAINER_TYPE = "docker"
 DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES = False
diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py
index 06c4ff7a4de..430986f2339 100644
--- a/lib/tool_shed/galaxy_install/install_manager.py
+++ b/lib/tool_shed/galaxy_install/install_manager.py
@@ -907,9 +907,11 @@ class InstallRepositoryManager( object ):
                 if install_resolver_dependencies:
                     requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
                     [self._view.install_dependency(id=None, **req) for req in requirements]
+                    cached_requirements = []
                     for tool_d in metadata['tools']:
                         tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
-                        if tool:
+                        if tool and tool.requirements not in cached_requirements:
+                            cached_requirements.append(tool.requirements)
                             tool.build_dependency_cache()
                 # Get the tool_versions from the tool shed for each tool in the installed change set.
                 self.update_tool_shed_repository_status( tool_shed_repository,

From 64bc9241ed785d4d552c5fac21cd412558c54567 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 23 Nov 2016 11:03:14 +0100
Subject: [PATCH 06/13] Fix NameError

---
 lib/galaxy/tools/deps/resolvers/conda.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index 935895a1ce9..5751212626e 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -67,6 +67,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
                 dependency_manager.default_base_path, DEFAULT_CONDARC_OVERRIDE
             )
 
+        copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
         conda_exec = get_option("exec")
         debug = _string_as_bool(get_option("debug"))
         ensure_channels = get_option("ensure_channels")
@@ -85,7 +86,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
             ensure_channels=ensure_channels,
             condarc_override=condarc_override,
             use_path_exec=use_path_exec,
-            copy_dependencies=_string_as_bool(get_option("copy_dependencies"))
+            copy_dependencies=copy_dependencies
         )
         self.ensure_channels = ensure_channels
 

From cf769dd09deb73624b22628208fa103cc4f27150 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 27 Nov 2016 16:38:34 +0100
Subject: [PATCH 07/13] Add API to install resolver tool dependencies

and to build a tool dependency cache (if activated in galaxy.ini).

An example to install dependencies for the twobit converter:
```
import bioblend.galaxy
url = 'http://localhost:8080/'
api_key = 'admin_api_key'
tool_id = 'CONVERTER_fasta_to_2bit'
endpoint = "api/tools/%s/install_dependencies" % tool_id
gi = bioblend.galaxy.GalaxyInstance(url, api_key)
gi.make_post_request("/".join((url, endpoint)), payload={'id': tool_id})
```
If `use_cached_dependency_manager` is activated in the galaxy.ini,
a cached environment can be built like this:
```
endpoint = "api/tools/%s/build_dependency_cache" % tool_id
gi.make_post_request("/".join((url, endpoint)), payload={'id': tool_id})
```
---
 .../datatypes/converters/fasta_to_2bit.xml    |  1 +
 lib/galaxy/webapps/galaxy/api/tools.py        | 27 +++++++++++++++++++
 lib/galaxy/webapps/galaxy/buildapp.py         |  2 ++
 3 files changed, 30 insertions(+)

diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
index a8e3bb3204f..f0e004661fb 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
@@ -2,6 +2,7 @@
     <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <!-- Used on the metadata edit page. -->
     <requirements>
+        <requirement type="package" version="332">ucsc-twobittofa</requirement>
         <requirement type="package">ucsc_tools</requirement>
     </requirements>
     <command>faToTwoBit '$input' '$output'</command>
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index 8dc7372738c..f91081d6b82 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -129,6 +129,33 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
         tool = self._get_tool(id)
         return tool.tool_requirements_status
 
+    @expose_api
+    @web.require_admin
+    def install_dependencies(self, trans, id, **kwds):
+        """
+        POST /api/tools/{tool_id}/install_dependencies
+        Attempts to install requirements via the dependency resolver
+        """
+        tool = self._get_tool(id)
+        [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements]
+        if kwds.get('build_dependency_cache'):
+            tool.build_dependency_cache()
+        # TODO: rework resolver install system to log and report what has been done.
+        # _view.install_dependency should return a dict with stdout, stderr and success status
+        return tool.tool_requirements_status
+
+    @expose_api
+    @web.require_admin
+    def build_dependency_cache(self, trans, id, **kwds):
+        """
+        POST /api/tools/{tool_id}/build_dependency_cache
+        Attempts to cache installed dependencies.
+        """
+        tool = self._get_tool(id)
+        tool.build_dependency_cache()
+        # TODO: Should also have a more meaningful return.
+        return tool.tool_requirements_status
+
     @expose_api
     @web.require_admin
     def diagnostics( self, trans, id, **kwd ):
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 53d0c948383..75eee3a7d4b 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -262,6 +262,8 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.connect( '/api/tools/{id:.+?}/citations', action='citations', controller="tools" )
     webapp.mapper.connect( '/api/tools/{id:.+?}/download', action='download', controller="tools" )
     webapp.mapper.connect( '/api/tools/{id:.+?}/requirements', action='requirements', controller="tools")
+    webapp.mapper.connect( '/api/tools/{id:.+?}/install_dependencies', action='install_dependencies', controller="tools", conditions=dict( method=[ "POST" ] ))
+    webapp.mapper.connect( '/api/tools/{id:.+?}/build_dependency_cache', action='build_dependency_cache', controller="tools", conditions=dict( method=[ "POST" ] ))
     webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" )
     webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' )
 

From cc7cd47ef54a09efbf8272d1e024f6f2ac92594b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 27 Nov 2016 17:49:59 +0100
Subject: [PATCH 08/13] Add integration test-case for install tool dependency
 API endpoint

---
 test/integration/test_resolvers.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py
index f1ee7d9b8eb..cf8993f0e8a 100644
--- a/test/integration/test_resolvers.py
+++ b/test/integration/test_resolvers.py
@@ -17,6 +17,7 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A
     @classmethod
     def handle_galaxy_config_kwds(cls, config):
         cls.conda_tmp_prefix = mkdtemp()
+        config["use_cached_dep_manager"] = True
         config["conda_auto_init"] = True
         config["conda_prefix"] = os.path.join(cls.conda_tmp_prefix, 'conda')
 
@@ -82,3 +83,15 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A
         self._assert_status_code_is( create_response, 200 )
         response = create_response.json()
         assert response['dependency_type'] == 'conda' and not response['exact']
+
+    def test_conda_install_through_tools_api( self ):
+        tool_id = 'mulled_example_multi_1'
+        endpoint = "tools/%s/install_dependencies" % tool_id
+        data = {'id': tool_id}
+        create_response = self._post(endpoint, data=data, admin=True)
+        self._assert_status_code_is( create_response, 200 )
+        response = create_response.json()
+        assert any([True for d in response if d['dependency_type'] == 'conda'])
+        endpoint = "tools/%s/build_dependency_cache" % tool_id
+        create_response = self._post(endpoint, data=data, admin=True)
+        self._assert_status_code_is( create_response, 200 )

From f10fd793a68ed9413bec708daa48cef65769d09f Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 27 Nov 2016 18:15:22 +0100
Subject: [PATCH 09/13] Install the correct converter dependency

---
 lib/galaxy/datatypes/converters/fasta_to_2bit.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
index f0e004661fb..3ccd7ab7db8 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
@@ -2,7 +2,7 @@
     <!-- <description>__NOT_USED_CURRENTLY_FOR_CONVERTERS__</description> -->
     <!-- Used on the metadata edit page. -->
     <requirements>
-        <requirement type="package" version="332">ucsc-twobittofa</requirement>
+        <requirement type="package" version="332">ucsc-fatotwobit</requirement>
         <requirement type="package">ucsc_tools</requirement>
     </requirements>
     <command>faToTwoBit '$input' '$output'</command>

From 045a2f0d471f7b4186aff625f3cea9abfa549fa2 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 27 Nov 2016 18:39:13 +0100
Subject: [PATCH 10/13] Add `force_rebuild` param to build_dependency_cache

This will remove a pre-existing cached dependency directory.
---
 lib/galaxy/tools/__init__.py           |  5 +++--
 lib/galaxy/tools/deps/__init__.py      |  7 +++++++
 lib/galaxy/webapps/galaxy/api/tools.py | 11 +++++++++--
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 34a34b37f31..ba8ee6ce1a2 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1306,7 +1306,7 @@ class Tool( object, Dictifiable ):
         visit_input_values( self.inputs, values, validate_inputs )
         return messages
 
-    def build_dependency_cache(self):
+    def build_dependency_cache(self, **kwds):
         if isinstance(self.app.toolbox.dependency_manager, CachedDependencyManager):
             self.app.toolbox.dependency_manager.build_cache(
                 requirements=self.requirements,
@@ -1314,7 +1314,8 @@ class Tool( object, Dictifiable ):
                 tool_dir=self.tool_dir,
                 job_directory=None,
                 metadata=False,
-                tool_instance=self
+                tool_instance=self,
+                **kwds
             )
 
     def build_dependency_shell_commands( self, job_directory=None, metadata=False ):
diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 4f0a6d2a7d4..640726f73d9 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -5,6 +5,7 @@ Dependency management for tools.
 import json
 import logging
 import os.path
+import shutil
 
 from collections import OrderedDict
 
@@ -175,6 +176,12 @@ class CachedDependencyManager(DependencyManager):
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
         cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
         hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
+        if kwds.get('force_rebuild', False) and os.path.exists(hashed_requirements_dir):
+            try:
+                shutil.rmtree(hashed_requirements_dir)
+            except Exception:
+                log.warning("Could not delete cached requirements directory '%s'" % hashed_requirements_dir)
+                pass
         [dep.build_cache(hashed_requirements_dir) for dep in cacheable_dependencies]
 
     def dependency_shell_commands( self, requirements, **kwds ):
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index f91081d6b82..b76dd9a3f40 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -135,11 +135,15 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
         """
         POST /api/tools/{tool_id}/install_dependencies
         Attempts to install requirements via the dependency resolver
+
+        parameters:
+            build_dependency_cache:  If true, attempts to cache dependencies for this tool
+            force_rebuild:           If true and chache dir exists, attempts to delete cache dir
         """
         tool = self._get_tool(id)
         [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements]
         if kwds.get('build_dependency_cache'):
-            tool.build_dependency_cache()
+            tool.build_dependency_cache(**kwds)
         # TODO: rework resolver install system to log and report what has been done.
         # _view.install_dependency should return a dict with stdout, stderr and success status
         return tool.tool_requirements_status
@@ -150,9 +154,12 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
         """
         POST /api/tools/{tool_id}/build_dependency_cache
         Attempts to cache installed dependencies.
+
+        parameters:
+            force_rebuild:           If true and chache dir exists, attempts to delete cache dir
         """
         tool = self._get_tool(id)
-        tool.build_dependency_cache()
+        tool.build_dependency_cache(**kwds)
         # TODO: Should also have a more meaningful return.
         return tool.tool_requirements_status
 

From 16b49d9140edbf116f83b51b957cd61c4f2d91df Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 27 Nov 2016 23:16:56 +0100
Subject: [PATCH 11/13] Add dependency_resolvers/clean endpoint

---
 lib/galaxy/tools/deps/resolvers/conda.py      |  4 ++++
 lib/galaxy/tools/deps/views.py                | 14 ++++++++++++++
 .../webapps/galaxy/api/tool_dependencies.py   | 19 +++++++++++++++++++
 lib/galaxy/webapps/galaxy/buildapp.py         |  1 +
 test/integration/test_resolvers.py            |  7 +++++++
 5 files changed, 45 insertions(+)

diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index 5751212626e..beb06871028 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -11,6 +11,7 @@ import galaxy.tools.deps.installable
 from ..conda_util import (
     build_isolated_environment,
     cleanup_failed_install,
+    exec_clean,
     CondaContext,
     CondaTarget,
     install_conda,
@@ -98,6 +99,9 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         self.auto_install = auto_install
         self.copy_dependencies = copy_dependencies
 
+    def clean(self, **kwds):
+        return exec_clean()
+
     def resolve(self, name, version, type, **kwds):
         # Check for conda just not being there, this way we can enable
         # conda by default and just do nothing in not configured.
diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py
index be4be6458b3..0735d4af4c8 100644
--- a/lib/galaxy/tools/deps/views.py
+++ b/lib/galaxy/tools/deps/views.py
@@ -125,5 +125,19 @@ class DependencyResolversView(object):
         """
         return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ]
 
+
     def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None):
         return [self.manager_dependency(installed_tool_dependencies=installed_tool_dependencies, **req) for req in requested_requirements]
+
+    def clean(self, index=None, **kwds):
+        if index:
+            resolver = self._dependency_resolver(index)
+            if not hasattr(resolver, "clean"):
+                raise NotImplemented()
+            else:
+                resolver.clean()
+                return "OK"
+        else:
+            [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')]
+            return "OK"
+
diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py
index f91c93ff5a5..cd961472355 100644
--- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py
+++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py
@@ -162,3 +162,22 @@ class ToolDependenciesAPIController( BaseAPIController ):
                     the corresponding resolver (keyed on 'index').
         """
         return self._view.manager_requirements()
+
+    @expose_api
+    @require_admin
+    def clean(self, trans, id=None, **kwds):
+        """
+        POST /api/dependencies_resolver/clean
+
+        Cleans up intermediate files created by resolvers during the dependency
+        installation.
+
+        :type   index:    int
+        :param  index:    index of the dependency resolver
+
+        :rtype:     dict
+        :returns:   a dictified description of the requirement that could
+                    be resolved (keyed on 'requirement') and the index of
+                    the corresponding resolver (keyed on 'index').
+        """
+        return self._view.clean(id, **kwds)
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 75eee3a7d4b..52b9fb3882a 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -270,6 +270,7 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" )
+    webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"]))
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" )
diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py
index cf8993f0e8a..d2f1208432f 100644
--- a/test/integration/test_resolvers.py
+++ b/test/integration/test_resolvers.py
@@ -95,3 +95,10 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A
         endpoint = "tools/%s/build_dependency_cache" % tool_id
         create_response = self._post(endpoint, data=data, admin=True)
         self._assert_status_code_is( create_response, 200 )
+
+    def test_conda_clean( self ):
+        endpoint = 'dependencies_resolvers/clean'
+        create_response = self._post(endpoint, data={}, admin=True)
+        self._assert_status_code_is(create_response, 200)
+        response = create_response.json()
+        assert response == "OK"
\ No newline at end of file

From 1efd4d4b3b99ec40071aa4d6fb7cf4030c51e2e0 Mon Sep 17 00:00:00 2001
From: Bjoern Gruening <bjoern.gruening@gmail.com>
Date: Fri, 16 Sep 2016 18:46:37 +0200
Subject: [PATCH 12/13] add a conda clean function to remove downloaded
 tarballs after installation

---
 lib/galaxy/tools/deps/conda_util.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py
index 42ddc44f77d..4f12716f927 100644
--- a/lib/galaxy/tools/deps/conda_util.py
+++ b/lib/galaxy/tools/deps/conda_util.py
@@ -214,6 +214,16 @@ class CondaContext(installable.InstallableContext):
         install_base_args.extend(args)
         return self.exec_command("install", install_base_args)
 
+    def exec_clean(self, args=[]):
+        """
+        Clean up after conda installation.
+        """
+        clean_base_args = [
+            "--tarballs"
+        ]
+        clean_base_args.extend(args)
+        return self.exec_command("clean", clean_base_args)
+
     def export_list(self, name, path):
         return self.exec_command("list", [
             "--name", name,
@@ -488,6 +498,7 @@ def build_isolated_environment(
 
         return (path or tempdir_name, exit_code)
     finally:
+        conda_context.exec_clean()
         shutil.rmtree(tempdir)
 
 

From 2a4ccac4039342c98d71bc72177aa951db6c4ba1 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 28 Nov 2016 00:07:40 +0100
Subject: [PATCH 13/13] Connect `clean` endpoint properly and add `-y` to
 exec_clean

---
 lib/galaxy/tools/deps/conda_util.py                | 3 ++-
 lib/galaxy/tools/deps/resolvers/conda.py           | 3 +--
 lib/galaxy/tools/deps/views.py                     | 2 --
 lib/galaxy/webapps/galaxy/api/tool_dependencies.py | 2 +-
 lib/galaxy/webapps/galaxy/buildapp.py              | 3 ++-
 test/integration/test_resolvers.py                 | 4 ++--
 6 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py
index 4f12716f927..47f1b090006 100644
--- a/lib/galaxy/tools/deps/conda_util.py
+++ b/lib/galaxy/tools/deps/conda_util.py
@@ -219,7 +219,8 @@ class CondaContext(installable.InstallableContext):
         Clean up after conda installation.
         """
         clean_base_args = [
-            "--tarballs"
+            "--tarballs",
+            "-y"
         ]
         clean_base_args.extend(args)
         return self.exec_command("clean", clean_base_args)
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index beb06871028..a21a37ebd4a 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -11,7 +11,6 @@ import galaxy.tools.deps.installable
 from ..conda_util import (
     build_isolated_environment,
     cleanup_failed_install,
-    exec_clean,
     CondaContext,
     CondaTarget,
     install_conda,
@@ -100,7 +99,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         self.copy_dependencies = copy_dependencies
 
     def clean(self, **kwds):
-        return exec_clean()
+        return self.conda_context.exec_clean()
 
     def resolve(self, name, version, type, **kwds):
         # Check for conda just not being there, this way we can enable
diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py
index 0735d4af4c8..8f23335d7df 100644
--- a/lib/galaxy/tools/deps/views.py
+++ b/lib/galaxy/tools/deps/views.py
@@ -125,7 +125,6 @@ class DependencyResolversView(object):
         """
         return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ]
 
-
     def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None):
         return [self.manager_dependency(installed_tool_dependencies=installed_tool_dependencies, **req) for req in requested_requirements]
 
@@ -140,4 +139,3 @@ class DependencyResolversView(object):
         else:
             [resolver.clean(**kwds) for resolver in self._dependency_resolvers if hasattr(resolver, 'clean')]
             return "OK"
-
diff --git a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py
index cd961472355..ea5821dfea3 100644
--- a/lib/galaxy/webapps/galaxy/api/tool_dependencies.py
+++ b/lib/galaxy/webapps/galaxy/api/tool_dependencies.py
@@ -167,7 +167,7 @@ class ToolDependenciesAPIController( BaseAPIController ):
     @require_admin
     def clean(self, trans, id=None, **kwds):
         """
-        POST /api/dependencies_resolver/clean
+        POST /api/dependencies_resolver/{index}/clean
 
         Cleans up intermediate files created by resolvers during the dependency
         installation.
diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py
index 52b9fb3882a..8bee0a15ddb 100644
--- a/lib/galaxy/webapps/galaxy/buildapp.py
+++ b/lib/galaxy/webapps/galaxy/buildapp.py
@@ -267,10 +267,11 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" )
     webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' )
 
+    webapp.mapper.connect( '/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict( method=[ "POST" ]) )
     webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="manager_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/requirements', action="manager_requirements", controller="tool_dependencies" )
-    webapp.mapper.connect('/api/dependency_resolvers/clean', action="clean", controller="tool_dependencies", conditions=dict(method=["POST"]))
+    webapp.mapper.connect( '/api/dependency_resolvers/{id}/clean', action="clean", controller="tool_dependencies", conditions=dict( method=[ "POST" ]) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="resolver_dependency", controller="tool_dependencies", conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/dependency', action="install_dependency", controller="tool_dependencies", conditions=dict( method=[ "POST" ] ) )
     webapp.mapper.connect( '/api/dependency_resolvers/{id}/requirements', action="resolver_requirements", controller="tool_dependencies" )
diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py
index d2f1208432f..5b134ac54ee 100644
--- a/test/integration/test_resolvers.py
+++ b/test/integration/test_resolvers.py
@@ -97,8 +97,8 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase, A
         self._assert_status_code_is( create_response, 200 )
 
     def test_conda_clean( self ):
-        endpoint = 'dependencies_resolvers/clean'
+        endpoint = 'dependency_resolvers/clean'
         create_response = self._post(endpoint, data={}, admin=True)
         self._assert_status_code_is(create_response, 200)
         response = create_response.json()
-        assert response == "OK"
\ No newline at end of file
+        assert response == "OK"