Rework cached dependencies to be more robust

to updates or changes in depedencies, folder structure and resolver
configuration. Instead of hashing name, type and version of a
dependency, hash the json representation of the dependencies returned by
the dependency resolver, which include the path to the environment and
the depedency type. This is only applied to resolvers whose cacheable
attribute is set to True (conda-only, currently).

config/galaxy.ini.sample

@@ -219,11 +219,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 
 # Certain dependency resolvers (namely Conda) take a considerable amount of
 # time to build an isolated job environment in the job_working_directory if the
-# job working diretory is on a network share.  Set the following option to True
+# job working directory is on a network share.  Set the following option to True
 # to cache the dependencies in a folder. This option is beta and should only be
 # used if you experience long waiting times before a job is actually submitted
-# to your cluster. If you activate this option and install or remove dependencies,
-# you may need to clear out old cached environments
+# to your cluster.
 #use_cached_dependency_manager = False
 
 # By default the tool_dependency_cache_dir is the _cache directory

lib/galaxy/tools/deps/__init__.py

@@ -172,35 +172,33 @@ class CachedDependencyManager(DependencyManager):
         super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
 
     def dependency_shell_commands( self, requirements, **kwds ):
-        commands = self.get_cached_commands(requirements, **kwds)
-        if not commands:
-            hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
-            kwds['conda_env'] = hashed_requirements_dir
-            requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds)
-            commands = [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()]
-            if not os.path.exists(hashed_requirements_dir):
-                # conda will create the hashed_requirements_dir, and fail if it already exists,
-                # while other resolvers may not create the hashed_requirements_dir
-                os.mkdir(hashed_requirements_dir)
-            with open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'), 'w') as cmds_f:
-                [cmds_f.write("%s\n" % line) for line in commands]
-            with open(os.path.join(hashed_requirements_dir, 'packages.json'), 'w') as packages:
-                # Keep a list of dependencies, so that on re-use we can log the resolved packages
-                packages.write(json.dumps([dep.to_dict() for dep in requirement_to_dependency.values()], sort_keys=True))
+        """
+        Runs a set of requirements through the dependency resolvers and returns
+        a list of commands required to activate the dependencies. For dependencies
+        that are cacheable (currently only conda), calculates a hash based on the name,
+        version, exact and dependency_type attributes for all dependencies. The hash
+        will be used as name for the folder where the environment will be created,
+        which allows re-using these environments.
+        """
+        resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
+        cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
+        hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
+        [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
+        commands = [dep.shell_commands(req) for req, dep in resolved_dependencies.items()]
         return commands
 
-    def hash_requirements(self, requirements):
+    def hash_requirements(self, resolved_dependencies):
         """Return hash for requirements"""
-        r_string = ':'.join(['_'.join((r.type, r.name, str(r.version))) for r in requirements])
-        return hash_util.new_secure_hash(r_string)
+        hashable_str = json.dumps([(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies])
+        return hash_util.new_secure_hash(hashable_str)[:8]  # short hash
 
-    def get_hashed_requirements_path(self, requirements):
+    def get_hashed_requirements_path(self, resolved_dependencies):
         """
         Returns the path to the hashed requirements directory (but does not evaluate whether the path exists)
         :param requirements:
         :return:
         """
-        req_hashes = self.hash_requirements(requirements)
+        req_hashes = self.hash_requirements(resolved_dependencies)
         return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes)
 
     def get_cached_commands(self, requirements, **kwargs):
@@ -209,6 +207,8 @@ class CachedDependencyManager(DependencyManager):
         :param requirements_hash:
         :return: list of commands
         """
+        if not requirements:  # if tool has no requirements
+            return []
         hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
         if not os.path.exists(os.path.join(hashed_requirements_dir)):
             return []

lib/galaxy/tools/deps/conda_util.py

@@ -54,7 +54,7 @@ class CondaContext(installable.InstallableContext):
 
     def __init__(self, conda_prefix=None, conda_exec=None,
                  shell_exec=None, debug=False, ensure_channels='',
-                 condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT):
+                 condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT, copy_dependencies=False):
         self.condarc_override = condarc_override
         if not conda_exec and use_path_exec:
             conda_exec = commands.which("conda")
@@ -63,6 +63,7 @@ class CondaContext(installable.InstallableContext):
         self.conda_exec = conda_exec
         self.debug = debug
         self.shell_exec = shell_exec or commands.shell
+        self.copy_dependencies = copy_dependencies
 
         if conda_prefix is None:
             info = self.conda_info()

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -81,8 +81,9 @@ class InstallableDependencyResolver:
 
 
 class Dependency(Dictifiable, object):
-    dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version']
+    dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version', 'cacheable']
     __metaclass__ = ABCMeta
+    cacheable = False
 
     @abstractmethod
     def shell_commands( self, requirement ):

lib/galaxy/tools/deps/resolvers/conda.py

@@ -84,12 +84,12 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
             ensure_channels=ensure_channels,
             condarc_override=condarc_override,
             use_path_exec=use_path_exec,
+            copy_dependencies=_string_as_bool(get_option("copy_dependencies"))
         )
         self.ensure_channels = ensure_channels
 
         # Conda operations options (these define how resolution will occur)
         auto_install = _string_as_bool(get_option("auto_install"))
-        copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
         self.auto_init = _string_as_bool(get_option("auto_init"))
         self.conda_context = conda_context
         self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
@@ -115,20 +115,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         )
 
         job_directory = kwds.get("job_directory", None)
-        if job_directory is None:  # Job directory is None when resolve() called by find_dep()
-            if is_installed:
-                return CondaDependency(
-                    False,
-                    os.path.join(self.conda_context.envs_path, conda_target.install_environment),
-                    exact,
-                    name=name,
-                    version=version
-                )
-            else:
-                log.warning("Conda dependency resolver not sent job directory.")
-                return NullDependency(version=version, name=name)
-
-        if not is_installed and self.auto_install:
+        if not is_installed and self.auto_install and job_directory:
             is_installed = self.install_dependency(name=name, version=version, type=type)
 
         if not is_installed:
@@ -136,31 +123,23 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
 
         # Have installed conda_target and job_directory to send it to.
         # If dependency is for metadata generation, store environment in conda-metadata-env
+        if kwds.get("metadata", False):
+            conda_env = "conda-metadata-env"
+        else:
+            conda_env = "conda-env"
 
-        if kwds.get('conda_env', False):
-            conda_environment = kwds.get('conda_env')
-        else:
-            if kwds.get("metadata", False):
-                conda_env = "conda-metadata-env"
-            else:
-                conda_env = "conda-env"
+        if job_directory:
             conda_environment = os.path.join(job_directory, conda_env)
-        env_path, exit_code = build_isolated_environment(
-            conda_target,
-            path=conda_environment,
-            copy=self.copy_dependencies,
-            conda_context=self.conda_context,
-        )
-        if not exit_code:
-            return CondaDependency(
-                self.conda_context.activate,
-                conda_environment,
-                exact,
-                name,
-                version
-            )
         else:
-            return NullDependency(version=version, name=name)
+            conda_environment = None
+
+        return CondaDependency(
+            self.conda_context,
+            conda_environment,
+            exact,
+            name,
+            version
+        )
 
     def list_dependencies(self):
         for install_target in installed_conda_targets(self.conda_context):
@@ -208,9 +187,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
 class CondaDependency(Dependency):
     dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
     dependency_type = 'conda'
+    cacheable = True
 
-    def __init__(self, activate, environment_path, exact, name=None, version=None):
-        self.activate = activate
+    def __init__(self, conda_context, environment_path, exact, name=None, version=None):
+        self.activate = conda_context.activate
+        self.conda_context = conda_context
         self.environment_path = environment_path
         self._exact = exact
         self._name = name
@@ -228,8 +209,24 @@ class CondaDependency(Dependency):
     def version(self):
         return self._version
 
-    def shell_commands(self, requirement):
-        return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' 2>&1 """ % (
+    def set_cache_path(self, cache_path):
+        self.environment_path = cache_path
+
+    def shell_commands(self, requirement, cache_path=None):
+        env_path, exit_code = build_isolated_environment(
+            CondaTarget(self.name, self.version),
+            path=self.environment_path,
+            copy=self.conda_context.copy_dependencies,
+            conda_context=self.conda_context,
+        )
+        if exit_code:
+            if len(self.conda_environment) > 79:
+                # TODO: remove this once conda_build version 2 is released and packages have been rebuilt.
+                raise Exception("Conda dependency failed to build job environment. "
+                                "This is most likely a limitation in conda. "
+                                "You can try to shorten the path to the job_working_directory.")
+            raise Exception("Conda dependency seemingly installed but failed to build job environment.")
+        return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
             self.environment_path,
             self.activate,
             self.environment_path