Build dependency cache at install time

and only activate cached environments if they exist.

lib/galaxy/tools/__init__.py

@@ -34,6 +34,7 @@ from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
 from galaxy.tools.deps import views
+from galaxy.tools.deps import CachedDependencyManager
 from galaxy.tools.parameters import params_to_incoming, check_param, params_from_strings, params_to_strings, visit_input_values
 from galaxy.tools.parameters import output_collect
 from galaxy.tools.parameters.basic import (BaseURLToolParameter,
@@ -1305,6 +1306,17 @@ class Tool( object, Dictifiable ):
         visit_input_values( self.inputs, values, validate_inputs )
         return messages
 
+    def build_dependency_cache(self):
+        if isinstance(self.app.toolbox.dependency_manager, CachedDependencyManager):
+            self.app.toolbox.dependency_manager.build_cache(
+                requirements=self.requirements,
+                installed_tool_dependencies=self.installed_tool_dependencies,
+                tool_dir=self.tool_dir,
+                job_directory=None,
+                metadata=False,
+                tool_instance=self
+            )
+
     def build_dependency_shell_commands( self, job_directory=None, metadata=False ):
         """
         Return a list of commands to be run to populate the current environment to include this tools requirements.

lib/galaxy/tools/deps/__init__.py

@@ -171,26 +171,32 @@ class CachedDependencyManager(DependencyManager):
     def __init__(self, default_base_path, conf_file=None, **extra_config):
         super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
 
+    def build_cache(self, requirements, **kwds):
+        resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
+        cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
+        hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
+        [dep.build_cache(hashed_requirements_dir) for dep in cacheable_dependencies]
+
     def dependency_shell_commands( self, requirements, **kwds ):
         """
         Runs a set of requirements through the dependency resolvers and returns
-        a list of commands required to activate the dependencies. For dependencies
-        that are cacheable (currently only conda), calculates a hash based on the name,
-        version, exact and dependency_type attributes for all dependencies. The hash
-        will be used as name for the folder where the environment will be created,
-        which allows re-using these environments.
+        a list of commands required to activate the dependencies. If dependencies
+        are cacheable and the cache exists, will generate commands to activate
+        cached environments.
         """
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
         cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable]
         hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies)
-        [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
+        if os.path.exists(hashed_requirements_dir):
+            [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies]
         commands = [dep.shell_commands(req) for req, dep in resolved_dependencies.items()]
         return commands
 
     def hash_requirements(self, resolved_dependencies):
         """Return hash for requirements"""
-        hashable_str = json.dumps([(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies])
-        return hash_util.new_secure_hash(hashable_str)[:8]  # short hash
+        resolved_dependencies = [[(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies]]
+        hash_str = json.dumps(sorted([resolved_dependencies]))
+        return hash_util.new_secure_hash(hash_str)[:8]  # short hash
 
     def get_hashed_requirements_path(self, resolved_dependencies):
         """
@@ -200,20 +206,3 @@ class CachedDependencyManager(DependencyManager):
         """
         req_hashes = self.hash_requirements(resolved_dependencies)
         return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes)
-
-    def get_cached_commands(self, requirements, **kwargs):
-        """
-        Return commands for activating cached env if it exists
-        :param requirements_hash:
-        :return: list of commands
-        """
-        if not requirements:  # if tool has no requirements
-            return []
-        hashed_requirements_dir = self.get_hashed_requirements_path(requirements)
-        if not os.path.exists(os.path.join(hashed_requirements_dir)):
-            return []
-        else:
-            if 'tool_instance' in kwargs:
-                dependencies = json.load(open(os.path.join(hashed_requirements_dir, 'packages.json')))
-                kwargs['tool_instance'].dependencies = dependencies
-            return [line.strip() for line in open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'))]

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -122,3 +122,7 @@ class NullDependency( Dependency ):
 
     def shell_commands( self, requirement ):
         return None
+
+
+class DependencyException(Exception):
+    pass

lib/galaxy/tools/deps/resolvers/conda.py

@@ -21,6 +21,7 @@ from ..conda_util import (
 )
 from ..resolvers import (
     Dependency,
+    DependencyException,
     DependencyResolver,
     InstallableDependencyResolver,
     ListableDependencyResolver,
@@ -196,6 +197,7 @@ class CondaDependency(Dependency):
         self._exact = exact
         self._name = name
         self._version = version
+        self.cache_path = None
 
     @property
     def exact(self):
@@ -209,10 +211,15 @@ class CondaDependency(Dependency):
     def version(self):
         return self._version
 
+    def build_cache(self, cache_path):
+        self.set_cache_path(cache_path)
+        self.build_environment()
+
     def set_cache_path(self, cache_path):
+        self.cache_path = cache_path
         self.environment_path = cache_path
 
-    def shell_commands(self, requirement, cache_path=None):
+    def build_environment(self):
         env_path, exit_code = build_isolated_environment(
             CondaTarget(self.name, self.version),
             path=self.environment_path,
@@ -220,12 +227,17 @@ class CondaDependency(Dependency):
             conda_context=self.conda_context,
         )
         if exit_code:
-            if len(self.conda_environment) > 79:
+            if len(os.path.abspath(self.environment_path)) > 79:
                 # TODO: remove this once conda_build version 2 is released and packages have been rebuilt.
-                raise Exception("Conda dependency failed to build job environment. "
-                                "This is most likely a limitation in conda. "
-                                "You can try to shorten the path to the job_working_directory.")
-            raise Exception("Conda dependency seemingly installed but failed to build job environment.")
+                raise DependencyException("Conda dependency failed to build job environment. "
+                                          "This is most likely a limitation in conda. "
+                                          "You can try to shorten the path to the job_working_directory.")
+            raise DependencyException("Conda dependency seemingly installed but failed to build job environment.")
+
+    def shell_commands(self, requirement):
+        if not self.cache_path:
+            # Build an isolated environment if not using a cached dependency manager
+            self.build_environment()
         return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
             self.environment_path,
             self.activate,

lib/tool_shed/galaxy_install/install_manager.py

@@ -904,6 +904,13 @@ class InstallRepositoryManager( object ):
             self.install_model.context.refresh( tool_shed_repository )
             metadata = tool_shed_repository.metadata
             if 'tools' in metadata:
+                if install_resolver_dependencies:
+                    requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
+                    [self._view.install_dependency(id=None, **req) for req in requirements]
+                    for tool_d in metadata['tools']:
+                        tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
+                        if tool:
+                            tool.build_dependency_cache()
                 # Get the tool_versions from the tool shed for each tool in the installed change set.
                 self.update_tool_shed_repository_status( tool_shed_repository,
                                                          self.install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
@@ -913,9 +920,6 @@ class InstallRepositoryManager( object ):
                     error_message += "Version information for the tools included in the <b>%s</b> repository is missing.  " % tool_shed_repository.name
                     error_message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions "
                     error_message += "from the installed repository's <b>Repository Actions</b> menu.  "
-                if install_resolver_dependencies:
-                    requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
-                    [self._view.install_dependency(id=None, **req) for req in requirements]
             if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
                 work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" )
                 # Install tool dependencies.