diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index af4799c7f70..2e4a325377f 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -219,11 +219,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory # Certain dependency resolvers (namely Conda) take a considerable amount of # time to build an isolated job environment in the job_working_directory if the -# job working diretory is on a network share. Set the following option to True +# job working directory is on a network share. Set the following option to True # to cache the dependencies in a folder. This option is beta and should only be # used if you experience long waiting times before a job is actually submitted -# to your cluster. If you activate this option and install or remove dependencies, -# you may need to clear out old cached environments +# to your cluster. #use_cached_dependency_manager = False # By default the tool_dependency_cache_dir is the _cache directory diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 85937d5b6da..13154cd65d8 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -172,35 +172,33 @@ class CachedDependencyManager(DependencyManager): super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config) def dependency_shell_commands( self, requirements, **kwds ): - commands = self.get_cached_commands(requirements, **kwds) - if not commands: - hashed_requirements_dir = self.get_hashed_requirements_path(requirements) - kwds['conda_env'] = hashed_requirements_dir - requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds) - commands = [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()] - if not os.path.exists(hashed_requirements_dir): - # conda will create the hashed_requirements_dir, and fail if it already exists, - # while other resolvers may not create the hashed_requirements_dir - os.mkdir(hashed_requirements_dir) - with open(os.path.join(hashed_requirements_dir, 'dep_commands.sh'), 'w') as cmds_f: - [cmds_f.write("%s\n" % line) for line in commands] - with open(os.path.join(hashed_requirements_dir, 'packages.json'), 'w') as packages: - # Keep a list of dependencies, so that on re-use we can log the resolved packages - packages.write(json.dumps([dep.to_dict() for dep in requirement_to_dependency.values()], sort_keys=True)) + """ + Runs a set of requirements through the dependency resolvers and returns + a list of commands required to activate the dependencies. For dependencies + that are cacheable (currently only conda), calculates a hash based on the name, + version, exact and dependency_type attributes for all dependencies. The hash + will be used as name for the folder where the environment will be created, + which allows re-using these environments. + """ + resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds) + cacheable_dependencies = [dep for req, dep in resolved_dependencies.items() if dep.cacheable] + hashed_requirements_dir = self.get_hashed_requirements_path(cacheable_dependencies) + [dep.set_cache_path(hashed_requirements_dir) for dep in cacheable_dependencies] + commands = [dep.shell_commands(req) for req, dep in resolved_dependencies.items()] return commands - def hash_requirements(self, requirements): + def hash_requirements(self, resolved_dependencies): """Return hash for requirements""" - r_string = ':'.join(['_'.join((r.type, r.name, str(r.version))) for r in requirements]) - return hash_util.new_secure_hash(r_string) + hashable_str = json.dumps([(dep.name, dep.version, dep.exact, dep.dependency_type) for dep in resolved_dependencies]) + return hash_util.new_secure_hash(hashable_str)[:8] # short hash - def get_hashed_requirements_path(self, requirements): + def get_hashed_requirements_path(self, resolved_dependencies): """ Returns the path to the hashed requirements directory (but does not evaluate whether the path exists) :param requirements: :return: """ - req_hashes = self.hash_requirements(requirements) + req_hashes = self.hash_requirements(resolved_dependencies) return os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes) def get_cached_commands(self, requirements, **kwargs): @@ -209,6 +207,8 @@ class CachedDependencyManager(DependencyManager): :param requirements_hash: :return: list of commands """ + if not requirements: # if tool has no requirements + return [] hashed_requirements_dir = self.get_hashed_requirements_path(requirements) if not os.path.exists(os.path.join(hashed_requirements_dir)): return [] diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index 9914fdd016f..42ddc44f77d 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -54,7 +54,7 @@ class CondaContext(installable.InstallableContext): def __init__(self, conda_prefix=None, conda_exec=None, shell_exec=None, debug=False, ensure_channels='', - condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT): + condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT, copy_dependencies=False): self.condarc_override = condarc_override if not conda_exec and use_path_exec: conda_exec = commands.which("conda") @@ -63,6 +63,7 @@ class CondaContext(installable.InstallableContext): self.conda_exec = conda_exec self.debug = debug self.shell_exec = shell_exec or commands.shell + self.copy_dependencies = copy_dependencies if conda_prefix is None: info = self.conda_info() diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 6165fdb6af8..226b7a5f063 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -81,8 +81,9 @@ class InstallableDependencyResolver: class Dependency(Dictifiable, object): - dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version'] + dict_collection_visible_keys = ['dependency_type', 'exact', 'name', 'version', 'cacheable'] __metaclass__ = ABCMeta + cacheable = False @abstractmethod def shell_commands( self, requirement ): diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index a45cf717952..0443093e4df 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -84,12 +84,12 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In ensure_channels=ensure_channels, condarc_override=condarc_override, use_path_exec=use_path_exec, + copy_dependencies=_string_as_bool(get_option("copy_dependencies")) ) self.ensure_channels = ensure_channels # Conda operations options (these define how resolution will occur) auto_install = _string_as_bool(get_option("auto_install")) - copy_dependencies = _string_as_bool(get_option("copy_dependencies")) self.auto_init = _string_as_bool(get_option("auto_init")) self.conda_context = conda_context self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init) @@ -115,20 +115,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In ) job_directory = kwds.get("job_directory", None) - if job_directory is None: # Job directory is None when resolve() called by find_dep() - if is_installed: - return CondaDependency( - False, - os.path.join(self.conda_context.envs_path, conda_target.install_environment), - exact, - name=name, - version=version - ) - else: - log.warning("Conda dependency resolver not sent job directory.") - return NullDependency(version=version, name=name) - - if not is_installed and self.auto_install: + if not is_installed and self.auto_install and job_directory: is_installed = self.install_dependency(name=name, version=version, type=type) if not is_installed: @@ -136,31 +123,23 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In # Have installed conda_target and job_directory to send it to. # If dependency is for metadata generation, store environment in conda-metadata-env + if kwds.get("metadata", False): + conda_env = "conda-metadata-env" + else: + conda_env = "conda-env" - if kwds.get('conda_env', False): - conda_environment = kwds.get('conda_env') - else: - if kwds.get("metadata", False): - conda_env = "conda-metadata-env" - else: - conda_env = "conda-env" + if job_directory: conda_environment = os.path.join(job_directory, conda_env) - env_path, exit_code = build_isolated_environment( - conda_target, - path=conda_environment, - copy=self.copy_dependencies, - conda_context=self.conda_context, - ) - if not exit_code: - return CondaDependency( - self.conda_context.activate, - conda_environment, - exact, - name, - version - ) else: - return NullDependency(version=version, name=name) + conda_environment = None + + return CondaDependency( + self.conda_context, + conda_environment, + exact, + name, + version + ) def list_dependencies(self): for install_target in installed_conda_targets(self.conda_context): @@ -208,9 +187,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In class CondaDependency(Dependency): dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version'] dependency_type = 'conda' + cacheable = True - def __init__(self, activate, environment_path, exact, name=None, version=None): - self.activate = activate + def __init__(self, conda_context, environment_path, exact, name=None, version=None): + self.activate = conda_context.activate + self.conda_context = conda_context self.environment_path = environment_path self._exact = exact self._name = name @@ -228,8 +209,24 @@ class CondaDependency(Dependency): def version(self): return self._version - def shell_commands(self, requirement): - return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' 2>&1 """ % ( + def set_cache_path(self, cache_path): + self.environment_path = cache_path + + def shell_commands(self, requirement, cache_path=None): + env_path, exit_code = build_isolated_environment( + CondaTarget(self.name, self.version), + path=self.environment_path, + copy=self.conda_context.copy_dependencies, + conda_context=self.conda_context, + ) + if exit_code: + if len(self.conda_environment) > 79: + # TODO: remove this once conda_build version 2 is released and packages have been rebuilt. + raise Exception("Conda dependency failed to build job environment. " + "This is most likely a limitation in conda. " + "You can try to shorten the path to the job_working_directory.") + raise Exception("Conda dependency seemingly installed but failed to build job environment.") + return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % ( self.environment_path, self.activate, self.environment_path