From cbcd3627f3833ada31b58863d5504666bb98b956 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 16:08:03 +0100 Subject: [PATCH 01/50] Allow passing unused keywords to container resolver --- lib/galaxy/tools/deps/container_resolvers/__init__.py | 2 +- lib/galaxy/tools/deps/container_resolvers/explicit.py | 2 +- lib/galaxy/tools/deps/container_resolvers/mulled.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/__init__.py b/lib/galaxy/tools/deps/container_resolvers/__init__.py index 0e819f9458a..485560bbec4 100644 --- a/lib/galaxy/tools/deps/container_resolvers/__init__.py +++ b/lib/galaxy/tools/deps/container_resolvers/__init__.py @@ -33,7 +33,7 @@ class ContainerResolver(Dictifiable): return default @abstractmethod - def resolve(self, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): """Find a container matching all supplied requirements for tool. The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description diff --git a/lib/galaxy/tools/deps/container_resolvers/explicit.py b/lib/galaxy/tools/deps/container_resolvers/explicit.py index 043ac20fed6..9f61ab95346 100644 --- a/lib/galaxy/tools/deps/container_resolvers/explicit.py +++ b/lib/galaxy/tools/deps/container_resolvers/explicit.py @@ -13,7 +13,7 @@ class ExplicitContainerResolver(ContainerResolver): resolver_type = "explicit" - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): """Find a container explicitly mentioned in tool description. This ignores the tool requirements and assumes the tool author crafted diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index fc53d5cf7a8..da6da85b358 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -225,7 +225,7 @@ class CachedMulledDockerContainerResolver(ContainerResolver): self.namespace = namespace self.hash_func = hash_func - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): if tool_info.requires_galaxy_python_environment: return None @@ -247,7 +247,7 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) self.hash_func = hash_func - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): if tool_info.requires_galaxy_python_environment: return None @@ -270,7 +270,7 @@ class MulledDockerContainerResolver(ContainerResolver): self.namespace = namespace self.hash_func = hash_func - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): if tool_info.requires_galaxy_python_environment: return None @@ -357,7 +357,7 @@ class BuildMulledDockerContainerResolver(ContainerResolver): } self.auto_init = self._get_config_option("involucro_auto_init", True) - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): if tool_info.requires_galaxy_python_environment: return None @@ -404,7 +404,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): } self.auto_init = self._get_config_option("involucro_auto_init", True) - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): if tool_info.requires_galaxy_python_environment: return None From 8f8642aa33e4c25e9f3bdff28b37a334af97b89c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 16:19:48 +0100 Subject: [PATCH 02/50] Merge container resolver with high-level dependency manager --- lib/galaxy/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e502bc4279a..0859859e596 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1059,6 +1059,8 @@ class ConfiguresGalaxyMixin(object): involucro_auto_init=self.config.involucro_auto_init, ) self.container_finder = containers.ContainerFinder(app_info) + self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes) + self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers) index_help = getattr(self.config, "index_tool_help", True) self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_help) self.reindex_tool_search() From b505fe37eac6b0331c20b26b7a0cddc49942a38a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 16:58:39 +0100 Subject: [PATCH 03/50] Introduce a ContainerDependency class --- lib/galaxy/tools/deps/resolvers/__init__.py | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index beca0dcedf1..7fd9099acde 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -257,6 +257,33 @@ class Dependency(Dictifiable): return "Using dependency %s version %s of type %s" % (self.name, self.version, self.dependency_type) +class ContainerDependency(Dependency): + + dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path'] + + def __init__(self, container_description, name, version): + self.dependency_type = container_description.type + self._name = name + self._version = version + self.environment_path = container_description.identifier + + @property + def name(self): + return self._name + + @property + def version(self): + return self._version + + @property + def exact(self): + return True + + @property + def shell_commands(self): + return None + + class NullDependency(Dependency): dependency_type = None exact = True From 360e605be965be5c414ca9ead34c536dd4003fd9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 17:02:21 +0100 Subject: [PATCH 04/50] Adjust _requirements_to_dependencies_dict to also report available containers in the manage dependencies menu --- lib/galaxy/tools/deps/__init__.py | 61 +++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 0e68d8c068d..bfceca5a000 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -13,11 +13,18 @@ from galaxy.util import ( plugin_config ) from galaxy.util.oset import OrderedSet + +from .container_resolvers import ContainerResolver +from .containers import ToolInfo from .requirements import ( + ContainerDescription, ToolRequirement, ToolRequirements ) -from .resolvers import NullDependency +from .resolvers import ( + ContainerDependency, + NullDependency, +) from .resolvers.conda import CondaDependencyResolver from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver @@ -133,6 +140,7 @@ class DependencyManager(object): return_null_dependencies = kwds.get('return_null', False) resolvable_requirements = requirements.resolvable + tool_info = ToolInfo(requirements=resolvable_requirements) for i, resolver in enumerate(self.dependency_resolvers): if index is not None and i != index: @@ -142,12 +150,31 @@ class DependencyManager(object): # Shortcut - resolution complete. break + if resolver.resolver_type.startswith('build_mulled'): + # don't want to build images here + continue + # Check requirements all at once all_unmet = len(requirement_to_dependency) == 0 - if all_unmet and hasattr(resolver, "resolve_all"): + if hasattr(resolver, "resolve_all"): + resolve = resolver.resolve_all + elif isinstance(resolver, ContainerResolver): + if not resolver.resolver_type.startswith('Cached'): + # These would look up available containers using the quay API, + # we only want ot do this if we search for containers + continue + resolve = resolver.resolve + else: + resolve = None + if all_unmet and resolve is not None: # TODO: Handle specs. - dependencies = resolver.resolve_all(resolvable_requirements, **kwds) + dependencies = resolve(requirements=resolvable_requirements, + enabled_container_types=['docker', 'singularity'], + tool_info=tool_info, + **kwds) if dependencies: + if isinstance(dependencies, ContainerDescription): + dependencies = [ContainerDependency(dependencies, name=r.name, version=r.version) for r in resolvable_requirements] assert len(dependencies) == len(resolvable_requirements) for requirement, dependency in zip(resolvable_requirements, dependencies): log.debug(dependency.resolver_msg) @@ -156,21 +183,23 @@ class DependencyManager(object): # Shortcut - resolution complete. break - # Check individual requirements - for requirement in resolvable_requirements: - if requirement in requirement_to_dependency: - continue + if not isinstance(resolver, ContainerResolver): - dependency = resolver.resolve(requirement, **kwds) - if require_exact and not dependency.exact: - continue + # Check individual requirements + for requirement in resolvable_requirements: + if requirement in requirement_to_dependency: + continue - if not isinstance(dependency, NullDependency): - log.debug(dependency.resolver_msg) - requirement_to_dependency[requirement] = dependency - elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): - log.debug(dependency.resolver_msg) - requirement_to_dependency[requirement] = dependency + dependency = resolver.resolve(requirement, **kwds) + if require_exact and not dependency.exact: + continue + + if not isinstance(dependency, NullDependency): + log.debug(dependency.resolver_msg) + requirement_to_dependency[requirement] = dependency + elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): + log.debug(dependency.resolver_msg) + requirement_to_dependency[requirement] = dependency return requirement_to_dependency From 40d8a38108fca6c21efc72f66660b3a1a7c42caa Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 18:27:22 +0100 Subject: [PATCH 05/50] Augment NullDependency with version and always record incoming null dependencies, since the container resolvers don't return NullDependency objects. --- lib/galaxy/tools/deps/__init__.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index bfceca5a000..2308aee6f1f 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -13,7 +13,6 @@ from galaxy.util import ( plugin_config ) from galaxy.util.oset import OrderedSet - from .container_resolvers import ContainerResolver from .containers import ToolInfo from .requirements import ( @@ -143,10 +142,13 @@ class DependencyManager(object): tool_info = ToolInfo(requirements=resolvable_requirements) for i, resolver in enumerate(self.dependency_resolvers): + + _requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)]) + if index is not None and i != index: continue - if len(requirement_to_dependency) == len(resolvable_requirements): + if len(_requirement_to_dependency) == len(resolvable_requirements): # Shortcut - resolution complete. break @@ -155,11 +157,11 @@ class DependencyManager(object): continue # Check requirements all at once - all_unmet = len(requirement_to_dependency) == 0 + all_unmet = len(_requirement_to_dependency) == 0 if hasattr(resolver, "resolve_all"): resolve = resolver.resolve_all elif isinstance(resolver, ContainerResolver): - if not resolver.resolver_type.startswith('Cached'): + if not resolver.resolver_type.startswith(('cached', 'explicit')): # These would look up available containers using the quay API, # we only want ot do this if we search for containers continue @@ -187,7 +189,7 @@ class DependencyManager(object): # Check individual requirements for requirement in resolvable_requirements: - if requirement in requirement_to_dependency: + if requirement in _requirement_to_dependency: continue dependency = resolver.resolve(requirement, **kwds) @@ -197,8 +199,9 @@ class DependencyManager(object): if not isinstance(dependency, NullDependency): log.debug(dependency.resolver_msg) requirement_to_dependency[requirement] = dependency - elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): + elif return_null_dependencies: log.debug(dependency.resolver_msg) + dependency.version = requirement.version requirement_to_dependency[requirement] = dependency return requirement_to_dependency From f2128692a4f42da2254a53b69cbf1931e74c10dc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 18:42:13 +0100 Subject: [PATCH 06/50] Fix matching single package mulled docker containers An image like `quay.io/biocontainers/ucsc-liftover:357--1` would otherwise have the package name `quay.io/biocontainers/ucsc-liftover` instead of `ucsc-liftover` --- lib/galaxy/tools/deps/container_resolvers/mulled.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index da6da85b358..2a993e93c91 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -112,7 +112,8 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None): build = None if version and "--" in version: version, build = split_tag(version) - + if prefix and image_name.startswith(prefix): + image_name = image_name[len(prefix):] image = CachedMulledImageSingleTarget(image_name, version, build, identifier) return image From fa6cca0224fd2b0239160df565a58507b05ac00f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Dec 2018 19:29:53 +0100 Subject: [PATCH 07/50] Only walk through enabled container resolvers --- lib/galaxy/tools/deps/__init__.py | 3 ++- lib/galaxy/tools/deps/containers.py | 4 ++-- lib/galaxy/tools/deps/views.py | 9 ++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 2308aee6f1f..b51441fff82 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -137,6 +137,7 @@ class DependencyManager(object): index = kwds.get('index', None) require_exact = kwds.get('exact', False) return_null_dependencies = kwds.get('return_null', False) + enabled_container_types = kwds.pop('enabled_container_types', []) resolvable_requirements = requirements.resolvable tool_info = ToolInfo(requirements=resolvable_requirements) @@ -171,7 +172,7 @@ class DependencyManager(object): if all_unmet and resolve is not None: # TODO: Handle specs. dependencies = resolve(requirements=resolvable_requirements, - enabled_container_types=['docker', 'singularity'], + enabled_container_types=enabled_container_types, tool_info=tool_info, **kwds) if dependencies: diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index a43516e615d..124d0fd9a77 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -75,7 +75,7 @@ class ContainerFinder(object): self.app_info = app_info self.container_registry = ContainerRegistry(app_info) - def __enabled_container_types(self, destination_info): + def _enabled_container_types(self, destination_info): return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)] def find_best_container_description(self, enabled_container_types, tool_info): @@ -86,7 +86,7 @@ class ContainerFinder(object): return container_description def find_container(self, tool_info, destination_info, job_info): - enabled_container_types = self.__enabled_container_types(destination_info) + enabled_container_types = self._enabled_container_types(destination_info) # Short-cut everything else and just skip checks if no container type is enabled. if not enabled_container_types: diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index b1f3af0bab1..5191c501e33 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -46,10 +46,17 @@ class DependencyResolversView(object): """ Resolves dependencies to build a requirements status in the admin panel/API """ + enabled_container_types = set() + for destinations in self._app.job_config.destinations.values(): + for destination in destinations: + enabled_container_types.update(self._app.container_finder._enabled_container_types(destination.params)) kwds = {'install': False, 'return_null': True, 'installed_tool_dependencies': installed_tool_dependencies} - dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()} + dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, + enabled_container_types=enabled_container_types, + **kwds) + for tool, requirements in tool_requirements_d.items()} return dependencies_per_tool def uninstall_dependencies(self, index=None, **payload): From 541f0ebf3d754f5a8eb4a3d8a56d0c197c29c5cb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 10:57:10 +0100 Subject: [PATCH 08/50] Avoid circular imports by moving {App,Job,Tool} Info classes to dependencies.py --- lib/galaxy/config.py | 3 +- lib/galaxy/jobs/rule_helper.py | 5 ++- lib/galaxy/jobs/runners/__init__.py | 9 +++-- lib/galaxy/tools/deps/__init__.py | 2 +- lib/galaxy/tools/deps/containers.py | 52 --------------------------- lib/galaxy/tools/deps/dependencies.py | 52 +++++++++++++++++++++++++++ 6 files changed, 63 insertions(+), 60 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 0859859e596..c0bf4097182 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1035,6 +1035,7 @@ class ConfiguresGalaxyMixin(object): from galaxy import tools from galaxy.managers.citations import CitationsManager from galaxy.tools.deps import containers + from galaxy.tools.deps.dependencies import AppInfo import galaxy.tools.search self.citations_manager = CitationsManager(self) @@ -1047,7 +1048,7 @@ class ConfiguresGalaxyMixin(object): self.toolbox = tools.ToolBox(tool_configs, self.config.tool_path, self) galaxy_root_dir = os.path.abspath(self.config.root) file_path = os.path.abspath(getattr(self.config, "file_path")) - app_info = containers.AppInfo( + app_info = AppInfo( galaxy_root_dir=galaxy_root_dir, default_file_path=file_path, outputs_to_working_directory=self.config.outputs_to_working_directory, diff --git a/lib/galaxy/jobs/rule_helper.py b/lib/galaxy/jobs/rule_helper.py index b45a6f57bf6..2dd339d6ae4 100644 --- a/lib/galaxy/jobs/rule_helper.py +++ b/lib/galaxy/jobs/rule_helper.py @@ -9,6 +9,7 @@ from galaxy import ( model, util ) +from galaxy.tools.deps.dependencies import ToolInfo log = logging.getLogger(__name__) @@ -42,9 +43,7 @@ class RuleHelper(object): else: # Have a Job object. tool = self.app.toolbox.get_tool(job_or_tool.tool_id, tool_version=job_or_tool.tool_version) - # Can't import at top because circular import between galaxy.tools and galaxy.jobs. - import galaxy.tools.deps.containers - tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) + tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) container_description = self.app.container_finder.find_best_container_description(["docker"], tool_info) return container_description is not None diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 3849b2299d8..3ddec1e070e 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -25,6 +25,10 @@ from galaxy.jobs.runners.util.job_script import ( job_script, write_script ) +from galaxy.tools.deps.dependencies import ( + JobInfo, + ToolInfo +) from galaxy.util import ( DATABASE_MAX_STRING_SIZE, ExecutionTimer, @@ -394,9 +398,8 @@ class BaseJobRunner(object): compute_tmp_directory = job_wrapper.tmp_directory() tool = job_wrapper.tool - from galaxy.tools.deps import containers - tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) - job_info = containers.JobInfo( + tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) + job_info = JobInfo( compute_working_directory, compute_tool_directory, compute_job_directory, diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index b51441fff82..0b291380768 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -14,7 +14,7 @@ from galaxy.util import ( ) from galaxy.util.oset import OrderedSet from .container_resolvers import ContainerResolver -from .containers import ToolInfo +from .dependencies import ToolInfo from .requirements import ( ContainerDescription, ToolRequirement, diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index 124d0fd9a77..39b2ec49578 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -266,58 +266,6 @@ class ContainerRegistry(object): return None -class AppInfo(object): - - def __init__( - self, - galaxy_root_dir=None, - default_file_path=None, - outputs_to_working_directory=False, - container_image_cache_path=None, - library_import_dir=None, - enable_beta_mulled_containers=False, - containers_resolvers_config_file=None, - involucro_path=None, - involucro_auto_init=True, - ): - self.galaxy_root_dir = galaxy_root_dir - self.default_file_path = default_file_path - # TODO: Vary default value for docker_volumes based on this... - self.outputs_to_working_directory = outputs_to_working_directory - self.container_image_cache_path = container_image_cache_path - self.library_import_dir = library_import_dir - self.enable_beta_mulled_containers = enable_beta_mulled_containers - self.containers_resolvers_config_file = containers_resolvers_config_file - self.involucro_path = involucro_path - self.involucro_auto_init = involucro_auto_init - - -class ToolInfo(object): - # TODO: Introduce tool XML syntax to annotate the optional environment - # variables they can consume (e.g. JVM options, license keys, etc..) - # and add these to env_path_through - - def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]): - self.container_descriptions = container_descriptions - self.requirements = requirements - self.requires_galaxy_python_environment = requires_galaxy_python_environment - self.env_pass_through = env_pass_through - - -class JobInfo(object): - - def __init__( - self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type - ): - self.working_directory = working_directory - self.job_directory = job_directory - # Tool files may be remote staged - so this is unintuitively a property - # of the job not of the tool. - self.tool_directory = tool_directory - self.tmp_directory = tmp_directory - self.job_directory_type = job_directory_type # "galaxy" or "pulsar" - - @six.add_metaclass(ABCMeta) class Container(object): diff --git a/lib/galaxy/tools/deps/dependencies.py b/lib/galaxy/tools/deps/dependencies.py index 35aafcfb329..028e1cf84d0 100644 --- a/lib/galaxy/tools/deps/dependencies.py +++ b/lib/galaxy/tools/deps/dependencies.py @@ -2,6 +2,58 @@ from galaxy.tools.deps.requirements import ToolRequirements from galaxy.util import bunch +class AppInfo(object): + + def __init__( + self, + galaxy_root_dir=None, + default_file_path=None, + outputs_to_working_directory=False, + container_image_cache_path=None, + library_import_dir=None, + enable_beta_mulled_containers=False, + containers_resolvers_config_file=None, + involucro_path=None, + involucro_auto_init=True, + ): + self.galaxy_root_dir = galaxy_root_dir + self.default_file_path = default_file_path + # TODO: Vary default value for docker_volumes based on this... + self.outputs_to_working_directory = outputs_to_working_directory + self.container_image_cache_path = container_image_cache_path + self.library_import_dir = library_import_dir + self.enable_beta_mulled_containers = enable_beta_mulled_containers + self.containers_resolvers_config_file = containers_resolvers_config_file + self.involucro_path = involucro_path + self.involucro_auto_init = involucro_auto_init + + +class ToolInfo(object): + # TODO: Introduce tool XML syntax to annotate the optional environment + # variables they can consume (e.g. JVM options, license keys, etc..) + # and add these to env_path_through + + def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]): + self.container_descriptions = container_descriptions + self.requirements = requirements + self.requires_galaxy_python_environment = requires_galaxy_python_environment + self.env_pass_through = env_pass_through + + +class JobInfo(object): + + def __init__( + self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type + ): + self.working_directory = working_directory + self.job_directory = job_directory + # Tool files may be remote staged - so this is unintuitively a property + # of the job not of the tool. + self.tool_directory = tool_directory + self.tmp_directory = tmp_directory + self.job_directory_type = job_directory_type # "galaxy" or "pulsar" + + class DependenciesDescription(object): """ Capture (in a readily serializable way) context related a tool dependencies - both the tool's listed requirements and the tool shed From 3b4b510ad01f48253587db1bcd87400237709673 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 10:58:46 +0100 Subject: [PATCH 09/50] Add `can_uninstall_dependencies` attribute to ContainerResolver --- lib/galaxy/tools/deps/container_resolvers/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/__init__.py b/lib/galaxy/tools/deps/container_resolvers/__init__.py index 485560bbec4..337b0328a4d 100644 --- a/lib/galaxy/tools/deps/container_resolvers/__init__.py +++ b/lib/galaxy/tools/deps/container_resolvers/__init__.py @@ -16,7 +16,8 @@ class ContainerResolver(Dictifiable): """Description of a technique for resolving container images for tool execution.""" # Keys for dictification. - dict_collection_visible_keys = ['resolver_type'] + dict_collection_visible_keys = ['resolver_type', 'can_uninstall_dependencies'] + can_uninstall_dependencies = False def __init__(self, app_info=None, **kwds): """Default initializer for ``ContainerResolver`` subclasses.""" From a4f18d8f601c83826a451d63af2fa67d319f58a7 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 18:48:24 +0100 Subject: [PATCH 10/50] Make DependencyManager container & destination-aware This will allow us to use the proper parameters when pulling containers. --- lib/galaxy/config.py | 10 ++++++++ lib/galaxy/tools/deps/__init__.py | 29 ++++++++++++++++++++-- lib/galaxy/tools/deps/container_classes.py | 0 lib/galaxy/tools/deps/views.py | 5 ---- 4 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 lib/galaxy/tools/deps/container_classes.py diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index c0bf4097182..4da8731565f 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -4,6 +4,7 @@ Universe configuration builder. # absolute_import needed for tool_shed package. from __future__ import absolute_import +import collections import ipaddress import logging import logging.config @@ -1062,6 +1063,7 @@ class ConfiguresGalaxyMixin(object): self.container_finder = containers.ContainerFinder(app_info) self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes) self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers) + self._set_enabled_container_types() index_help = getattr(self.config, "index_tool_help", True) self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_help) self.reindex_tool_search() @@ -1071,6 +1073,14 @@ class ConfiguresGalaxyMixin(object): self.toolbox_search.build_index(tool_cache=self.tool_cache) self.tool_cache.reset_status() + def _set_enabled_container_types(self): + container_types_to_destinations = collections.defaultdict(list) + for destinations in self.job_config.destinations.values(): + for destination in destinations: + for enabled_container_type in self.container_finder._enabled_container_types(destination.params): + container_types_to_destinations[enabled_container_type].append(destination) + self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) + def _configure_tool_data_tables(self, from_shed_config): from galaxy.tools.data import ToolDataTableManager diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 0b291380768..b76653973d2 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -52,6 +52,10 @@ def build_dependency_manager(config): class NullDependencyManager(object): dependency_resolvers = [] + enabled_container_types = [] + + def set_enabled_container_types(self, enabled_container_types): + return def uses_tool_shed_dependencies(self): return False @@ -88,6 +92,27 @@ class DependencyManager(object): self.default_base_path = os.path.abspath(default_base_path) self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers(conf_file) + self._enabled_container_types = [] + self._destination_for_container_type = {} + + def set_enabled_container_types(self, container_types_to_destinations): + """Set the union of all enabled container types.""" + self._enabled_container_types = [container_type for container_type in container_types_to_destinations.keys()] + # Just pick first enabled destination for a container type, probably covers the most common deployment scenarios + self._destination_for_container_type = container_types_to_destinations + + def get_destination_info_for_container_type(self, container_type, destination_id=None): + if destination_id is None: + return next(iter(self._destination_for_container_type[container_type])).params + else: + for destination in self._destination_for_container_type[container_type]: + if destination.id == destination_id: + return destination.params + + @property + def enabled_container_types(self): + """Returns the union of enabled container types.""" + return self._enabled_container_types def get_resolver_option(self, resolver, key, explicit_resolver_options={}): """Look in resolver-specific settings for option and then fallback to global settings. @@ -137,7 +162,6 @@ class DependencyManager(object): index = kwds.get('index', None) require_exact = kwds.get('exact', False) return_null_dependencies = kwds.get('return_null', False) - enabled_container_types = kwds.pop('enabled_container_types', []) resolvable_requirements = requirements.resolvable tool_info = ToolInfo(requirements=resolvable_requirements) @@ -172,7 +196,8 @@ class DependencyManager(object): if all_unmet and resolve is not None: # TODO: Handle specs. dependencies = resolve(requirements=resolvable_requirements, - enabled_container_types=enabled_container_types, + enabled_container_types=self.enabled_container_types, + destination_for_container_type=self.get_destination_info_for_container_type, tool_info=tool_info, **kwds) if dependencies: diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 5191c501e33..14ecfce0dab 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -46,15 +46,10 @@ class DependencyResolversView(object): """ Resolves dependencies to build a requirements status in the admin panel/API """ - enabled_container_types = set() - for destinations in self._app.job_config.destinations.values(): - for destination in destinations: - enabled_container_types.update(self._app.container_finder._enabled_container_types(destination.params)) kwds = {'install': False, 'return_null': True, 'installed_tool_dependencies': installed_tool_dependencies} dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, - enabled_container_types=enabled_container_types, **kwds) for tool, requirements in tool_requirements_d.items()} return dependencies_per_tool From 2d3ab4ce1205aec24664225d74b2a851cb94b2b0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 18:58:30 +0100 Subject: [PATCH 11/50] Allow specifying resolver index and resolver_type when installing dependencies via tools API --- lib/galaxy/tools/deps/__init__.py | 13 +++++++++---- lib/galaxy/tools/deps/views.py | 4 ++-- lib/galaxy/webapps/galaxy/api/tools.py | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index b76653973d2..1defc4c3917 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -156,10 +156,12 @@ class DependencyManager(object): return requirement_to_dependency - def _requirements_to_dependencies_dict(self, requirements, **kwds): + def _requirements_to_dependencies_dict(self, requirements, search=False, **kwds): """Build simple requirements to dependencies dict for resolution.""" requirement_to_dependency = OrderedDict() - index = kwds.get('index', None) + index = kwds.get('index') + install = kwds.get('install', False) + resolver_type = kwds.get('resolver_type') require_exact = kwds.get('exact', False) return_null_dependencies = kwds.get('return_null', False) @@ -173,6 +175,9 @@ class DependencyManager(object): if index is not None and i != index: continue + if resolver_type is not None and resolver.resolver_type != resolver_type: + continue + if len(_requirement_to_dependency) == len(resolvable_requirements): # Shortcut - resolution complete. break @@ -186,9 +191,9 @@ class DependencyManager(object): if hasattr(resolver, "resolve_all"): resolve = resolver.resolve_all elif isinstance(resolver, ContainerResolver): - if not resolver.resolver_type.startswith(('cached', 'explicit')): + if not resolver.resolver_type.startswith(('cached', 'explicit')) and not (search or install): # These would look up available containers using the quay API, - # we only want ot do this if we search for containers + # we only want to do this if we search for containers continue resolve = resolver.resolve else: diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index 14ecfce0dab..fce45eb317d 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -99,8 +99,8 @@ class DependencyResolversView(object): envs_to_remove = envs_to_remove.difference(can_remove) return list(removed_environments) - def install_dependencies(self, requirements): - return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True}) + def install_dependencies(self, requirements, **kwds): + return self._dependency_manager._requirements_to_dependencies_dict(requirements, **kwds) def install_dependency(self, index=None, **payload): """ diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index c439f0d2041..2f8c192f8f6 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -253,7 +253,8 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): force_rebuild: If true and cache dir exists, attempts to delete cache dir """ tool = self._get_tool(id) - tool._view.install_dependencies(tool.requirements) + kwds['install'] = True + tool._view.install_dependencies(tool.requirements, **kwds) if kwds.get('build_dependency_cache'): tool.build_dependency_cache(**kwds) # TODO: rework resolver install system to log and report what has been done. From 7ca17e2b83d45e3c5a63643cc59785f6162803c5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 19:01:22 +0100 Subject: [PATCH 12/50] Drop NullDependency only when not skipping resolver --- lib/galaxy/tools/deps/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 1defc4c3917..d721cd3288c 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -170,14 +170,14 @@ class DependencyManager(object): for i, resolver in enumerate(self.dependency_resolvers): - _requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)]) - if index is not None and i != index: continue if resolver_type is not None and resolver.resolver_type != resolver_type: continue + _requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)]) + if len(_requirement_to_dependency) == len(resolvable_requirements): # Shortcut - resolution complete. break From 59c56b8f563fd4298d37d40af0b0f429761c4392 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 19:03:01 +0100 Subject: [PATCH 13/50] Mention index and resolver_type in API docstring --- lib/galaxy/webapps/galaxy/api/tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 2f8c192f8f6..3eb5c802bfd 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -249,6 +249,9 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): Attempts to install requirements via the dependency resolver parameters: + index: index of dependency resolver to use when installing dependency. + Defaults to using the highest ranking resolver + resolver_type: Use the dependency resolver of this resolver_type to install dependency. build_dependency_cache: If true, attempts to cache dependencies for this tool force_rebuild: If true and cache dir exists, attempts to delete cache dir """ From 2aab490807245c30935389001605c170bbebbf98 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 19:04:02 +0100 Subject: [PATCH 14/50] Move Container classes into container_classes to avoid circular imports --- lib/galaxy/tools/deps/container_classes.py | 347 +++++++++++++++++++++ lib/galaxy/tools/deps/containers.py | 339 +------------------- lib/galaxy/tools/deps/dockerfiles.py | 2 +- 3 files changed, 354 insertions(+), 334 deletions(-) diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py index e69de29bb2d..822af86138e 100644 --- a/lib/galaxy/tools/deps/container_classes.py +++ b/lib/galaxy/tools/deps/container_classes.py @@ -0,0 +1,347 @@ +import os +import string +from abc import ( + ABCMeta, + abstractmethod +) + +import six + +from galaxy.containers.docker_model import DockerVolume +from galaxy.util import ( + asbool, + in_directory +) +from . import ( + docker_util, + singularity_util +) +from .requirements import ( + DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, + DEFAULT_CONTAINER_SHELL, +) + +DOCKER_CONTAINER_TYPE = "docker" +SINGULARITY_CONTAINER_TYPE = "singularity" + +LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r''' +python << EOF +from __future__ import print_function + +import json +import re +import subprocess +import tarfile + +t = tarfile.TarFile("${cached_image_file}") +meta_str = t.extractfile('repositories').read() +meta = json.loads(meta_str) +tag, tag_value = meta.items()[0] +rev, rev_value = tag_value.items()[0] +cmd = "${images_cmd}" +proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) +stdo, stde = proc.communicate() +found = False +for line in stdo.split("\n"): + tmp = re.split(r'\s+', line) + if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value: + found = True +if not found: + print("Loading image") + cmd = "cat ${cached_image_file} | ${load_cmd}" + subprocess.check_call(cmd, shell=True) +EOF +''' + + +@six.add_metaclass(ABCMeta) +class Container(object): + + def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description): + self.container_id = container_id + self.app_info = app_info + self.tool_info = tool_info + self.destination_info = destination_info + self.job_info = job_info + self.container_description = container_description + + def prop(self, name, default): + destination_name = "docker_%s" % name + return self.destination_info.get(destination_name, default) + + @property + def resolve_dependencies(self): + return DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES if not self.container_description else self.container_description.resolve_dependencies + + @property + def shell(self): + return DEFAULT_CONTAINER_SHELL if not self.container_description else self.container_description.shell + + @abstractmethod + def containerize_command(self, command): + """ + Use destination supplied container configuration parameters, + container_id, and command to build a new command that runs + input command in container. + """ + + +def preprocess_volumes(volumes_raw_str, container_type): + """Process Galaxy volume specification string to either Docker or Singularity specification. + + Galaxy allows the mount try "default_ro" which translates to ro for Docker and + ro for Singularity iff no subdirectories are rw (Singularity does not allow ro + parent directories with rw subdirectories). + + >>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE) + ['/a/b:rw'] + >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) + ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) + ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) + ['/a/b:rw', '/a/b/c:rw'] + """ + + volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")] + volumes = [] + rw_paths = [] + + for volume_raw_str in volumes_raw_strs: + volume_parts = volume_raw_str.split(":") + if len(volume_parts) > 2: + raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str) + if len(volume_parts) == 1: + volume_parts.append("rw") + volumes.append(volume_parts) + if volume_parts[1] == "rw": + rw_paths.append(volume_parts[0]) + + for volume in volumes: + path = volume[0] + how = volume[1] + + if how == "default_ro": + how = "ro" + if container_type == SINGULARITY_CONTAINER_TYPE: + for rw_path in rw_paths: + if in_directory(rw_path, path): + how = "rw" + + volume[1] = how + + return [":".join(v) for v in volumes] + + +class HasDockerLikeVolumes(object): + """Mixin to share functionality related to Docker volume handling. + + Singularity seems to have a fairly compatible syntax for volume handling. + """ + + def _expand_volume_str(self, value): + if not value: + return value + + template = string.Template(value) + variables = dict() + + def add_var(name, value): + if value: + if not value.startswith("$"): + value = os.path.abspath(value) + variables[name] = value + + add_var("working_directory", self.job_info.working_directory) + add_var("tmp_directory", self.job_info.tmp_directory) + add_var("job_directory", self.job_info.job_directory) + add_var("tool_directory", self.job_info.tool_directory) + add_var("galaxy_root", self.app_info.galaxy_root_dir) + add_var("default_file_path", self.app_info.default_file_path) + add_var("library_import_dir", self.app_info.library_import_dir) + + if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar": + # We have a Pulsar job directory, so everything needed (excluding index + # files) should be available in job_directory... + defaults = "$job_directory:default_ro,$tool_directory:default_ro,$job_directory/outputs:rw,$working_directory:rw" + else: + defaults = "$galaxy_root:default_ro,$tool_directory:default_ro" + if self.job_info.job_directory: + defaults += ",$job_directory:default_ro" + if self.job_info.tmp_directory is not None: + defaults += ",$tmp_directory:rw" + if self.app_info.outputs_to_working_directory: + # Should need default_file_path (which is of course an estimate given + # object stores anyway). + defaults += ",$working_directory:rw,$default_file_path:default_ro" + else: + defaults += ",$working_directory:rw,$default_file_path:rw" + + if self.app_info.library_import_dir: + defaults += ",$library_import_dir:default_ro" + + # Define $defaults that can easily be extended with external library and + # index data without deployer worrying about above details. + variables["defaults"] = string.Template(defaults).safe_substitute(variables) + + return template.safe_substitute(variables) + + +class DockerContainer(Container, HasDockerLikeVolumes): + + container_type = DOCKER_CONTAINER_TYPE + + @property + def docker_host_props(self): + docker_host_props = dict( + docker_cmd=self.prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND), + sudo=asbool(self.prop("sudo", docker_util.DEFAULT_SUDO)), + sudo_cmd=self.prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND), + host=self.prop("host", docker_util.DEFAULT_HOST), + ) + return docker_host_props + + def build_pull_command(self): + return docker_util.build_pull_command(self.container_id, **self.docker_host_props) + + def containerize_command(self, command): + env_directives = [] + for pass_through_var in self.tool_info.env_pass_through: + env_directives.append('"%s=$%s"' % (pass_through_var, pass_through_var)) + + # Allow destinations to explicitly set environment variables just for + # docker container. Better approach is to set for destination and then + # pass through only what tool needs however. (See todo in ToolInfo.) + for key, value in six.iteritems(self.destination_info): + if key.startswith("docker_env_"): + env = key[len("docker_env_"):] + env_directives.append('"%s=%s"' % (env, value)) + + working_directory = self.job_info.working_directory + if not working_directory: + raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) + + volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults")) + preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) + # TODO: Remove redundant volumes... + volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] + # If a tool definitely has a temp directory available set it to /tmp in container for compat. + # with CWL. This is part of that spec and should make it easier to share containers between CWL + # and Galaxy. + if self.job_info.tmp_directory is not None: + volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory)) + volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM) + + docker_host_props = self.docker_host_props + + cached_image_file = self.__get_cached_image_file() + if not cached_image_file: + # TODO: Add option to cache it once here and create cached_image_file. + cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) + else: + cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props) + run_command = docker_util.build_docker_run_command( + command, + self.container_id, + volumes=volumes, + volumes_from=volumes_from, + env_directives=env_directives, + working_directory=working_directory, + net=self.prop("net", "none"), # By default, docker instance has networking disabled + auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)), + set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER), + run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS), + **docker_host_props + ) + return "%s\n%s" % (cache_command, run_command) + + def __cache_from_file_command(self, cached_image_file, docker_host_props): + images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props) + load_cmd = docker_util.build_docker_load_command(**docker_host_props) + + return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute( + cached_image_file=cached_image_file, + images_cmd=images_cmd, + load_cmd=load_cmd + ) + + def __get_cached_image_file(self): + container_id = self.container_id + cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path")) + cache_path = docker_cache_path(cache_directory, container_id) + return cache_path if os.path.exists(cache_path) else None + + def __get_destination_overridable_property(self, name): + prop_name = "docker_%s" % name + if prop_name in self.destination_info: + return self.destination_info[prop_name] + else: + return getattr(self.app_info, name) + + +def docker_cache_path(cache_directory, container_id): + file_container_id = container_id.replace("/", "_slash_") + cache_file_name = "docker_%s.tar" % file_container_id + return os.path.join(cache_directory, cache_file_name) + + +class SingularityContainer(Container, HasDockerLikeVolumes): + + container_type = SINGULARITY_CONTAINER_TYPE + + def containerize_command(self, command): + + env = [] + for pass_through_var in self.tool_info.env_pass_through: + env.append((pass_through_var, "$%s" % pass_through_var)) + + # Allow destinations to explicitly set environment variables just for + # docker container. Better approach is to set for destination and then + # pass through only what tool needs however. (See todo in ToolInfo.) + for key, value in six.iteritems(self.destination_info): + if key.startswith("singularity_env_"): + real_key = key[len("singularity_env_"):] + env.append((real_key, value)) + + working_directory = self.job_info.working_directory + if not working_directory: + raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) + + volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults")) + preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) + volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] + + singularity_target_kwds = dict( + singularity_cmd=self.prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND), + sudo=asbool(self.prop("sudo", singularity_util.DEFAULT_SUDO)), + sudo_cmd=self.prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND), + ) + run_command = singularity_util.build_singularity_run_command( + command, + self.container_id, + volumes=volumes, + env=env, + working_directory=working_directory, + run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS), + **singularity_target_kwds + ) + return run_command + + +CONTAINER_CLASSES = dict( + docker=DockerContainer, + singularity=SingularityContainer, +) + + +class NullContainer(object): + + def __init__(self): + pass + + def __bool__(self): + return False + __nonzero__ = __bool__ + + +NULL_CONTAINER = NullContainer() diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index 39b2ec49578..7417fa8575c 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -1,22 +1,15 @@ import logging import os -import string -from abc import ( - ABCMeta, - abstractmethod -) -import six - -from galaxy.containers.docker_model import DockerVolume from galaxy.util import ( asbool, - in_directory, plugin_config ) -from . import ( - docker_util, - singularity_util +from .container_classes import ( + CONTAINER_CLASSES, + DOCKER_CONTAINER_TYPE, + NULL_CONTAINER, + SINGULARITY_CONTAINER_TYPE, ) from .container_resolvers.explicit import ExplicitContainerResolver from .container_resolvers.mulled import ( @@ -28,46 +21,14 @@ from .container_resolvers.mulled import ( ) from .requirements import ( ContainerDescription, - DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, - DEFAULT_CONTAINER_SHELL ) log = logging.getLogger(__name__) -DOCKER_CONTAINER_TYPE = "docker" -SINGULARITY_CONTAINER_TYPE = "singularity" + DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE] -LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r''' -python << EOF -from __future__ import print_function - -import json -import re -import subprocess -import tarfile - -t = tarfile.TarFile("${cached_image_file}") -meta_str = t.extractfile('repositories').read() -meta = json.loads(meta_str) -tag, tag_value = meta.items()[0] -rev, rev_value = tag_value.items()[0] -cmd = "${images_cmd}" -proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) -stdo, stde = proc.communicate() -found = False -for line in stdo.split("\n"): - tmp = re.split(r'\s+', line) - if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value: - found = True -if not found: - print("Loading image") - cmd = "cat ${cached_image_file} | ${load_cmd}" - subprocess.check_call(cmd, shell=True) -EOF -''' - class ContainerFinder(object): @@ -264,291 +225,3 @@ class ContainerRegistry(object): return container_description return None - - -@six.add_metaclass(ABCMeta) -class Container(object): - - def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description): - self.container_id = container_id - self.app_info = app_info - self.tool_info = tool_info - self.destination_info = destination_info - self.job_info = job_info - self.container_description = container_description - - @property - def resolve_dependencies(self): - return DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES if not self.container_description else self.container_description.resolve_dependencies - - @property - def shell(self): - return DEFAULT_CONTAINER_SHELL if not self.container_description else self.container_description.shell - - @abstractmethod - def containerize_command(self, command): - """ - Use destination supplied container configuration parameters, - container_id, and command to build a new command that runs - input command in container. - """ - - -def preprocess_volumes(volumes_raw_str, container_type): - """Process Galaxy volume specification string to either Docker or Singularity specification. - - Galaxy allows the mount try "default_ro" which translates to ro for Docker and - ro for Singularity iff no subdirectories are rw (Singularity does not allow ro - parent directories with rw subdirectories). - - >>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE) - ['/a/b:rw'] - >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) - ['/a/b:ro', '/a/b/c:rw'] - >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) - ['/a/b:ro', '/a/b/c:rw'] - >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) - ['/a/b:rw', '/a/b/c:rw'] - """ - - volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")] - volumes = [] - rw_paths = [] - - for volume_raw_str in volumes_raw_strs: - volume_parts = volume_raw_str.split(":") - if len(volume_parts) > 2: - raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str) - if len(volume_parts) == 1: - volume_parts.append("rw") - volumes.append(volume_parts) - if volume_parts[1] == "rw": - rw_paths.append(volume_parts[0]) - - for volume in volumes: - path = volume[0] - how = volume[1] - - if how == "default_ro": - how = "ro" - if container_type == SINGULARITY_CONTAINER_TYPE: - for rw_path in rw_paths: - if in_directory(rw_path, path): - how = "rw" - - volume[1] = how - - return [":".join(v) for v in volumes] - - -class HasDockerLikeVolumes(object): - """Mixin to share functionality related to Docker volume handling. - - Singularity seems to have a fairly compatible syntax for volume handling. - """ - - def _expand_volume_str(self, value): - if not value: - return value - - template = string.Template(value) - variables = dict() - - def add_var(name, value): - if value: - if not value.startswith("$"): - value = os.path.abspath(value) - variables[name] = value - - add_var("working_directory", self.job_info.working_directory) - add_var("tmp_directory", self.job_info.tmp_directory) - add_var("job_directory", self.job_info.job_directory) - add_var("tool_directory", self.job_info.tool_directory) - add_var("galaxy_root", self.app_info.galaxy_root_dir) - add_var("default_file_path", self.app_info.default_file_path) - add_var("library_import_dir", self.app_info.library_import_dir) - - if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar": - # We have a Pulsar job directory, so everything needed (excluding index - # files) should be available in job_directory... - defaults = "$job_directory:default_ro,$tool_directory:default_ro,$job_directory/outputs:rw,$working_directory:rw" - else: - defaults = "$galaxy_root:default_ro,$tool_directory:default_ro" - if self.job_info.job_directory: - defaults += ",$job_directory:default_ro" - if self.job_info.tmp_directory is not None: - defaults += ",$tmp_directory:rw" - if self.app_info.outputs_to_working_directory: - # Should need default_file_path (which is of course an estimate given - # object stores anyway). - defaults += ",$working_directory:rw,$default_file_path:default_ro" - else: - defaults += ",$working_directory:rw,$default_file_path:rw" - - if self.app_info.library_import_dir: - defaults += ",$library_import_dir:default_ro" - - # Define $defaults that can easily be extended with external library and - # index data without deployer worrying about above details. - variables["defaults"] = string.Template(defaults).safe_substitute(variables) - - return template.safe_substitute(variables) - - -class DockerContainer(Container, HasDockerLikeVolumes): - - container_type = DOCKER_CONTAINER_TYPE - - def containerize_command(self, command): - def prop(name, default): - destination_name = "docker_%s" % name - return self.destination_info.get(destination_name, default) - - env_directives = [] - for pass_through_var in self.tool_info.env_pass_through: - env_directives.append('"%s=$%s"' % (pass_through_var, pass_through_var)) - - # Allow destinations to explicitly set environment variables just for - # docker container. Better approach is to set for destination and then - # pass through only what tool needs however. (See todo in ToolInfo.) - for key, value in six.iteritems(self.destination_info): - if key.startswith("docker_env_"): - env = key[len("docker_env_"):] - env_directives.append('"%s=%s"' % (env, value)) - - working_directory = self.job_info.working_directory - if not working_directory: - raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) - - volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults")) - preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) - # TODO: Remove redundant volumes... - volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] - # If a tool definitely has a temp directory available set it to /tmp in container for compat. - # with CWL. This is part of that spec and should make it easier to share containers between CWL - # and Galaxy. - if self.job_info.tmp_directory is not None: - volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory)) - volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM) - - docker_host_props = dict( - docker_cmd=prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND), - sudo=asbool(prop("sudo", docker_util.DEFAULT_SUDO)), - sudo_cmd=prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND), - host=prop("host", docker_util.DEFAULT_HOST), - ) - - cached_image_file = self.__get_cached_image_file() - if not cached_image_file: - # TODO: Add option to cache it once here and create cached_image_file. - cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) - else: - cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props) - run_command = docker_util.build_docker_run_command( - command, - self.container_id, - volumes=volumes, - volumes_from=volumes_from, - env_directives=env_directives, - working_directory=working_directory, - net=prop("net", "none"), # By default, docker instance has networking disabled - auto_rm=asbool(prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)), - set_user=prop("set_user", docker_util.DEFAULT_SET_USER), - run_extra_arguments=prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS), - **docker_host_props - ) - return "%s\n%s" % (cache_command, run_command) - - def __cache_from_file_command(self, cached_image_file, docker_host_props): - images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props) - load_cmd = docker_util.build_docker_load_command(**docker_host_props) - - return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute( - cached_image_file=cached_image_file, - images_cmd=images_cmd, - load_cmd=load_cmd - ) - - def __get_cached_image_file(self): - container_id = self.container_id - cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path")) - cache_path = docker_cache_path(cache_directory, container_id) - return cache_path if os.path.exists(cache_path) else None - - def __get_destination_overridable_property(self, name): - prop_name = "docker_%s" % name - if prop_name in self.destination_info: - return self.destination_info[prop_name] - else: - return getattr(self.app_info, name) - - -def docker_cache_path(cache_directory, container_id): - file_container_id = container_id.replace("/", "_slash_") - cache_file_name = "docker_%s.tar" % file_container_id - return os.path.join(cache_directory, cache_file_name) - - -class SingularityContainer(Container, HasDockerLikeVolumes): - - container_type = SINGULARITY_CONTAINER_TYPE - - def containerize_command(self, command): - def prop(name, default): - destination_name = "singularity_%s" % name - return self.destination_info.get(destination_name, default) - - env = [] - for pass_through_var in self.tool_info.env_pass_through: - env.append((pass_through_var, "$%s" % pass_through_var)) - - # Allow destinations to explicitly set environment variables just for - # docker container. Better approach is to set for destination and then - # pass through only what tool needs however. (See todo in ToolInfo.) - for key, value in six.iteritems(self.destination_info): - if key.startswith("singularity_env_"): - real_key = key[len("singularity_env_"):] - env.append((real_key, value)) - - working_directory = self.job_info.working_directory - if not working_directory: - raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) - - volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults")) - preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) - volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] - - singularity_target_kwds = dict( - singularity_cmd=prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND), - sudo=asbool(prop("sudo", singularity_util.DEFAULT_SUDO)), - sudo_cmd=prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND), - ) - run_command = singularity_util.build_singularity_run_command( - command, - self.container_id, - volumes=volumes, - env=env, - working_directory=working_directory, - run_extra_arguments=prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS), - **singularity_target_kwds - ) - return run_command - - -CONTAINER_CLASSES = dict( - docker=DockerContainer, - singularity=SingularityContainer, -) - - -class NullContainer(object): - - def __init__(self): - pass - - def __bool__(self): - return False - __nonzero__ = __bool__ - - -NULL_CONTAINER = NullContainer() diff --git a/lib/galaxy/tools/deps/dockerfiles.py b/lib/galaxy/tools/deps/dockerfiles.py index bb0a292235c..67944dcbc04 100644 --- a/lib/galaxy/tools/deps/dockerfiles.py +++ b/lib/galaxy/tools/deps/dockerfiles.py @@ -3,7 +3,7 @@ import os from ..deps import commands from ..deps import docker_util -from ..deps.containers import docker_cache_path +from ..deps.container_classes import docker_cache_path from ..deps.requirements import parse_requirements_from_xml from ...tools import loader_directory From 55f989c23650e94e1f28a3ac60d10931dfd70076 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 19:04:29 +0100 Subject: [PATCH 15/50] Make mulled docker containers installable --- .../tools/deps/container_resolvers/mulled.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index 2a993e93c91..c35640fff2d 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -8,6 +8,8 @@ import subprocess import six from galaxy.util import unicodify +from ..commands import shell +from ..container_classes import CONTAINER_CLASSES from ..container_resolvers import ( ContainerResolver, ) @@ -271,7 +273,7 @@ class MulledDockerContainerResolver(ContainerResolver): self.namespace = namespace self.hash_func = hash_func - def resolve(self, enabled_container_types, tool_info, **kwds): + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): if tool_info.requires_galaxy_python_environment: return None @@ -327,10 +329,21 @@ class MulledDockerContainerResolver(ContainerResolver): name = "%s:%s" % (base_image_name, tags[0]) if name: - return ContainerDescription( + container_description = ContainerDescription( "quay.io/%s/%s" % (self.namespace, name), type=self.container_type, ) + destination_for_container_type = kwds.get('destination_for_container_type') + if install and destination_for_container_type and not docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func): + container = CONTAINER_CLASSES[self.container_type](container_description.identifier, + self.app_info, + tool_info, + destination_for_container_type(self.container_type), + {}, + container_description) + command = container.build_pull_command() + shell(command) + return container_description def __str__(self): return "MulledDockerContainerResolver[namespace=%s]" % self.namespace From 19bc4985c91c14295d7c2048355a0bf71d4ec1cd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 19:35:17 +0100 Subject: [PATCH 16/50] Allow selecting resolver when uninstalling through tools API --- lib/galaxy/tools/deps/views.py | 6 +++++- lib/galaxy/webapps/galaxy/api/tools.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index fce45eb317d..a6f41b84606 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -54,7 +54,7 @@ class DependencyResolversView(object): for tool, requirements in tool_requirements_d.items()} return dependencies_per_tool - def uninstall_dependencies(self, index=None, **payload): + def uninstall_dependencies(self, index=None, resolver_type=None, **payload): """Attempt to uninstall requirements. Returns 0 if successfull, else None.""" requirements = payload.get('requirements') if not requirements: @@ -63,6 +63,10 @@ class DependencyResolversView(object): resolver = self._dependency_resolvers[index] if resolver.can_uninstall_dependencies: return resolver.uninstall(requirements) + elif resolver_type: + for resolver in self._dependency_resolvers: + if resolver.resolver_type == resolver_type and resolver.can_uninstall_dependencies: + return resolver.uninstall(requirements) else: for index in self.uninstallable_resolvers: return_code = self._dependency_resolvers[index].uninstall(requirements) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 3eb5c802bfd..339e550188f 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -271,9 +271,13 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): DELETE /api/tools/{tool_id}/dependencies Attempts to uninstall requirements via the dependency resolver + parameters: + index: index of dependency resolver to use when installing dependency. + Defaults to using the highest ranking resolver + resolver_type: Use the dependency resolver of this resolver_type to install dependency """ tool = self._get_tool(id) - tool._view.uninstall_dependencies(index=None, requirements=tool.requirements) + tool._view.uninstall_dependencies(requirements=tool.requirements, **kwds) # TODO: rework resolver install system to log and report what has been done. return tool.tool_requirements_status From 50d8e3079d873fe4161149b00365cf0d5702eb56 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Dec 2018 19:35:44 +0100 Subject: [PATCH 17/50] Add integration test for un/install with resolver_type --- test/integration/test_resolvers.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index acca9fc1ffc..94a55562b8d 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -1,4 +1,4 @@ -"""Integration tests for conda dependency resolution.""" +"""Integration tests for dependency resolution.""" import os from tempfile import mkdtemp @@ -125,6 +125,29 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase): response = create_response.json() assert not [True for d in response if d['dependency_type'] == 'conda'] + def _uninstall_mulled_example_multi_1(self, resolver_type=None): + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/dependencies" % tool_id + data = {'id': tool_id, 'resolver_type': resolver_type} + create_response = self._delete(endpoint, data=data, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert not [True for d in response if d['dependency_type'] == 'conda'] + + def test_conda_install_with_resolver_type_via_tools_api(self): + # Makes sure dependency is not already installed + self._uninstall_mulled_example_multi_1(resolver_type='conda') + # Now do the actual test + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/dependencies" % tool_id + data = {'id': tool_id, 'resolver_type': 'conda'} + create_response = self._post(endpoint, data=data, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert any([True for d in response if d['dependency_type'] == 'conda']) + # Now that we know install was successfullt we can also doube check that the uninstall works + self._uninstall_mulled_example_multi_1(resolver_type='conda') + def test_conda_clean(self): endpoint = 'dependency_resolvers/clean' create_response = self._post(endpoint, data={}, admin=True) From 66a3f556abeb410fd59f1b408efae94f168699d8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 18 Dec 2018 00:04:25 +0100 Subject: [PATCH 18/50] Use singularity build with biocontainers This makes it possible to pull a container from quay.io/biocontainers and to convert it to singularity format for later execution (via the CachedMulledSingularityContainerResolver). This is nicely integrated with the DependencyManager view, so you can (as for conda or docker) do: ``` requests.post(galaxy_url+'/api/tools/deseq2/dependencies', {'key': API_KEY, 'resolver_type': 'mulled_singularity'}) ``` This is quite lightweight, the resulting image is significantly smaller than the conda env (since it is compressed), and the install process is much faster. --- lib/galaxy/tools/deps/container_classes.py | 22 ++- .../tools/deps/container_resolvers/mulled.py | 136 +++++++++++------- lib/galaxy/tools/deps/containers.py | 4 +- lib/galaxy/tools/deps/singularity_util.py | 26 +++- 4 files changed, 129 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py index 822af86138e..ecb10f46362 100644 --- a/lib/galaxy/tools/deps/container_classes.py +++ b/lib/galaxy/tools/deps/container_classes.py @@ -289,6 +289,21 @@ class SingularityContainer(Container, HasDockerLikeVolumes): container_type = SINGULARITY_CONTAINER_TYPE + def get_singularity_target_kwds(self): + return dict( + singularity_cmd=self.prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND), + sudo=asbool(self.prop("sudo", singularity_util.DEFAULT_SUDO)), + sudo_cmd=self.prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND), + ) + + def build_mulled_singularity_pull_command(self, cache_directory, namespace="biocontainers"): + return singularity_util.pull_mulled_singularity_command( + docker_image_identifier=self.container_id, + cache_directory=cache_directory, + namespace=namespace, + **self.get_singularity_target_kwds() + ) + def containerize_command(self, command): env = [] @@ -311,11 +326,6 @@ class SingularityContainer(Container, HasDockerLikeVolumes): preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] - singularity_target_kwds = dict( - singularity_cmd=self.prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND), - sudo=asbool(self.prop("sudo", singularity_util.DEFAULT_SUDO)), - sudo_cmd=self.prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND), - ) run_command = singularity_util.build_singularity_run_command( command, self.container_id, @@ -323,7 +333,7 @@ class SingularityContainer(Container, HasDockerLikeVolumes): env=env, working_directory=working_directory, run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS), - **singularity_target_kwds + **self.get_singularity_target_kwds() ) return run_command diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index c35640fff2d..a4802fc6855 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -261,6 +261,55 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory +def targets_to_mulled_name(targets, hash_func, namespace): + name = None + if len(targets) == 1: + target = targets[0] + target_version = target.version + tags = mulled_tags_for(namespace, target.package_name) + + if not tags: + return None + + if target_version: + for tag in tags: + version, build = split_tag(tag) + if version == target_version: + name = "%s:%s--%s" % (target.package_name, version, build) + break + else: + version, build = split_tag(tags[0]) + name = "%s:%s--%s" % (target.package_name, version, build) + else: + def tags_if_available(image_name): + if ":" in image_name: + repo_name, tag_prefix = image_name.split(":", 2) + else: + repo_name = image_name + tag_prefix = None + tags = mulled_tags_for(namespace, repo_name, tag_prefix=tag_prefix) + return tags + + if hash_func == "v2": + base_image_name = v2_image_name(targets) + tags = tags_if_available(base_image_name) + if tags: + if ":" in base_image_name: + # base_image_name of form :, expand tag + # to include build number in tag. + name = "%s:%s" % (base_image_name.split(":")[0], tags[0]) + else: + # base_image_name of form , simply add build number + # as tag to fully qualify image. + name = "%s:%s" % (base_image_name, tags[0]) + elif hash_func == "v1": + base_image_name = v1_image_name(targets) + tags = tags_if_available(base_image_name) + if tags: + name = "%s:%s" % (base_image_name, tags[0]) + return name + + @six.python_2_unicode_compatible class MulledDockerContainerResolver(ContainerResolver): """Look for mulled images matching tool dependencies.""" @@ -273,6 +322,13 @@ class MulledDockerContainerResolver(ContainerResolver): self.namespace = namespace self.hash_func = hash_func + def cached_container_description(self, targets, namespace, hash_func): + return docker_cached_container_description(targets, namespace, hash_func) + + def pull(self, container): + command = container.build_pull_command() + shell(command) + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): if tool_info.requires_galaxy_python_environment: return None @@ -281,74 +337,54 @@ class MulledDockerContainerResolver(ContainerResolver): if len(targets) == 0: return None - name = None - - if len(targets) == 1: - target = targets[0] - target_version = target.version - tags = mulled_tags_for(self.namespace, target.package_name) - - if not tags: - return None - - if target_version: - for tag in tags: - version, build = split_tag(tag) - if version == target_version: - name = "%s:%s--%s" % (target.package_name, version, build) - break - else: - version, build = split_tag(tags[0]) - name = "%s:%s--%s" % (target.package_name, version, build) - else: - def tags_if_available(image_name): - if ":" in image_name: - repo_name, tag_prefix = image_name.split(":", 2) - else: - repo_name = image_name - tag_prefix = None - tags = mulled_tags_for(self.namespace, repo_name, tag_prefix=tag_prefix) - return tags - - if self.hash_func == "v2": - base_image_name = v2_image_name(targets) - tags = tags_if_available(base_image_name) - if tags: - if ":" in base_image_name: - # base_image_name of form :, expand tag - # to include build number in tag. - name = "%s:%s" % (base_image_name.split(":")[0], tags[0]) - else: - # base_image_name of form , simply add build number - # as tag to fully qualify image. - name = "%s:%s" % (base_image_name, tags[0]) - elif self.hash_func == "v1": - base_image_name = v1_image_name(targets) - tags = tags_if_available(base_image_name) - if tags: - name = "%s:%s" % (base_image_name, tags[0]) - + name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace) if name: container_description = ContainerDescription( "quay.io/%s/%s" % (self.namespace, name), type=self.container_type, ) destination_for_container_type = kwds.get('destination_for_container_type') - if install and destination_for_container_type and not docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func): + if install and destination_for_container_type and not self.cached_container_description(targets, + namespace=self.namespace, + hash_func=self.hash_func): container = CONTAINER_CLASSES[self.container_type](container_description.identifier, self.app_info, tool_info, destination_for_container_type(self.container_type), {}, container_description) - command = container.build_pull_command() - shell(command) + self.pull(container) return container_description def __str__(self): return "MulledDockerContainerResolver[namespace=%s]" % self.namespace +@six.python_2_unicode_compatible +class MulledSingularityContainerResolver(MulledDockerContainerResolver): + + resolver_type = "mulled_singularity" + container_type = "singularity" + + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds): + super(MulledSingularityContainerResolver, self).__init__(app_info) + self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) + self.namespace = namespace + self.hash_func = hash_func + + def cached_container_description(self, targets, namespace, hash_func): + return singularity_cached_container_description(targets, + cache_directory=self.cache_directory, + hash_func=hash_func) + + def pull(self, container): + cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace) + shell(cmds=cmds) + + def __str__(self): + return "MulledSingularityContainerResolver[namespace=%s]" % self.namespace + + @six.python_2_unicode_compatible class BuildMulledDockerContainerResolver(ContainerResolver): """Build for Docker mulled images matching tool dependencies.""" diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index 7417fa8575c..fc43b7d0311 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -18,6 +18,7 @@ from .container_resolvers.mulled import ( CachedMulledDockerContainerResolver, CachedMulledSingularityContainerResolver, MulledDockerContainerResolver, + MulledSingularityContainerResolver, ) from .requirements import ( ContainerDescription, @@ -202,8 +203,9 @@ class ContainerRegistry(object): default_resolvers.extend([ CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"), MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), + MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), BuildMulledDockerContainerResolver(self.app_info), - CachedMulledSingularityContainerResolver(self.app_info), BuildMulledSingularityContainerResolver(self.app_info), ]) return default_resolvers diff --git a/lib/galaxy/tools/deps/singularity_util.py b/lib/galaxy/tools/deps/singularity_util.py index f9ac33a551f..dbdce4a2dbc 100644 --- a/lib/galaxy/tools/deps/singularity_util.py +++ b/lib/galaxy/tools/deps/singularity_util.py @@ -1,5 +1,6 @@ -from six.moves import shlex_quote +import os +from six.moves import shlex_quote DEFAULT_WORKING_DIRECTORY = None DEFAULT_SINGULARITY_COMMAND = "singularity" @@ -8,6 +9,27 @@ DEFAULT_SUDO_COMMAND = "sudo" DEFAULT_RUN_EXTRA_ARGUMENTS = None +def pull_mulled_singularity_command(docker_image_identifier, + cache_directory, + namespace=None, + singularity_cmd=DEFAULT_SINGULARITY_COMMAND, + sudo=DEFAULT_SUDO, + sudo_cmd=DEFAULT_SUDO_COMMAND): + command_parts = [] + command_parts += _singularity_prefix( + singularity_cmd=singularity_cmd, + sudo=sudo, + sudo_cmd=sudo_cmd, + ) + save_path = docker_image_identifier + if namespace: + prefix = "quay.io/%s/" % namespace + if docker_image_identifier.startswith(prefix): + save_path = docker_image_identifier[len(prefix):] + command_parts.extend(["build", os.path.join(cache_directory, save_path), "docker://%s" % docker_image_identifier]) + return command_parts + + def build_singularity_run_command( container_command, image, @@ -56,4 +78,4 @@ def _singularity_prefix( return command_parts -__all__ = ("build_singularity_run_command",) +__all__ = ("build_singularity_run_command", "pull_mulled_singularity_command") From 1fcbd971c8aa743b3e035ddc27a4ea78e01bac0a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 18 Dec 2018 14:34:42 +0100 Subject: [PATCH 19/50] Try cached resolver before non-cached --- config/container_resolvers_conf.xml.sample | 16 ++++++++++++---- lib/galaxy/tools/deps/__init__.py | 2 +- lib/galaxy/tools/deps/containers.py | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/config/container_resolvers_conf.xml.sample b/config/container_resolvers_conf.xml.sample index 40cfd3b1fbd..9e927158ba7 100644 --- a/config/container_resolvers_conf.xml.sample +++ b/config/container_resolvers_conf.xml.sample @@ -3,7 +3,7 @@ - @@ -21,15 +21,23 @@ + + + - + diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index d721cd3288c..03e28c8182d 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -141,7 +141,7 @@ class DependencyManager(object): def dependency_shell_commands(self, requirements, **kwds): requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds) ordered_dependencies = OrderedSet(requirements_to_dependencies.values()) - return [dependency.shell_commands() for dependency in ordered_dependencies] + return [dependency.shell_commands() for dependency in ordered_dependencies if not isinstance(dependency, ContainerDependency)] def requirements_to_dependencies(self, requirements, **kwds): """ diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index fc43b7d0311..e0dafd10321 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -202,8 +202,8 @@ class ContainerRegistry(object): if self.enable_beta_mulled_containers: default_resolvers.extend([ CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"), - MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), + MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), BuildMulledDockerContainerResolver(self.app_info), BuildMulledSingularityContainerResolver(self.app_info), From b41d969d681e3d427fd9f42ca9502c91634f061d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 18 Dec 2018 15:31:14 +0100 Subject: [PATCH 20/50] Add integration test for singularity resolution --- lib/galaxy/tools/deps/__init__.py | 1 + .../tools/deps/container_resolvers/mulled.py | 7 +++++- lib/galaxy/tools/deps/singularity_util.py | 4 +-- test/integration/singularity_job_conf.xml | 25 +++++++++++++++++++ ...zed_jobs.py => test_containerized_jobs.py} | 11 +++++++- 5 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 test/integration/singularity_job_conf.xml rename test/integration/{test_dockerized_jobs.py => test_containerized_jobs.py} (87%) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 03e28c8182d..c51be994c77 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -53,6 +53,7 @@ def build_dependency_manager(config): class NullDependencyManager(object): dependency_resolvers = [] enabled_container_types = [] + resolver_classes = set() def set_enabled_container_types(self, enabled_container_types): return diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index a4802fc6855..1d3292b2ead 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -316,6 +316,7 @@ class MulledDockerContainerResolver(ContainerResolver): resolver_type = "mulled" container_type = "docker" + protocol = None def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): super(MulledDockerContainerResolver, self).__init__(app_info) @@ -339,8 +340,11 @@ class MulledDockerContainerResolver(ContainerResolver): name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace) if name: + container_id = "quay.io/%s/%s" % (self.namespace, name) + if self.protocol: + container_id = "%s%s" % (self.protocol, container_id) container_description = ContainerDescription( - "quay.io/%s/%s" % (self.namespace, name), + container_id, type=self.container_type, ) destination_for_container_type = kwds.get('destination_for_container_type') @@ -365,6 +369,7 @@ class MulledSingularityContainerResolver(MulledDockerContainerResolver): resolver_type = "mulled_singularity" container_type = "singularity" + protocol = 'docker://' def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds): super(MulledSingularityContainerResolver, self).__init__(app_info) diff --git a/lib/galaxy/tools/deps/singularity_util.py b/lib/galaxy/tools/deps/singularity_util.py index dbdce4a2dbc..141d66be478 100644 --- a/lib/galaxy/tools/deps/singularity_util.py +++ b/lib/galaxy/tools/deps/singularity_util.py @@ -23,10 +23,10 @@ def pull_mulled_singularity_command(docker_image_identifier, ) save_path = docker_image_identifier if namespace: - prefix = "quay.io/%s/" % namespace + prefix = "docker://quay.io/%s/" % namespace if docker_image_identifier.startswith(prefix): save_path = docker_image_identifier[len(prefix):] - command_parts.extend(["build", os.path.join(cache_directory, save_path), "docker://%s" % docker_image_identifier]) + command_parts.extend(["build", os.path.join(cache_directory, save_path), docker_image_identifier]) return command_parts diff --git a/test/integration/singularity_job_conf.xml b/test/integration/singularity_job_conf.xml new file mode 100644 index 00000000000..8743b71ef7e --- /dev/null +++ b/test/integration/singularity_job_conf.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + true + + + + + + + + + + + + diff --git a/test/integration/test_dockerized_jobs.py b/test/integration/test_containerized_jobs.py similarity index 87% rename from test/integration/test_dockerized_jobs.py rename to test/integration/test_containerized_jobs.py index 43e56492c00..ebe64607801 100644 --- a/test/integration/test_dockerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -11,20 +11,24 @@ from .test_job_environments import RunsEnvironmentJobs SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml") +SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.xml") @integration_util.skip_unless_docker() class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs): framework_tool_and_types = True + job_config_file = DOCKERIZED_JOB_CONFIG_FILE @classmethod def handle_galaxy_config_kwds(cls, config): cls.jobs_directory = cls._test_driver.mkdtemp() config["jobs_directory"] = cls.jobs_directory - config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE + config["job_config_file"] = cls.job_config_file # Disable tool dependency resolution. config["tool_dependency_dir"] = "none" + config["conda_auto_init"] = False + config["conda_auto_install"] = False config["enable_beta_mulled_containers"] = "true" def setUp(self): @@ -70,3 +74,8 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru # Should we change env_pass_through to just always include TMP and HOME for docker? # I'm not sure, if yes this would change. assert job_env.home == "/", job_env.home + + +class SingularityJobsIntegrationTestCase(DockerizedJobsIntegrationTestCase): + + job_config_file = SINGULARITY_JOB_CONFIG_FILE From fd4b009089d226e05a6191c3067fad560c5a77c8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 18 Dec 2018 17:26:37 +0100 Subject: [PATCH 21/50] Skip singualrity integration tests if singularity not available --- test/base/integration_util.py | 4 ++++ test/integration/test_containerized_jobs.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/base/integration_util.py b/test/base/integration_util.py index 54b166bfe9a..185fb3c26cd 100644 --- a/test/base/integration_util.py +++ b/test/base/integration_util.py @@ -33,6 +33,10 @@ def skip_unless_docker(): return skip_unless_executable("docker") +def skip_unless_singularity(): + return skip_unless_executable("singularity") + + class IntegrationTestCase(TestCase, UsesApiTestCaseMixin): """Unit test case with utilities for spinning up Galaxy.""" diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index ebe64607801..590498f5f95 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -48,7 +48,7 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru output = self.dataset_populator.get_history_dataset_content(self.history_id) assert "0.7.15-r1140" in output - def test_docker_job_enviornment(self): + def test_container_job_enviornment(self): job_env = self._run_and_get_environment_properties("job_environment_default") euid = os.geteuid() @@ -61,7 +61,7 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru assert job_env.home.startswith(self.jobs_directory) assert job_env.home.endswith("/home") - def test_docker_job_environment_legacy(self): + def test_container_job_environment_legacy(self): job_env = self._run_and_get_environment_properties("job_environment_default_legacy") euid = os.geteuid() @@ -76,6 +76,7 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru assert job_env.home == "/", job_env.home +@integration_util.skip_unless_singularity() class SingularityJobsIntegrationTestCase(DockerizedJobsIntegrationTestCase): job_config_file = SINGULARITY_JOB_CONFIG_FILE From 2d6863067554f38ac05625bee1b81663a7973a1a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 19 Dec 2018 15:11:30 +0100 Subject: [PATCH 22/50] Add MulledSingularityContainerResolver to __all__ Otherwise speicifying the resolver in containers_conf.xml fails. --- lib/galaxy/tools/deps/container_resolvers/mulled.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index 1d3292b2ead..f4806259f2e 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -491,6 +491,7 @@ __all__ = ( "CachedMulledDockerContainerResolver", "CachedMulledSingularityContainerResolver", "MulledDockerContainerResolver", + "MulledSingularityContainerResolver", "BuildMulledDockerContainerResolver", "BuildMulledSingularityContainerResolver", ) From 337bddfd4841d51291f940effc99a4622f10649a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 10:05:16 +0100 Subject: [PATCH 23/50] Pass in user, fixes dependency install for data manager tools --- lib/galaxy/webapps/galaxy/api/tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 339e550188f..519af7bfb6e 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -234,7 +234,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): Return the resolver status for a specific tool id. [{"status": "installed", "name": "hisat2", "versionless": false, "resolver_type": "conda", "version": "2.0.3", "type": "package"}] """ - tool = self._get_tool(id) + tool = self._get_tool(id, user=trans.user) return tool.tool_requirements_status @expose_api @@ -255,7 +255,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): build_dependency_cache: If true, attempts to cache dependencies for this tool force_rebuild: If true and cache dir exists, attempts to delete cache dir """ - tool = self._get_tool(id) + tool = self._get_tool(id, user=trans.user) kwds['install'] = True tool._view.install_dependencies(tool.requirements, **kwds) if kwds.get('build_dependency_cache'): @@ -276,7 +276,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): Defaults to using the highest ranking resolver resolver_type: Use the dependency resolver of this resolver_type to install dependency """ - tool = self._get_tool(id) + tool = self._get_tool(id, user=trans.user) tool._view.uninstall_dependencies(requirements=tool.requirements, **kwds) # TODO: rework resolver install system to log and report what has been done. return tool.tool_requirements_status From 83473f075d9e8cce04dc775d37b44b87baca9931 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 10:47:04 +0100 Subject: [PATCH 24/50] Update testing docker image; drop mysql tests, add singularity --- test/docker/base/Dockerfile | 26 ++++----------- test/docker/base/run_test_wrapper.sh | 6 ---- test/docker/base/start_mysql.sh | 47 ---------------------------- 3 files changed, 6 insertions(+), 73 deletions(-) delete mode 100644 test/docker/base/start_mysql.sh diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index ac85272d5bb..dc5ba791f57 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -18,21 +18,14 @@ ENV DEBIAN_FRONTEND=noninteractive \ # Pre-install a bunch of packages to speed up ansible steps. RUN apt-get update -y && apt-get install -y software-properties-common apt-transport-https curl && \ apt-add-repository -y ppa:ansible/ansible && \ + curl -s http://neuro.debian.net/lists/xenial.us-ca.full | tee /etc/apt/sources.list.d/neurodebian.sources.list && \ curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ curl -sL https://deb.nodesource.com/setup_8.x | bash - && \ - apt-key adv --keyserver pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5 && \ - echo "deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list && \ + curl -s http://neuro.debian.net/_static/neuro.debian.net.asc|apt-key add - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ - { \ - echo mysql-community-server mysql-community-server/data-dir select ''; \ - echo mysql-community-server mysql-community-server/root-pass password ''; \ - echo mysql-community-server mysql-community-server/re-root-pass password ''; \ - echo mysql-community-server mysql-community-server/remove-test-db select false; \ - } | debconf-set-selections && \ apt-get update -y && \ - apt-get install -y libpq-dev postgresql postgresql-client \ - ansible wget mysql-server="${MYSQL_MAJOR}"* libmysqlclient-dev="${MYSQL_MAJOR}"* \ + apt-get install -y libpq-dev postgresql postgresql-client ansible wget \ slurm-llnl libmunge-dev slurm-drmaa-dev ant cmake curl \ g++ gcc gfortran git-core libffi-dev liblapack-dev \ libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \ @@ -41,6 +34,7 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans python-prettytable python-psycopg2 python-virtualenv python-pip \ rsync slurm-drmaa-dev swig sysstat unzip \ autoconf automake build-essential libatlas-base-dev libblas-dev openssl \ + singularity-container \ bzip2 \ ca-certificates \ openjdk-8-jre-headless \ @@ -57,10 +51,8 @@ RUN npm install -g grunt grunt-cli RUN mkdir -p /tmp/ansible && \ mkdir -p /opt/galaxy/db && \ - chown -R postgres:postgres /opt/galaxy/db && \ - sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf + chown -R postgres:postgres /opt/galaxy/db -ADD start_mysql.sh /opt/galaxy/start_mysql.sh ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml ADD provision.yml /tmp/ansible/provision.yml @@ -93,7 +85,6 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \ export GALAXY_VIRTUAL_ENV=$VENV && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ pip install psycopg2; done && \ - pip install mysql && \ cd $GALAXY_ROOT && \ echo "Prepopulating postgres database" && \ su -c '/usr/lib/postgresql/${POSTGRES_MAJOR}/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ @@ -101,15 +92,10 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \ GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" bash create_db.sh && \ echo "Prepopulating sqlite database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" bash create_db.sh && \ - sh /opt/galaxy/start_mysql.sh && \ - echo "Prepopulating mysql database" && \ - GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" bash create_db.sh && \ echo "Prepopulating toolshed postgres database" && \ TOOL_SHED_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" bash create_db.sh tool_shed && \ echo "Prepopulating toolshed sqlite database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed && \ - echo "Prepopulating toolshed mysql database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" bash create_db.sh tool_shed + TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed # bcftools for Galaxy. RUN mkdir -p /tmp/install && \ diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index 6d2c3c70368..d9686ab4d1c 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -30,12 +30,6 @@ then GALAXY_TEST_DBURI="postgres://root@localhost:5930/galaxy?client_encoding=utf8" TOOL_SHED_TEST_DBURI="postgres://root@localhost:5930/toolshed?client_encoding=utf8" export GALAXY_CONFIG_OVERRIDE_DATABASE_ENCODING="UTF-8" -elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ]; -then - sh /opt/galaxy/start_mysql.sh - GALAXY_TEST_INSTALL_DB_MERGED="true" - GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" - TOOL_SHED_TEST_DBURI="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ]; then GALAXY_TEST_INSTALL_DB_MERGED="true" diff --git a/test/docker/base/start_mysql.sh b/test/docker/base/start_mysql.sh deleted file mode 100644 index cd378a056fe..00000000000 --- a/test/docker/base/start_mysql.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e - -MYSQL_USER=galaxy -MYSQL_PASSWORD=galaxy -MYSQL_DATABASE=galaxy -SHED_MYSQL_DATABASE=toolshed - -mkdir -p /var/lib/mysql -chown -R mysql:mysql /var/lib/mysql - -# Derived from -# https://github.com/docker-library/mysql/blob/master/5.7/docker-entrypoint.sh - -DATADIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')" - -tempSqlFile='/tmp/mysql-first-time.sql' -cat > "$tempSqlFile" <<-EOSQL - DELETE FROM mysql.user ; - CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; - GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ; - DROP DATABASE IF EXISTS test ; -EOSQL - -if [ "$MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" -fi - -if [ "$SHED_MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$SHED_MYSQL_DATABASE\` ;" >> "$tempSqlFile" -fi - -if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then - echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" - - if [ "$MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" - fi - if [ "$SHED_MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$SHED_MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" - fi -fi - -echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" - -mysqld_safe --init-file="$tempSqlFile" & -sleep 5 \ No newline at end of file From 6b25cace916de7a694b0e83ee640d7170c24cd10 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 14:14:10 +0100 Subject: [PATCH 25/50] Do run through build_mulled resolver if install is set --- lib/galaxy/tools/deps/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index c51be994c77..8466547561e 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -183,7 +183,7 @@ class DependencyManager(object): # Shortcut - resolution complete. break - if resolver.resolver_type.startswith('build_mulled'): + if resolver.resolver_type.startswith('build_mulled') and not install: # don't want to build images here continue From 0d86c9098c032742b091d4691634ada81da980f0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 14:16:58 +0100 Subject: [PATCH 26/50] Make NullDependencyManager a class with empty non-container dependencies --- lib/galaxy/tools/deps/__init__.py | 36 ++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 8466547561e..1072259c84f 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -50,23 +50,6 @@ def build_dependency_manager(config): return dependency_manager -class NullDependencyManager(object): - dependency_resolvers = [] - enabled_container_types = [] - resolver_classes = set() - - def set_enabled_container_types(self, enabled_container_types): - return - - def uses_tool_shed_dependencies(self): - return False - - def dependency_shell_commands(self, requirements, **kwds): - return [] - - def find_dep(self, name, version=None, type='package', **kwds): - return NullDependency(version=version, name=name) - class DependencyManager(object): """ @@ -337,3 +320,22 @@ class CachedDependencyManager(DependencyManager): """ req_hashes = self.hash_dependencies(resolved_dependencies) return os.path.abspath(os.path.join(self.tool_dependency_cache_dir, req_hashes)) + + +class NullDependencyManager(DependencyManager): + + def __init__(self, default_base_path=None, conf_file=None, app_config={}): + self.__app_config = app_config + self.resolver_classes = set() + self.dependency_resolvers = [] + self._enabled_container_types = [] + self._destination_for_container_type = {} + + def uses_tool_shed_dependencies(self): + return False + + def dependency_shell_commands(self, requirements, **kwds): + return [] + + def find_dep(self, name, version=None, type='package', **kwds): + return NullDependency(version=version, name=name) From 13592e5e85195cb0c457457fb1a350fa8df213d4 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 14:17:34 +0100 Subject: [PATCH 27/50] Fix __str__ for BuildSingularityContainerResolver --- lib/galaxy/tools/deps/container_resolvers/mulled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index f4806259f2e..be26edefab8 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -480,7 +480,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): return involucro_context def __str__(self): - return "BuildDockerContainerResolver[cache_directory=%s]" % self.cache_directory + return "BuildSingularityContainerResolver[cache_directory=%s]" % self.cache_directory def mulled_targets(tool_info): From ccb8429a84e8d4a63f57e320510b5de2524eb5e6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 14:17:54 +0100 Subject: [PATCH 28/50] Download a binary that works on OSX --- lib/galaxy/tools/deps/mulled/mulled_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index 15aa0f2affc..f5154969c22 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -68,7 +68,7 @@ From: bgruening/busybox-bash:0.1 def involucro_link(): if IS_OS_X: - url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION + url = "https://github.com/mvdbeek/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION else: url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION return url From 4ef3460c041f210b0c0513a3786b186bac7a05e9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:02:00 +0100 Subject: [PATCH 29/50] DependencyManager lint fix --- lib/galaxy/tools/deps/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 1072259c84f..116193ddc7e 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -50,7 +50,6 @@ def build_dependency_manager(config): return dependency_manager - class DependencyManager(object): """ A DependencyManager attempts to resolve named and versioned dependencies by From ac5e7d994912b292fb3d3464e4e5e58859070bc9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:03:14 +0100 Subject: [PATCH 30/50] Make `resolver` methods respect enabled_container_types --- lib/galaxy/config.py | 4 ++-- lib/galaxy/tools/deps/container_resolvers/mulled.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 4da8731565f..8fdeb0c6d11 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1061,8 +1061,6 @@ class ConfiguresGalaxyMixin(object): involucro_auto_init=self.config.involucro_auto_init, ) self.container_finder = containers.ContainerFinder(app_info) - self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes) - self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers) self._set_enabled_container_types() index_help = getattr(self.config, "index_tool_help", True) self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_help) @@ -1080,6 +1078,8 @@ class ConfiguresGalaxyMixin(object): for enabled_container_type in self.container_finder._enabled_container_types(destination.params): container_types_to_destinations[enabled_container_type].append(destination) self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) + self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes) + self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers) def _configure_tool_data_tables(self, from_shed_config): from galaxy.tools.data import ToolDataTableManager diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index be26edefab8..7d71263e377 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -229,7 +229,7 @@ class CachedMulledDockerContainerResolver(ContainerResolver): self.hash_func = hash_func def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment: + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) @@ -251,7 +251,7 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): self.hash_func = hash_func def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment: + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) @@ -331,7 +331,7 @@ class MulledDockerContainerResolver(ContainerResolver): shell(command) def resolve(self, enabled_container_types, tool_info, install=False, **kwds): - if tool_info.requires_galaxy_python_environment: + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) @@ -413,7 +413,7 @@ class BuildMulledDockerContainerResolver(ContainerResolver): self.auto_init = self._get_config_option("involucro_auto_init", True) def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment: + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) @@ -460,7 +460,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): self.auto_init = self._get_config_option("involucro_auto_init", True) def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment: + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) From 62863e8991728a1fe4b92cf42cd9d1e825ff1a02 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:03:57 +0100 Subject: [PATCH 31/50] Don't fail if build exists and also don't remove build dir if it already exists --- lib/galaxy/tools/deps/mulled/mulled_build.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index f5154969c22..ce228e46da6 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -275,12 +275,16 @@ class InvolucroContext(installable.InstallableContext): def exec_command(self, involucro_args): cmd = self.build_command(involucro_args) # Create ./build dir manually, otherwise Docker will do it as root - os.mkdir('./build') + created_build_dir = False + if not os.path.exists('build'): + created_build_dir = True + os.mkdir('./build') try: res = self.shell_exec(" ".join(cmd)) finally: # delete build directory in any case - shutil.rmtree('./build') + if created_build_dir: + shutil.rmtree('./build') return res def is_installed(self): From 4a307ac5bea000d097e8488017f437e678a97a7f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:04:37 +0100 Subject: [PATCH 32/50] Add ExplicitSingularityContainerResolver --- .../deps/container_resolvers/explicit.py | 26 ++++++++++++++++++- lib/galaxy/tools/deps/containers.py | 6 ++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/explicit.py b/lib/galaxy/tools/deps/container_resolvers/explicit.py index 9f61ab95346..1480f078dce 100644 --- a/lib/galaxy/tools/deps/container_resolvers/explicit.py +++ b/lib/galaxy/tools/deps/container_resolvers/explicit.py @@ -26,4 +26,28 @@ class ExplicitContainerResolver(ContainerResolver): return None -__all__ = ("ExplicitContainerResolver", ) +class ExplicitSingularityContainerResolver(ExplicitContainerResolver): + + resolver_type = 'explicit_singularity' + container_type = 'singularity' + + def resolve(self, enabled_container_types, tool_info, **kwds): + """Find a container explicitly mentioned in tool description. + + This ignores the tool requirements and assumes the tool author crafted + a correct container. We use singularity here to fetch docker containers, + hence the container_description hack here. + """ + for container_description in tool_info.container_descriptions: + if container_description.type == 'docker': + desc_dict = container_description.to_dict() + desc_dict['type'] = self.container_type + desc_dict['identifier'] = "docker://%s" % container_description.identifier + container_description = container_description.from_dict(desc_dict) + if self._container_type_enabled(container_description, enabled_container_types): + return container_description + + return None + + +__all__ = ("ExplicitContainerResolver", "ExplicitSingularityContainerResolver") diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index e0dafd10321..5a8234f84db 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -11,7 +11,10 @@ from .container_classes import ( NULL_CONTAINER, SINGULARITY_CONTAINER_TYPE, ) -from .container_resolvers.explicit import ExplicitContainerResolver +from .container_resolvers.explicit import ( + ExplicitContainerResolver, + ExplicitSingularityContainerResolver, +) from .container_resolvers.mulled import ( BuildMulledDockerContainerResolver, BuildMulledSingularityContainerResolver, @@ -198,6 +201,7 @@ class ContainerRegistry(object): def __default_containers_resolvers(self): default_resolvers = [ ExplicitContainerResolver(self.app_info), + ExplicitSingularityContainerResolver(self.app_info), ] if self.enable_beta_mulled_containers: default_resolvers.extend([ From a6a186a64251f206294f2fd3416fb08db4234f47 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:05:26 +0100 Subject: [PATCH 33/50] Make containers list, map can only be iterated once --- lib/galaxy/tools/deps/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index ea92b7f49b2..8df34a96d11 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -240,7 +240,7 @@ def parse_requirements_from_xml(xml_root): if requirements_elem is not None: container_elems = requirements_elem.findall('container') - containers = map(container_from_element, container_elems) + containers = [container_from_element(c) for c in container_elems] return requirements, containers From 44d73f72c8967b114114d662ddb42098514026ee Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:07:36 +0100 Subject: [PATCH 34/50] Update Dockefile and run_test_wrapper.sh We use the Galaxy users' home dir for HOME. Otherwise os.environ.get('HOME') would return '/home/galaxy', but jobs run would see /galaxy. --- run_tests.sh | 1 + test/docker/base/Dockerfile | 4 ++++ test/docker/base/run_test_wrapper.sh | 5 +++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 676bf7c1d50..713a840efc3 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -311,6 +311,7 @@ then DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS \ + --cap-add=SYS_ADMIN \ -e "BUILD_NUMBER=$BUILD_NUMBER" \ -e "GALAXY_TEST_DATABASE_TYPE=$db_type" \ -e "LC_ALL=C" \ diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index dc5ba791f57..91cd7820644 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -127,6 +127,10 @@ RUN sudo mkdir -p /opt/selenium \ && wget --no-verbose https://selenium-release.storage.googleapis.com/3.6/selenium-server-standalone-3.6.0.jar \ -O /opt/selenium/selenium-server-standalone.jar +USER galaxy + +RUN mkdir /home/galaxy/.singularity + USER root #============================== diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index d9686ab4d1c..38ef982a2a3 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -16,9 +16,10 @@ echo "Creating galaxy group with gid $GALAXY_TEST_UID - it may already exist and groupadd -r galaxy -g "$GALAXY_TEST_UID" | true echo "Creating galaxy user with uid $GALAXY_TEST_UID - it may already exist and this is fine." useradd -u $GALAXY_TEST_UID -r -g galaxy -d /home/galaxy -c "Galaxy User" galaxy -s /bin/bash | true +mkdir -p /home/galaxy/.singularity echo "Setting galaxy user password - the operation may fail." echo "galaxy:galaxy" | chpasswd | true -chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" +chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" /home/galaxy/ : ${GALAXY_TEST_DATABASE_TYPE:-"postgres"} if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; @@ -48,7 +49,7 @@ cd /galaxy : ${GALAXY_VIRTUAL_ENV:=.venv} -HOME=/galaxy +HOME=/home/galaxy echo "Running common startup for updated dependencies (if any)" sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } From c0b63a18c7a2fa7d7fb45ffbddfcef5aa4c00c67 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:09:49 +0100 Subject: [PATCH 35/50] Fix container integration tests --- .../tools/job_environment_default.xml | 2 +- .../tools/job_environment_default_legacy.xml | 2 +- test/integration/test_containerized_jobs.py | 36 ++++++++++++++++--- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/test/functional/tools/job_environment_default.xml b/test/functional/tools/job_environment_default.xml index e8268469862..22c884657f1 100644 --- a/test/functional/tools/job_environment_default.xml +++ b/test/functional/tools/job_environment_default.xml @@ -2,7 +2,7 @@ busybox:ubuntu-14.04 - '$user_id'; echo `id -g` > '$group_id'; echo `pwd` > '$pwd'; diff --git a/test/functional/tools/job_environment_default_legacy.xml b/test/functional/tools/job_environment_default_legacy.xml index 07412ee774a..a00a661ebca 100644 --- a/test/functional/tools/job_environment_default_legacy.xml +++ b/test/functional/tools/job_environment_default_legacy.xml @@ -2,7 +2,7 @@ busybox:ubuntu-14.04 - '$user_id'; echo `id -g` > '$group_id'; echo `pwd` > '$pwd'; diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index 590498f5f95..49e30430d54 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -1,24 +1,29 @@ """Integration tests for running tools in Docker containers.""" import os +import unittest from base import integration_util from base.populators import ( DatasetPopulator, ) +from galaxy.tools.deps.commands import which from .test_job_environments import RunsEnvironmentJobs SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml") SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.xml") +EXTENDED_TIMEOUT = 120 -@integration_util.skip_unless_docker() class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs): framework_tool_and_types = True job_config_file = DOCKERIZED_JOB_CONFIG_FILE + build_mulled_resolver = 'build_mulled' + container_type = 'docker' + default_container_home_dir = '/' @classmethod def handle_galaxy_config_kwds(cls, config): @@ -31,6 +36,12 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru config["conda_auto_install"] = False config["enable_beta_mulled_containers"] = "true" + @classmethod + def setUpClass(cls): + if not which(cls.container_type): + raise unittest.SkipTest("Executable '%s' not found on PATH" % cls.container_type) + super(DockerizedJobsIntegrationTestCase, cls).setUpClass() + def setUp(self): super(DockerizedJobsIntegrationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) @@ -39,13 +50,13 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru def test_explicit(self): self.dataset_populator.run_tool("mulled_example_explicit", {}, self.history_id) self.dataset_populator.wait_for_history(self.history_id, assert_ok=True) - output = self.dataset_populator.get_history_dataset_content(self.history_id) + output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT) assert "0.7.15-r1140" in output def test_mulled_simple(self): self.dataset_populator.run_tool("mulled_example_simple", {}, self.history_id) self.dataset_populator.wait_for_history(self.history_id, assert_ok=True) - output = self.dataset_populator.get_history_dataset_content(self.history_id) + output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT) assert "0.7.15-r1140" in output def test_container_job_enviornment(self): @@ -73,10 +84,25 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru assert job_env.pwd.endswith("/working") # Should we change env_pass_through to just always include TMP and HOME for docker? # I'm not sure, if yes this would change. - assert job_env.home == "/", job_env.home + assert job_env.home == self.default_container_home_dir, job_env.home + + def test_build_mulled(self): + if not which('docker'): + raise unittest.SkipTest("Docker not found on PATH, required for building images via involucro") + resolver_type = self.build_mulled_resolver + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/dependencies" % tool_id + data = {'id': tool_id, 'resolver_type': resolver_type} + create_response = self._post(endpoint, data=data, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert any([True for d in response if d['dependency_type'] == self.container_type]) -@integration_util.skip_unless_singularity() class SingularityJobsIntegrationTestCase(DockerizedJobsIntegrationTestCase): job_config_file = SINGULARITY_JOB_CONFIG_FILE + build_mulled_resolver = 'build_mulled_singularity' + container_type = 'singularity' + # singularity passes $HOME by default + default_container_home_dir = os.environ.get('HOME', '/') From a0bd7072adf7f03a82adfddf3216b15ce7105c87 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 3 Jan 2019 14:46:45 +0100 Subject: [PATCH 36/50] Use bash for mulled ContainerDescriptions The base mulled container contains bash, and unfortunately bashisms such as <(cat bla) are very common in widely used Galaxy wrappers. So I think it makes sense to default to bash here. --- .../tools/deps/container_resolvers/mulled.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index 7d71263e377..b80f2051e23 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -27,7 +27,10 @@ from ..mulled.util import ( v1_image_name, v2_image_name, ) -from ..requirements import ContainerDescription +from ..requirements import ( + ContainerDescription, + DEFAULT_CONTAINER_SHELL, +) log = logging.getLogger(__name__) @@ -180,7 +183,7 @@ def find_best_matching_cached_image(targets, cached_images, hash_func): return image -def docker_cached_container_description(targets, namespace, hash_func="v2"): +def docker_cached_container_description(targets, namespace, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL): if len(targets) == 0: return None @@ -192,12 +195,13 @@ def docker_cached_container_description(targets, namespace, hash_func="v2"): container = ContainerDescription( image.image_identifier, type="docker", + shell=shell, ) return container -def singularity_cached_container_description(targets, cache_directory, hash_func="v2"): +def singularity_cached_container_description(targets, cache_directory, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL): if len(targets) == 0: return None @@ -212,6 +216,7 @@ def singularity_cached_container_description(targets, cache_directory, hash_func container = ContainerDescription( os.path.abspath(os.path.join(cache_directory, image.image_identifier)), type="singularity", + shell=shell, ) return container @@ -222,6 +227,7 @@ class CachedMulledDockerContainerResolver(ContainerResolver): resolver_type = "cached_mulled" container_type = "docker" + shell = '/bin/bash' def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): super(CachedMulledDockerContainerResolver, self).__init__(app_info) @@ -233,7 +239,7 @@ class CachedMulledDockerContainerResolver(ContainerResolver): return None targets = mulled_targets(tool_info) - return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) def __str__(self): return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace @@ -244,6 +250,7 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): resolver_type = "cached_mulled_singularity" container_type = "singularity" + shell = '/bin/bash' def __init__(self, app_info=None, hash_func="v2", **kwds): super(CachedMulledSingularityContainerResolver, self).__init__(app_info) @@ -255,7 +262,7 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): return None targets = mulled_targets(tool_info) - return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) def __str__(self): return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory @@ -316,6 +323,7 @@ class MulledDockerContainerResolver(ContainerResolver): resolver_type = "mulled" container_type = "docker" + shell = '/bin/bash' protocol = None def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): @@ -346,6 +354,7 @@ class MulledDockerContainerResolver(ContainerResolver): container_description = ContainerDescription( container_id, type=self.container_type, + shell=self.shell, ) destination_for_container_type = kwds.get('destination_for_container_type') if install and destination_for_container_type and not self.cached_container_description(targets, @@ -396,6 +405,7 @@ class BuildMulledDockerContainerResolver(ContainerResolver): resolver_type = "build_mulled" container_type = "docker" + shell = '/bin/bash' def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds): super(BuildMulledDockerContainerResolver, self).__init__(app_info) @@ -425,7 +435,7 @@ class BuildMulledDockerContainerResolver(ContainerResolver): involucro_context=self._get_involucro_context(), **self._mulled_kwds ) - return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) def _get_involucro_context(self): involucro_context = InvolucroContext(**self._involucro_context_kwds) @@ -442,6 +452,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): resolver_type = "build_mulled_singularity" container_type = "singularity" + shell = '/bin/bash' def __init__(self, app_info=None, hash_func="v2", **kwds): super(BuildMulledSingularityContainerResolver, self).__init__(app_info) @@ -472,7 +483,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): involucro_context=self._get_involucro_context(), **self._mulled_kwds ) - return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) def _get_involucro_context(self): involucro_context = InvolucroContext(**self._involucro_context_kwds) From 99a0ab03e90d2f9355c130403b1a0f5cb1ca5706 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 18:15:02 +0100 Subject: [PATCH 37/50] Temporarily test from mvdbeek/testing-base --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 713a840efc3..03647a1e379 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -270,7 +270,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='galaxy/testing-base:18.09.0' +DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.01.0' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" From a056f1746069a38b43bd0cc62bbe12d96aaa5145 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 24 Dec 2018 08:14:17 +0100 Subject: [PATCH 38/50] Document and use mulled_channels option --- config/galaxy.yml.sample | 20 +++++++++------ doc/source/admin/galaxy_options.rst | 28 +++++++++++++++------ lib/galaxy/config.py | 7 ++++++ lib/galaxy/tools/deps/dependencies.py | 3 +++ lib/galaxy/webapps/galaxy/config_schema.yml | 11 ++++++-- 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 12e57d2778f..820c96ab484 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -330,17 +330,23 @@ galaxy: # by enable_beta_mulled_containers. #containers_resolvers_config_file: '' - # involucro is a tool used to build Docker containers for tools from - # Conda dependencies referenced in tools as `requirement`s. The - # following path is the location of involucro on the Galaxy host. This - # is ignored if the relevant container resolver isn't enabled, and - # will install on demand unless involucro_auto_init is set to False. + # involucro is a tool used to build Docker ir Singularity containers + # for tools from Conda dependencies referenced in tools as + # `requirement`s. The following path is the location of involucro on + # the Galaxy host. This is ignored if the relevant container resolver + # isn't enabled, and will install on demand unless involucro_auto_init + # is set to False. #involucro_path: database/dependencies/involucro - # Install involucro as needed to build Docker containers for tools. - # Ignored if relevant container resolver is not used. + # Install involucro as needed to build Docker or Singularity + # containers for tools. Ignored if relevant container resolver is not + # used. #involucro_auto_init: true + # Conda channels to use when building Docker or Singularity containers + # using involucro. + #mulled_channels: 'conda-forge,bioconda' + # Enable automatic polling of relative tool sheds to see if any # updates are available for installed repositories. Ideally only one # Galaxy server process should be able to check for repository diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index a3c509888df..57868982388 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -521,12 +521,12 @@ ~~~~~~~~~~~~~~~~~~ :Description: - involucro is a tool used to build Docker containers for tools from - Conda dependencies referenced in tools as `requirement`s. The - following path is the location of involucro on the Galaxy host. - This is ignored if the relevant container resolver isn't enabled, - and will install on demand unless involucro_auto_init is set to - False. + involucro is a tool used to build Docker ir Singularity containers + for tools from Conda dependencies referenced in tools as + `requirement`s. The following path is the location of involucro on + the Galaxy host. This is ignored if the relevant container + resolver isn't enabled, and will install on demand unless + involucro_auto_init is set to False. :Default: ``database/dependencies/involucro`` :Type: str @@ -536,12 +536,24 @@ ~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Install involucro as needed to build Docker containers for tools. - Ignored if relevant container resolver is not used. + Install involucro as needed to build Docker or Singularity + containers for tools. Ignored if relevant container resolver is + not used. :Default: ``true`` :Type: bool +~~~~~~~~~~~~~~~~~~~ +``mulled_channels`` +~~~~~~~~~~~~~~~~~~~ + +:Description: + Conda channels to use when building Docker or Singularity + containers using involucro. +:Default: ``conda-forge,bioconda`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~ ``enable_tool_shed_check`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 8fdeb0c6d11..c7355929ad6 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -24,6 +24,7 @@ from six.moves import configparser from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError +from galaxy.tools.deps.container_resolvers.mulled import DEFAULT_CHANNELS from galaxy.util import ExecutionTimer from galaxy.util import listify from galaxy.util import string_as_bool @@ -519,6 +520,11 @@ class Configuration(object): involucro_path = os.path.join(tool_dependency_dir or "database", "involucro") self.involucro_path = resolve_path(involucro_path, self.root) self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True)) + mulled_channels = kwargs.get('mulled_channels') + if mulled_channels: + self.mulled_channels = [c.strip() for c in mulled_channels.split(',')] + else: + self.mulled_channels = DEFAULT_CHANNELS default_job_resubmission_condition = kwargs.get('default_job_resubmission_condition', '') if not default_job_resubmission_condition.strip(): @@ -1059,6 +1065,7 @@ class ConfiguresGalaxyMixin(object): containers_resolvers_config_file=self.config.containers_resolvers_config_file, involucro_path=self.config.involucro_path, involucro_auto_init=self.config.involucro_auto_init, + mulled_channels=self.config.mulled_channels, ) self.container_finder = containers.ContainerFinder(app_info) self._set_enabled_container_types() diff --git a/lib/galaxy/tools/deps/dependencies.py b/lib/galaxy/tools/deps/dependencies.py index 028e1cf84d0..194cfab1f5e 100644 --- a/lib/galaxy/tools/deps/dependencies.py +++ b/lib/galaxy/tools/deps/dependencies.py @@ -1,5 +1,6 @@ from galaxy.tools.deps.requirements import ToolRequirements from galaxy.util import bunch +from .mulled.mulled_build import DEFAULT_CHANNELS class AppInfo(object): @@ -15,6 +16,7 @@ class AppInfo(object): containers_resolvers_config_file=None, involucro_path=None, involucro_auto_init=True, + mulled_channels=DEFAULT_CHANNELS, ): self.galaxy_root_dir = galaxy_root_dir self.default_file_path = default_file_path @@ -26,6 +28,7 @@ class AppInfo(object): self.containers_resolvers_config_file = containers_resolvers_config_file self.involucro_path = involucro_path self.involucro_auto_init = involucro_auto_init + self.mulled_channels = mulled_channels class ToolInfo(object): diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 626fc0b14c6..96806f50522 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -414,7 +414,7 @@ mapping: default: database/dependencies/involucro required: false desc: | - involucro is a tool used to build Docker containers for tools from Conda + involucro is a tool used to build Docker ir Singularity containers for tools from Conda dependencies referenced in tools as `requirement`s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless @@ -425,9 +425,16 @@ mapping: default: true required: false desc: | - Install involucro as needed to build Docker containers for tools. Ignored if + Install involucro as needed to build Docker or Singularity containers for tools. Ignored if relevant container resolver is not used. + mulled_channels: + type: str + default: 'conda-forge,bioconda' + required: false + desc: | + Conda channels to use when building Docker or Singularity containers using involucro. + enable_tool_shed_check: type: bool default: false From 93622f026bacef4328e2415aad72f521d92192c9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 24 Dec 2018 12:08:00 +0100 Subject: [PATCH 39/50] Only parse build if build in name Otherwise this fails with ``` Traceback (most recent call last): File "lib/galaxy/web/framework/decorators.py", line 277, in decorator rval = func(self, trans, *args, **kwargs) File "lib/galaxy/web/framework/decorators.py", line 101, in decorator return func(self, trans, *args, **kwargs) File "lib/galaxy/webapps/galaxy/api/tools.py", line 239, in install_dependencies tool._view.install_dependencies(tool.requirements, **kwds) File "lib/galaxy/tools/deps/views.py", line 107, in install_dependencies return self._dependency_manager._requirements_to_dependencies_dict(requirements, **kwds) File "lib/galaxy/tools/deps/__init__.py", line 190, in _requirements_to_dependencies_dict **kwds) File "lib/galaxy/tools/deps/container_resolvers/mulled.py", line 343, in resolve name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace) File "lib/galaxy/tools/deps/container_resolvers/mulled.py", line 278, in targets_to_mulled_name version, build = split_tag(tag) File "lib/galaxy/tools/deps/mulled/util.py", line 69, in split_tag build = tag.split('--', 1)[1] IndexError: list index out of range ``` if a single-target image was build with a build_mulled resolver. --- lib/galaxy/tools/deps/container_resolvers/mulled.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index b80f2051e23..4218f564b96 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -280,9 +280,15 @@ def targets_to_mulled_name(targets, hash_func, namespace): if target_version: for tag in tags: - version, build = split_tag(tag) + if '--' in tag: + version, build = split_tag(tag) + else: + version = tag + build = None if version == target_version: - name = "%s:%s--%s" % (target.package_name, version, build) + name = "%s:%s" % (target.package_name, version) + if build: + name = "%s--%s" % (name, build) break else: version, build = split_tag(tags[0]) From b3af5176d35b0a1e818975c263018fb7ea800bcd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 25 Dec 2018 12:01:43 +0100 Subject: [PATCH 40/50] Replace deprecated singularity import with build And upgrade singularity build image to 2.4.6 (latest on biocontainers). Overall this is necessary because singularity import 2.3X doesn't work behind proxies. The change from `Singularity` to `Singularity.def` is necessary because the script uses the `file` command if the Singularity file doesn't end in .def or is not exactly Singularity (it is `/import/Singularity` in our case). `file` isn't available in the biocontainers image. --- lib/galaxy/tools/deps/mulled/invfile.lua | 7 ++----- lib/galaxy/tools/deps/mulled/mulled_build.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tools/deps/mulled/invfile.lua b/lib/galaxy/tools/deps/mulled/invfile.lua index 4b771e4b8ec..6c340647fb3 100644 --- a/lib/galaxy/tools/deps/mulled/invfile.lua +++ b/lib/galaxy/tools/deps/mulled/invfile.lua @@ -48,7 +48,7 @@ end local singularity_image = VAR.SINGULARITY_IMAGE if singularity_image == '' then - singularity_image = 'quay.io/biocontainers/singularity:2.3--0' + singularity_image = 'quay.io/biocontainers/singularity:2.4.6--0' end local singularity_image_dir = VAR.SINGULARITY_IMAGE_DIR @@ -103,10 +103,7 @@ if VAR.SINGULARITY ~= '' then .using(singularity_image) .withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true}) .withConfig({entrypoint = {'/bin/sh', '-c'}}) - -- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size - .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi") - .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) - .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity') + .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity build /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity.def') .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME) end diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index ce228e46da6..c3c84f5a586 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -236,7 +236,7 @@ def mull_targets( if singularity: if not os.path.exists(singularity_image_dir): safe_makedirs(singularity_image_dir) - with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def: + with open(os.path.join(singularity_image_dir, 'Singularity.def'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % {'container_test': test} sin_def.write(fill_template) with PrintProgress(): From 5f389dd11ea3da935b382871305bf0a7c1e2d3d4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sat, 29 Dec 2018 12:18:07 +0100 Subject: [PATCH 41/50] Fix spelling in resolver docs Co-Authored-By: mvdbeek --- config/container_resolvers_conf.xml.sample | 4 ++-- config/galaxy.yml.sample | 2 +- doc/source/admin/galaxy_options.rst | 2 +- lib/galaxy/webapps/galaxy/config_schema.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/container_resolvers_conf.xml.sample b/config/container_resolvers_conf.xml.sample index 9e927158ba7..6b1212d5a12 100644 --- a/config/container_resolvers_conf.xml.sample +++ b/config/container_resolvers_conf.xml.sample @@ -27,11 +27,11 @@ --> - diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 820c96ab484..63338e62a81 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -330,7 +330,7 @@ galaxy: # by enable_beta_mulled_containers. #containers_resolvers_config_file: '' - # involucro is a tool used to build Docker ir Singularity containers + # involucro is a tool used to build Docker or Singularity containers # for tools from Conda dependencies referenced in tools as # `requirement`s. The following path is the location of involucro on # the Galaxy host. This is ignored if the relevant container resolver diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 57868982388..28ba3ae7a1b 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -521,7 +521,7 @@ ~~~~~~~~~~~~~~~~~~ :Description: - involucro is a tool used to build Docker ir Singularity containers + involucro is a tool used to build Docker or Singularity containers for tools from Conda dependencies referenced in tools as `requirement`s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 96806f50522..e545dd02da7 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -414,7 +414,7 @@ mapping: default: database/dependencies/involucro required: false desc: | - involucro is a tool used to build Docker ir Singularity containers for tools from Conda + involucro is a tool used to build Docker or Singularity containers for tools from Conda dependencies referenced in tools as `requirement`s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless From 0343d3d87f438bdb669959bc5141030169cc01ba Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 30 Dec 2018 16:53:26 +0100 Subject: [PATCH 42/50] Cache requests to mulled registry for 5 minutes When going through the resolvers we can easily exceed the allowed API request rate. --- lib/galaxy/tools/deps/mulled/util.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/deps/mulled/util.py b/lib/galaxy/tools/deps/mulled/util.py index eb7c0f18e9b..f394a907278 100644 --- a/lib/galaxy/tools/deps/mulled/util.py +++ b/lib/galaxy/tools/deps/mulled/util.py @@ -13,6 +13,8 @@ try: except ImportError: requests = None +MULLED_TAG_CACHE = collections.defaultdict(dict) + def create_repository(namespace, repo_name, oauth_token): assert oauth_token @@ -56,7 +58,19 @@ def mulled_tags_for(namespace, image, tag_prefix=None): The result will be sorted so newest tags are first. """ - tags = quay_versions(namespace, image) + tags_cached = False + if namespace in MULLED_TAG_CACHE: + if image in MULLED_TAG_CACHE[namespace]: + tags, last_checked = MULLED_TAG_CACHE[namespace][image] + if not tags and time.time() - last_checked < 300: + # it's possible we haven't seen the tags before, we check every 5 minutes + tags_cached = False + else: + tags_cached = True + if not tags_cached: + tags = quay_versions(namespace, image) + last_checked = time.time() + MULLED_TAG_CACHE[namespace][image] = (tags, last_checked) if tag_prefix is not None: tags = [t for t in tags if t.startswith(tag_prefix)] tags = version_sorted(tags) From 9051c10cba8360bc5a4ee199ff01657c3fda3b20 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 31 Dec 2018 16:25:30 +0100 Subject: [PATCH 43/50] More docker debug --- run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index 03647a1e379..ae63280d16b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -309,6 +309,10 @@ then # Skip client build process in the Docker container for all tests, the Jenkins task builds the client # locally before testing - you will need to do this also if using this script for Selenium testing. DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" + echo "Docker version:" + docker --version + echo "Test singularity inside docker" + docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS --privileged --entrypoint /usr/bin/singularity mvdbeek/testing-base:19.01.0 exec docker://busybox hostname echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS \ --cap-add=SYS_ADMIN \ From a03f8bdffd272e0195a6bf88931fb17b6e428d13 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jan 2019 09:34:53 +0100 Subject: [PATCH 44/50] Test singularity exec as Galaxy user --- run_tests.sh | 12 ------------ test/docker/base/run_test_wrapper.sh | 4 ++++ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index ae63280d16b..13e860cbbce 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -295,24 +295,12 @@ then else db_type="sqlite" fi - if [ "$1" = "--external_tmp" ]; then - # If /tmp is a tmpfs there may be better performance by reusing - # the parent's temp file system. Also, it seems to decrease the - # frequency or errors such as the following: - # /bin/sh: 1: /tmp/tmpiWU3kJ/tmp_8zLxx/job_working_directory_mwwDmg/000/274/galaxy_274.sh: Text file busy - tmp=$(mktemp -d) - chmod 1777 $tmp - DOCKER_RUN_EXTRA_ARGS="-v ${tmp}:/tmp ${DOCKER_RUN_EXTRA_ARGS}" - shift - fi MY_UID=$(id -u) # Skip client build process in the Docker container for all tests, the Jenkins task builds the client # locally before testing - you will need to do this also if using this script for Selenium testing. DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" echo "Docker version:" docker --version - echo "Test singularity inside docker" - docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS --privileged --entrypoint /usr/bin/singularity mvdbeek/testing-base:19.01.0 exec docker://busybox hostname echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS \ --cap-add=SYS_ADMIN \ diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index 38ef982a2a3..5cd620628ad 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -50,6 +50,10 @@ cd /galaxy : ${GALAXY_VIRTUAL_ENV:=.venv} HOME=/home/galaxy + +echo "Testing singularity exec as galaxy user" +sudo -E -u "#${GALAXY_TEST_UID}" singularity -vvv exec docker://busybox hostname + echo "Running common startup for updated dependencies (if any)" sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } From 658816264a8e84bacfdf83e9c9029338b2e1f85d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jan 2019 12:55:39 +0100 Subject: [PATCH 45/50] Explicitly pass home dir to singularity command --- lib/galaxy/tools/deps/singularity_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/tools/deps/singularity_util.py b/lib/galaxy/tools/deps/singularity_util.py index 141d66be478..ba281f8880f 100644 --- a/lib/galaxy/tools/deps/singularity_util.py +++ b/lib/galaxy/tools/deps/singularity_util.py @@ -43,7 +43,10 @@ def build_singularity_run_command( ): command_parts = [] # http://singularity.lbl.gov/docs-environment-metadata + home = None for (key, value) in env: + if key == 'HOME': + home = value command_parts.extend(["SINGULARITYENV_%s=%s" % (key, value)]) command_parts += _singularity_prefix( singularity_cmd=singularity_cmd, @@ -56,6 +59,8 @@ def build_singularity_run_command( command_parts.extend(["-B", shlex_quote(str(volume))]) if working_directory: command_parts.extend(["--pwd", shlex_quote(working_directory)]) + if home is not None: + command_parts.extend(["--home", "%s:%s" % (home, home)]) if run_extra_arguments: command_parts.append(run_extra_arguments) full_image = image From 06f5abe1bc16eccf750baf7a754d48dfdedaaf5a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jan 2019 12:56:16 +0100 Subject: [PATCH 46/50] Install singularity 3.0.1 from source --- test/docker/base/Dockerfile | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 91cd7820644..2399868d3ff 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -18,24 +18,21 @@ ENV DEBIAN_FRONTEND=noninteractive \ # Pre-install a bunch of packages to speed up ansible steps. RUN apt-get update -y && apt-get install -y software-properties-common apt-transport-https curl && \ apt-add-repository -y ppa:ansible/ansible && \ - curl -s http://neuro.debian.net/lists/xenial.us-ca.full | tee /etc/apt/sources.list.d/neurodebian.sources.list && \ curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ curl -sL https://deb.nodesource.com/setup_8.x | bash - && \ - curl -s http://neuro.debian.net/_static/neuro.debian.net.asc|apt-key add - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ apt-get update -y && \ apt-get install -y libpq-dev postgresql postgresql-client ansible wget \ slurm-llnl libmunge-dev slurm-drmaa-dev ant cmake curl \ g++ gcc gfortran git-core libffi-dev liblapack-dev \ libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \ - mercurial nginx-extras patch postgresql postgresql \ - postgresql-client python-boto python-dev python3-dev \ + patch postgresql postgresql \ + postgresql-client python-dev python3-dev \ python-prettytable python-psycopg2 python-virtualenv python-pip \ rsync slurm-drmaa-dev swig sysstat unzip \ autoconf automake build-essential libatlas-base-dev libblas-dev openssl \ - singularity-container \ - bzip2 \ + bzip2 uuid-dev libssl-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \ ca-certificates \ openjdk-8-jre-headless \ tzdata \ @@ -47,7 +44,9 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans libnss3 libgconf-2-4 && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN npm install -g grunt grunt-cli +# Install golang +RUN export VERSION=1.11 OS=linux ARCH=amd64 && curl -sL https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz \ + | tar -C /usr/local -xzf - RUN mkdir -p /tmp/ansible && \ mkdir -p /opt/galaxy/db && \ @@ -127,9 +126,17 @@ RUN sudo mkdir -p /opt/selenium \ && wget --no-verbose https://selenium-release.storage.googleapis.com/3.6/selenium-server-standalone-3.6.0.jar \ -O /opt/selenium/selenium-server-standalone.jar -USER galaxy +USER seluser -RUN mkdir /home/galaxy/.singularity +RUN export GOPATH=${HOME}/go && export PATH=/usr/local/go/bin:${PATH}:${GOPATH}/bin && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + git clone https://github.com/sylabs/singularity.git && \ + cd singularity && \ + git checkout v3.0.1 && \ + ./mconfig && \ + cd builddir && make && sudo make install && \ + rm -Rf ${HOME}/go USER root From bed8b54af604a2caf9161a5265953c90ae5a6c8a Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 20 Dec 2018 19:38:54 +0100 Subject: [PATCH 47/50] Only run integration tests with --privileged --- .ci/jenkins/integration-py3/run_tests.sh | 2 +- .ci/jenkins/integration/run_tests.sh | 2 +- run_tests.sh | 1 - test/docker/base/run_test_wrapper.sh | 3 --- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.ci/jenkins/integration-py3/run_tests.sh b/.ci/jenkins/integration-py3/run_tests.sh index b45d68d67b5..ffc6c87e98c 100755 --- a/.ci/jenkins/integration-py3/run_tests.sh +++ b/.ci/jenkins/integration-py3/run_tests.sh @@ -1,3 +1,3 @@ #!/bin/bash -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@" +DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@" diff --git a/.ci/jenkins/integration/run_tests.sh b/.ci/jenkins/integration/run_tests.sh index f5b8f320f8e..4119aef60b2 100755 --- a/.ci/jenkins/integration/run_tests.sh +++ b/.ci/jenkins/integration/run_tests.sh @@ -1,3 +1,3 @@ #!/bin/bash -./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@" +DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@" diff --git a/run_tests.sh b/run_tests.sh index 13e860cbbce..d79f87332f6 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -303,7 +303,6 @@ then docker --version echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS \ - --cap-add=SYS_ADMIN \ -e "BUILD_NUMBER=$BUILD_NUMBER" \ -e "GALAXY_TEST_DATABASE_TYPE=$db_type" \ -e "LC_ALL=C" \ diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index 5cd620628ad..26b6de3df3a 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -51,9 +51,6 @@ cd /galaxy HOME=/home/galaxy -echo "Testing singularity exec as galaxy user" -sudo -E -u "#${GALAXY_TEST_UID}" singularity -vvv exec docker://busybox hostname - echo "Running common startup for updated dependencies (if any)" sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } From 9517871b4af27c9647436d84ea2bfe76bb242ca8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 2 Jan 2019 13:00:29 +0100 Subject: [PATCH 48/50] Bump image to use --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index d79f87332f6..6b030fdd597 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -270,7 +270,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.01.0' +DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.01.3' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" From 6b3af3b2ffa8d90946a6b2d9b803779df35e19fd Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 6 Jan 2019 12:39:01 +0100 Subject: [PATCH 49/50] Make dependency_shell_commands a property If we know we use a container we don't need to pass through the dependency resolver. --- lib/galaxy/jobs/__init__.py | 15 +++++++++++---- lib/galaxy/jobs/command_factory.py | 3 +-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 16fb348e72c..5418dfeba3d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -685,6 +685,7 @@ class JobWrapper(HasResourceParameters): self.extra_filenames = [] self.command_line = None self.dependencies = [] + self._dependency_shell_commands = None # Tool versioning variables self.write_version_cmd = None self.version_string = "" @@ -728,6 +729,15 @@ class JobWrapper(HasResourceParameters): def dataset_path_rewriter(self): return self._job_dataset_path_rewriter + @property + def dependency_shell_commands(self): + """Shell fragment to inject dependencies.""" + if self._dependency_shell_commands is None: + self._dependency_shell_commands = self.tool.build_dependency_shell_commands( + job_directory=self.working_directory + ) + return self._dependency_shell_commands + @property def cleanup_job(self): """ Remove the job after it is complete, should return "always", "onsuccess", or "never". @@ -860,8 +870,6 @@ class JobWrapper(HasResourceParameters): self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() # Ensure galaxy_lib_dir is set in case there are any later chdirs self.galaxy_lib_dir - # Shell fragment to inject dependencies - self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory) if self.tool.requires_galaxy_python_environment: # These tools (upload, metadata, data_source) may need access to the datatypes registry. self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) @@ -1959,8 +1967,7 @@ class TaskWrapper(JobWrapper): # Ensure galaxy_lib_dir is set in case there are any later chdirs self.galaxy_lib_dir - # Shell fragment to inject dependencies - self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory) + # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished task.command_line = self.command_line diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 7922531ee8b..ebeaa4bc94a 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -158,9 +158,8 @@ def __handle_task_splitting(commands_builder, job_wrapper): def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params): local_dependency_resolution = remote_command_params.get("dependency_resolution", "local") == "local" - # Prepend dependency injection - if job_wrapper.dependency_shell_commands and local_dependency_resolution: + if local_dependency_resolution and job_wrapper.dependency_shell_commands: commands_builder.prepend_commands(job_wrapper.dependency_shell_commands) From 8a34101089085c6203cdd11468de4a0866922534 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 6 Jan 2019 14:43:00 +0100 Subject: [PATCH 50/50] Introduce `auto_install` ContainerResolver flag If set to True (thats's the default), the container resolvers that can pull or build images will do that, if they are enabled. You can set this to `False` and instead pass through the DependencyManager to use these resolvers to pull or build images. --- config/container_resolvers_conf.xml.sample | 29 +++- .../tools/deps/container_resolvers/mulled.py | 150 ++++++++++-------- 2 files changed, 105 insertions(+), 74 deletions(-) diff --git a/config/container_resolvers_conf.xml.sample b/config/container_resolvers_conf.xml.sample index 6b1212d5a12..039f2f5bfe1 100644 --- a/config/container_resolvers_conf.xml.sample +++ b/config/container_resolvers_conf.xml.sample @@ -4,7 +4,7 @@ tags in the tool XML wrapper. --> - + - + - + + packages listed in the tools definition. + Set auto_install to False if Galaxy should build container images + through the admin interface or API, but not automatically when + a tool is run. + --> - + + or more package requirements listed as part of the tool's definition. + Set auto_install to False if Galaxy should build container images + through the admin interface or API, but not automatically when + a tool is run. + + --> diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index 4218f564b96..8d108a3da55 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -7,7 +7,10 @@ import subprocess import six -from galaxy.util import unicodify +from galaxy.util import ( + string_as_bool, + unicodify, +) from ..commands import shell from ..container_classes import CONTAINER_CLASSES from ..container_resolvers import ( @@ -222,52 +225,6 @@ def singularity_cached_container_description(targets, cache_directory, hash_func return container -@six.python_2_unicode_compatible -class CachedMulledDockerContainerResolver(ContainerResolver): - - resolver_type = "cached_mulled" - container_type = "docker" - shell = '/bin/bash' - - def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): - super(CachedMulledDockerContainerResolver, self).__init__(app_info) - self.namespace = namespace - self.hash_func = hash_func - - def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: - return None - - targets = mulled_targets(tool_info) - return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) - - def __str__(self): - return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace - - -@six.python_2_unicode_compatible -class CachedMulledSingularityContainerResolver(ContainerResolver): - - resolver_type = "cached_mulled_singularity" - container_type = "singularity" - shell = '/bin/bash' - - def __init__(self, app_info=None, hash_func="v2", **kwds): - super(CachedMulledSingularityContainerResolver, self).__init__(app_info) - self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) - self.hash_func = hash_func - - def resolve(self, enabled_container_types, tool_info, **kwds): - if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: - return None - - targets = mulled_targets(tool_info) - return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) - - def __str__(self): - return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory - - def targets_to_mulled_name(targets, hash_func, namespace): name = None if len(targets) == 1: @@ -323,6 +280,52 @@ def targets_to_mulled_name(targets, hash_func, namespace): return name +@six.python_2_unicode_compatible +class CachedMulledDockerContainerResolver(ContainerResolver): + + resolver_type = "cached_mulled" + container_type = "docker" + shell = '/bin/bash' + + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds): + super(CachedMulledDockerContainerResolver, self).__init__(app_info) + self.namespace = namespace + self.hash_func = hash_func + + def resolve(self, enabled_container_types, tool_info, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) + + def __str__(self): + return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace + + +@six.python_2_unicode_compatible +class CachedMulledSingularityContainerResolver(ContainerResolver): + + resolver_type = "cached_mulled_singularity" + container_type = "singularity" + shell = '/bin/bash' + + def __init__(self, app_info=None, hash_func="v2", **kwds): + super(CachedMulledSingularityContainerResolver, self).__init__(app_info) + self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) + self.hash_func = hash_func + + def resolve(self, enabled_container_types, tool_info, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: + return None + + targets = mulled_targets(tool_info) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) + + def __str__(self): + return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory + + @six.python_2_unicode_compatible class MulledDockerContainerResolver(ContainerResolver): """Look for mulled images matching tool dependencies.""" @@ -332,10 +335,11 @@ class MulledDockerContainerResolver(ContainerResolver): shell = '/bin/bash' protocol = None - def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): super(MulledDockerContainerResolver, self).__init__(app_info) self.namespace = namespace self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) def cached_container_description(self, targets, namespace, hash_func): return docker_cached_container_description(targets, namespace, hash_func) @@ -363,9 +367,11 @@ class MulledDockerContainerResolver(ContainerResolver): shell=self.shell, ) destination_for_container_type = kwds.get('destination_for_container_type') - if install and destination_for_container_type and not self.cached_container_description(targets, - namespace=self.namespace, - hash_func=self.hash_func): + if install and destination_for_container_type and not self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func + ): container = CONTAINER_CLASSES[self.container_type](container_description.identifier, self.app_info, tool_info, @@ -373,6 +379,12 @@ class MulledDockerContainerResolver(ContainerResolver): {}, container_description) self.pull(container) + if not self.auto_install: + container_description = self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func + ) return container_description def __str__(self): @@ -386,11 +398,12 @@ class MulledSingularityContainerResolver(MulledDockerContainerResolver): container_type = "singularity" protocol = 'docker://' - def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds): + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): super(MulledSingularityContainerResolver, self).__init__(app_info) self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) self.namespace = namespace self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) def cached_container_description(self, targets, namespace, hash_func): return singularity_cached_container_description(targets, @@ -413,13 +426,14 @@ class BuildMulledDockerContainerResolver(ContainerResolver): container_type = "docker" shell = '/bin/bash' - def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds): + def __init__(self, app_info=None, namespace="local", hash_func="v2", auto_install=True, **kwds): super(BuildMulledDockerContainerResolver, self).__init__(app_info) self._involucro_context_kwds = { 'involucro_bin': self._get_config_option("involucro_path", None) } self.namespace = namespace self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) self._mulled_kwds = { 'namespace': namespace, 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), @@ -428,19 +442,19 @@ class BuildMulledDockerContainerResolver(ContainerResolver): } self.auto_init = self._get_config_option("involucro_auto_init", True) - def resolve(self, enabled_container_types, tool_info, **kwds): + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) if len(targets) == 0: return None - - mull_targets( - targets, - involucro_context=self._get_involucro_context(), - **self._mulled_kwds - ) + if self.auto_install or install: + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) def _get_involucro_context(self): @@ -460,13 +474,14 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): container_type = "singularity" shell = '/bin/bash' - def __init__(self, app_info=None, hash_func="v2", **kwds): + def __init__(self, app_info=None, hash_func="v2", auto_install=True, **kwds): super(BuildMulledSingularityContainerResolver, self).__init__(app_info) self._involucro_context_kwds = { 'involucro_bin': self._get_config_option("involucro_path", None) } self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) self._mulled_kwds = { 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), 'hash_func': self.hash_func, @@ -476,7 +491,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): } self.auto_init = self._get_config_option("involucro_auto_init", True) - def resolve(self, enabled_container_types, tool_info, **kwds): + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None @@ -484,11 +499,12 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): if len(targets) == 0: return None - mull_targets( - targets, - involucro_context=self._get_involucro_context(), - **self._mulled_kwds - ) + if self.auto_install or install: + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) def _get_involucro_context(self):