From 8d224edb5729114dd100fe730619d666df7e84dc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sat, 10 Jun 2017 06:34:28 -0400 Subject: [PATCH] Allow Galaxy to use cached mulled containers for Singularity. - Change undocumented (and probably unused) config option container_image_cache_path default to "database/contianer_cache". - Update container resolver framework to let plugins assert they only produce containers of a single type. - Update mulled container resolvers to indicate they produce Docker containers only - update names also. - Refactor mulled caching stuff to allow reuse with Singularity (where the names come from is different - but a lot of the logic is the same). - Add a cached singularity container resolver - that searches "config.container_image_cache_path/singularity/mulled" for an image with the correct name. Next up - similarly refactor the BuildMulledDockerContainerResolver to allow reuse and implement the BuildMulledSingularityContainerResolver. --- lib/galaxy/config.py | 4 + .../tools/deps/container_resolvers/mulled.py | 176 ++++++++++++------ lib/galaxy/tools/deps/containers.py | 17 +- 3 files changed, 137 insertions(+), 60 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 83ab02a901a..e6a04cae318 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -247,7 +247,11 @@ class Configuration( object ): log.warn("preserve_python_environment set to unknown value [%s], defaulting to legacy_only") preserve_python_environment = "legacy_only" self.preserve_python_environment = preserve_python_environment + # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we + # should probably drop the backward compatiblity to save the path check. self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) ) + if not os.path.exists( self.container_image_cache_path ): + self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_cache" ) ) self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) ) self.retry_job_output_collection = int( kwargs.get( 'retry_job_output_collection', 0 ) ) diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index cccc84320b9..3e355d96818 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -2,6 +2,7 @@ import collections import logging +import os import six @@ -37,7 +38,7 @@ CachedV1MulledImageMultiTarget.multi_target = "v1" CachedV2MulledImageMultiTarget.multi_target = "v2" -def list_cached_mulled_images(namespace=None, hash_func="v2"): +def list_docker_cached_mulled_images(namespace=None, hash_func="v2"): command = build_docker_images_command(truncate=True, sudo=False) command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command images_and_versions = check_output(command) @@ -46,42 +47,7 @@ def list_cached_mulled_images(namespace=None, hash_func="v2"): def output_line_to_image(line): image_name, version = line.split(" ", 1) identifier = "%s:%s" % (image_name, version) - url, namespace, package_description = image_name.split("/") - if not version or version == "latest": - version = None - - image = None - if package_description.startswith("mulled-v1-"): - if hash_func == "v2": - return None - - hash = package_description - build = None - if version and version.isdigit(): - build = version - image = CachedV1MulledImageMultiTarget(hash, build, identifier) - elif package_description.startswith("mulled-v2-"): - if hash_func == "v1": - return None - - version_hash = None - build = None - - if version and "-" in version: - version_hash, build = version.rsplit("-", 1) - elif version.isdigit(): - version_hash, build = None, version - elif version: - log.debug("Unparsable mulled image tag encountered [%s]" % version) - - image = CachedV2MulledImageMultiTarget(package_description, version_hash, build, identifier) - else: - build = None - if version and "--" in version: - version, build = split_tag(version) - - image = CachedMulledImageSingleTarget(image_name, version, build, identifier) - + image = identifier_to_cached_target(identifier, hash_func) return image # TODO: Sort on build ... @@ -89,16 +55,62 @@ def list_cached_mulled_images(namespace=None, hash_func="v2"): return [i for i in raw_images if i is not None] +def identifier_to_cached_target(identifier, hash_func): + image_name, version = identifier.rsplit(":", 1) + _, package_description = image_name.rsplit("/", 1) + if not version or version == "latest": + version = None + + image = None + if package_description.startswith("mulled-v1-"): + if hash_func == "v2": + return None + + hash = package_description + build = None + if version and version.isdigit(): + build = version + image = CachedV1MulledImageMultiTarget(hash, build, identifier) + elif package_description.startswith("mulled-v2-"): + if hash_func == "v1": + return None + + version_hash = None + build = None + + if version and "-" in version: + version_hash, build = version.rsplit("-", 1) + elif version.isdigit(): + version_hash, build = None, version + elif version: + log.debug("Unparsable mulled image tag encountered [%s]" % version) + + image = CachedV2MulledImageMultiTarget(package_description, version_hash, build, identifier) + else: + build = None + if version and "--" in version: + version, build = split_tag(version) + + image = CachedMulledImageSingleTarget(image_name, version, build, identifier) + + return image + + +def list_cached_mulled_images_from_path(directory, hash_func="v2"): + contents = os.listdir(directory) + raw_images = map(lambda name: identifier_to_cached_target(name, hash_func), contents) + return [i for i in raw_images if i is not None] + + def get_filter(namespace): prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace return lambda name: name.startswith(prefix) and name.count("/") == 2 -def cached_container_description(targets, namespace, hash_func="v2"): +def find_best_matching_cached_image(targets, cached_images, hash_func): if len(targets) == 0: return None - cached_images = list_cached_mulled_images(namespace, hash_func=hash_func) image = None if len(targets) == 1: target = targets[0] @@ -142,6 +154,16 @@ def cached_container_description(targets, namespace, hash_func="v2"): image = cached_image break + return image + + +def docker_cached_container_description(targets, namespace, hash_func="v2"): + if len(targets) == 0: + return None + + cached_images = list_docker_cached_mulled_images(namespace, hash_func=hash_func) + image = find_best_matching_cached_image(targets, cached_images, hash_func) + container = None if image: container = ContainerDescription( @@ -152,13 +174,34 @@ def cached_container_description(targets, namespace, hash_func="v2"): return container +def singularity_cached_container_description(targets, cache_directory, hash_func="v2"): + if len(targets) == 0: + return None + + if not os.path.exists(cache_directory): + return None + + cached_images = list_cached_mulled_images_from_path(cache_directory, hash_func=hash_func) + image = find_best_matching_cached_image(targets, cached_images, hash_func) + + container = None + if image: + container = ContainerDescription( + os.path.join(cache_directory, image.image_identifier), + type="singularity", + ) + + return container + + @six.python_2_unicode_compatible -class CachedMulledContainerResolver(ContainerResolver): +class CachedMulledDockerContainerResolver(ContainerResolver): resolver_type = "cached_mulled" + container_type = "docker" def __init__(self, app_info=None, namespace=None, hash_func="v2"): - super(CachedMulledContainerResolver, self).__init__(app_info) + super(CachedMulledDockerContainerResolver, self).__init__(app_info) self.namespace = namespace self.hash_func = hash_func @@ -167,20 +210,43 @@ class CachedMulledContainerResolver(ContainerResolver): return None targets = mulled_targets(tool_info) - return cached_container_description(targets, self.namespace, hash_func=self.hash_func) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func) def __str__(self): - return "CachedMulledContainerResolver[namespace=%s]" % self.namespace + return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace @six.python_2_unicode_compatible -class MulledContainerResolver(ContainerResolver): +class CachedMulledSingularityContainerResolver(ContainerResolver): + + resolver_type = "cached_mulled_singularity" + container_type = "singularity" + + def __init__(self, app_info=None, hash_func="v2"): + super(CachedMulledDockerContainerResolver, self).__init__(app_info) + self.cache_directory = os.path.join(app_info.container_image_cache_path, "singularity", "mulled") + self.hash_func = hash_func + + def resolve(self, enabled_container_types, tool_info): + if tool_info.requires_galaxy_python_environment: + return None + + targets = mulled_targets(tool_info) + return singularity_cached_container_description(targets, hash_func=self.hash_func) + + def __str__(self): + return "CachedMulledSingularityContainerResolver[namespace=%s]" % self.namespace + + +@six.python_2_unicode_compatible +class MulledDockerContainerResolver(ContainerResolver): """Look for mulled images matching tool dependencies.""" resolver_type = "mulled" + container_type = "docker" def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): - super(MulledContainerResolver, self).__init__(app_info) + super(MulledDockerContainerResolver, self).__init__(app_info) self.namespace = namespace self.hash_func = hash_func @@ -235,21 +301,22 @@ class MulledContainerResolver(ContainerResolver): if name: return ContainerDescription( "quay.io/%s/%s" % (self.namespace, name), - type="docker", + type=self.container_type, ) def __str__(self): - return "MulledContainerResolver[namespace=%s]" % self.namespace + return "MulledDockerContainerResolver[namespace=%s]" % self.namespace @six.python_2_unicode_compatible -class BuildMulledContainerResolver(ContainerResolver): +class BuildMulledDockerContainerResolver(ContainerResolver): """Look for mulled images matching tool dependencies.""" resolver_type = "build_mulled" + container_type = "docker" def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds): - super(BuildMulledContainerResolver, self).__init__(app_info) + super(BuildMulledDockerContainerResolver, self).__init__(app_info) self._involucro_context_kwds = { 'involucro_bin': self._get_config_option("involucro_path", None) } @@ -275,7 +342,7 @@ class BuildMulledContainerResolver(ContainerResolver): hash_func=self.hash_func, **self._mulled_kwds ) - return cached_container_description(targets, self.namespace, hash_func=self.hash_func) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func) def _get_involucro_context(self): involucro_context = InvolucroContext(**self._involucro_context_kwds) @@ -283,7 +350,7 @@ class BuildMulledContainerResolver(ContainerResolver): return involucro_context def __str__(self): - return "BuildContainerResolver[namespace=%s]" % self.namespace + return "BuildDockerContainerResolver[namespace=%s]" % self.namespace def mulled_targets(tool_info): @@ -291,7 +358,8 @@ def mulled_targets(tool_info): __all__ = ( - "CachedMulledContainerResolver", - "MulledContainerResolver", - "BuildMulledContainerResolver", + "CachedMulledDockerContainerResolver", + "CachedMulledSingularityContainerResolver", + "MulledDockerContainerResolver", + "BuildMulledDockerContainerResolver", ) diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index ca9bbf2038e..973fca0d7ad 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -14,9 +14,10 @@ from galaxy.util import plugin_config from .container_resolvers.explicit import ExplicitContainerResolver from .container_resolvers.mulled import ( - BuildMulledContainerResolver, - CachedMulledContainerResolver, - MulledContainerResolver, + BuildMulledDockerContainerResolver, + CachedMulledDockerContainerResolver, + CachedMulledSingularityContainerResolver, + MulledDockerContainerResolver, ) from .requirements import ContainerDescription from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL @@ -218,9 +219,10 @@ class ContainerRegistry(object): ] if self.enable_beta_mulled_containers: default_resolvers.extend([ - CachedMulledContainerResolver(self.app_info), - MulledContainerResolver(self.app_info, namespace="biocontainers"), - BuildMulledContainerResolver(self.app_info), + CachedMulledDockerContainerResolver(self.app_info), + MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + BuildMulledDockerContainerResolver(self.app_info), + CachedMulledSingularityContainerResolver(self.app_info), ]) return default_resolvers @@ -231,6 +233,9 @@ class ContainerRegistry(object): def find_best_container_description(self, enabled_container_types, tool_info): """Yield best container description of supplied types matching tool info.""" for container_resolver in self.container_resolvers: + if hasattr(container_resolver, "container_type"): + if container_resolver.container_type not in enabled_container_types: + continue container_description = container_resolver.resolve(enabled_container_types, tool_info) log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description)) if container_description: