Allow Galaxy to use cached mulled containers for Singularity.

- Change undocumented (and probably unused) config option container_image_cache_path default to "database/contianer_cache". - Update container resolver framework to let plugins assert they only produce containers of a single type. - Update mulled container resolvers to indicate they produce Docker containers only - update names also. - Refactor mulled caching stuff to allow reuse with Singularity (where the names come from is different - but a lot of the logic is the same). - Add a cached singularity container resolver - that searches "config.container_image_cache_path/singularity/mulled" for an image with the correct name. Next up - similarly refactor the BuildMulledDockerContainerResolver to allow reuse and implement the BuildMulledSingularityContainerResolver.
2026-10-24 02:04:04 +08:00 · 2017-06-12 08:33:26 -04:00
parent d36c4053cd
commit 8d224edb57
3 changed files with 137 additions and 60 deletions
@@ -247,7 +247,11 @@ class Configuration( object ):
            log.warn("preserve_python_environment set to unknown value [%s], defaulting to legacy_only")
            preserve_python_environment = "legacy_only"
        self.preserve_python_environment = preserve_python_environment
+        # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we
+        # should probably drop the backward compatiblity to save the path check.
        self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) )
+        if not os.path.exists( self.container_image_cache_path ):
+            self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_cache" ) )
        self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
        self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )
        self.retry_job_output_collection = int( kwargs.get( 'retry_job_output_collection', 0 ) )
@@ -2,6 +2,7 @@

 import collections
 import logging
+import os

 import six

@@ -37,7 +38,7 @@ CachedV1MulledImageMultiTarget.multi_target = "v1"
 CachedV2MulledImageMultiTarget.multi_target = "v2"


-def list_cached_mulled_images(namespace=None, hash_func="v2"):
+def list_docker_cached_mulled_images(namespace=None, hash_func="v2"):
    command = build_docker_images_command(truncate=True, sudo=False)
    command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
    images_and_versions = check_output(command)
@@ -46,42 +47,7 @@ def list_cached_mulled_images(namespace=None, hash_func="v2"):
    def output_line_to_image(line):
        image_name, version = line.split(" ", 1)
        identifier = "%s:%s" % (image_name, version)
-        url, namespace, package_description = image_name.split("/")
-        if not version or version == "latest":
-            version = None
-
-        image = None
-        if package_description.startswith("mulled-v1-"):
-            if hash_func == "v2":
-                return None
-
-            hash = package_description
-            build = None
-            if version and version.isdigit():
-                build = version
-            image = CachedV1MulledImageMultiTarget(hash, build, identifier)
-        elif package_description.startswith("mulled-v2-"):
-            if hash_func == "v1":
-                return None
-
-            version_hash = None
-            build = None
-
-            if version and "-" in version:
-                version_hash, build = version.rsplit("-", 1)
-            elif version.isdigit():
-                version_hash, build = None, version
-            elif version:
-                log.debug("Unparsable mulled image tag encountered [%s]" % version)
-
-            image = CachedV2MulledImageMultiTarget(package_description, version_hash, build, identifier)
-        else:
-            build = None
-            if version and "--" in version:
-                version, build = split_tag(version)
-
-            image = CachedMulledImageSingleTarget(image_name, version, build, identifier)
-
+        image = identifier_to_cached_target(identifier, hash_func)
        return image

    # TODO: Sort on build ...
@@ -89,16 +55,62 @@ def list_cached_mulled_images(namespace=None, hash_func="v2"):
    return [i for i in raw_images if i is not None]


+def identifier_to_cached_target(identifier, hash_func):
+    image_name, version = identifier.rsplit(":", 1)
+    _, package_description = image_name.rsplit("/", 1)
+    if not version or version == "latest":
+        version = None
+
+    image = None
+    if package_description.startswith("mulled-v1-"):
+        if hash_func == "v2":
+            return None
+
+        hash = package_description
+        build = None
+        if version and version.isdigit():
+            build = version
+        image = CachedV1MulledImageMultiTarget(hash, build, identifier)
+    elif package_description.startswith("mulled-v2-"):
+        if hash_func == "v1":
+            return None
+
+        version_hash = None
+        build = None
+
+        if version and "-" in version:
+            version_hash, build = version.rsplit("-", 1)
+        elif version.isdigit():
+            version_hash, build = None, version
+        elif version:
+            log.debug("Unparsable mulled image tag encountered [%s]" % version)
+
+        image = CachedV2MulledImageMultiTarget(package_description, version_hash, build, identifier)
+    else:
+        build = None
+        if version and "--" in version:
+            version, build = split_tag(version)
+
+        image = CachedMulledImageSingleTarget(image_name, version, build, identifier)
+
+    return image
+
+
+def list_cached_mulled_images_from_path(directory, hash_func="v2"):
+    contents = os.listdir(directory)
+    raw_images = map(lambda name: identifier_to_cached_target(name, hash_func), contents)
+    return [i for i in raw_images if i is not None]
+
+
 def get_filter(namespace):
    prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace
    return lambda name: name.startswith(prefix) and name.count("/") == 2


-def cached_container_description(targets, namespace, hash_func="v2"):
+def find_best_matching_cached_image(targets, cached_images, hash_func):
    if len(targets) == 0:
        return None

-    cached_images = list_cached_mulled_images(namespace, hash_func=hash_func)
    image = None
    if len(targets) == 1:
        target = targets[0]
@@ -142,6 +154,16 @@ def cached_container_description(targets, namespace, hash_func="v2"):
                image = cached_image
                break

+    return image
+
+
+def docker_cached_container_description(targets, namespace, hash_func="v2"):
+    if len(targets) == 0:
+        return None
+
+    cached_images = list_docker_cached_mulled_images(namespace, hash_func=hash_func)
+    image = find_best_matching_cached_image(targets, cached_images, hash_func)
+
    container = None
    if image:
        container = ContainerDescription(
@@ -152,13 +174,34 @@ def cached_container_description(targets, namespace, hash_func="v2"):
    return container


+def singularity_cached_container_description(targets, cache_directory, hash_func="v2"):
+    if len(targets) == 0:
+        return None
+
+    if not os.path.exists(cache_directory):
+        return None
+
+    cached_images = list_cached_mulled_images_from_path(cache_directory, hash_func=hash_func)
+    image = find_best_matching_cached_image(targets, cached_images, hash_func)
+
+    container = None
+    if image:
+        container = ContainerDescription(
+            os.path.join(cache_directory, image.image_identifier),
+            type="singularity",
+        )
+
+    return container
+
+
@six.python_2_unicode_compatible
-class CachedMulledContainerResolver(ContainerResolver):
+class CachedMulledDockerContainerResolver(ContainerResolver):

    resolver_type = "cached_mulled"
+    container_type = "docker"

    def __init__(self, app_info=None, namespace=None, hash_func="v2"):
-        super(CachedMulledContainerResolver, self).__init__(app_info)
+        super(CachedMulledDockerContainerResolver, self).__init__(app_info)
        self.namespace = namespace
        self.hash_func = hash_func

@@ -167,20 +210,43 @@ class CachedMulledContainerResolver(ContainerResolver):
            return None

        targets = mulled_targets(tool_info)
-        return cached_container_description(targets, self.namespace, hash_func=self.hash_func)
+        return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func)

    def __str__(self):
-        return "CachedMulledContainerResolver[namespace=%s]" % self.namespace
+        return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace


@six.python_2_unicode_compatible
-class MulledContainerResolver(ContainerResolver):
+class CachedMulledSingularityContainerResolver(ContainerResolver):
+
+    resolver_type = "cached_mulled_singularity"
+    container_type = "singularity"
+
+    def __init__(self, app_info=None, hash_func="v2"):
+        super(CachedMulledDockerContainerResolver, self).__init__(app_info)
+        self.cache_directory = os.path.join(app_info.container_image_cache_path, "singularity", "mulled")
+        self.hash_func = hash_func
+
+    def resolve(self, enabled_container_types, tool_info):
+        if tool_info.requires_galaxy_python_environment:
+            return None
+
+        targets = mulled_targets(tool_info)
+        return singularity_cached_container_description(targets, hash_func=self.hash_func)
+
+    def __str__(self):
+        return "CachedMulledSingularityContainerResolver[namespace=%s]" % self.namespace
+
+
+@six.python_2_unicode_compatible
+class MulledDockerContainerResolver(ContainerResolver):
    """Look for mulled images matching tool dependencies."""

    resolver_type = "mulled"
+    container_type = "docker"

    def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"):
-        super(MulledContainerResolver, self).__init__(app_info)
+        super(MulledDockerContainerResolver, self).__init__(app_info)
        self.namespace = namespace
        self.hash_func = hash_func

@@ -235,21 +301,22 @@ class MulledContainerResolver(ContainerResolver):
        if name:
            return ContainerDescription(
                "quay.io/%s/%s" % (self.namespace, name),
-                type="docker",
+                type=self.container_type,
            )

    def __str__(self):
-        return "MulledContainerResolver[namespace=%s]" % self.namespace
+        return "MulledDockerContainerResolver[namespace=%s]" % self.namespace


@six.python_2_unicode_compatible
-class BuildMulledContainerResolver(ContainerResolver):
+class BuildMulledDockerContainerResolver(ContainerResolver):
    """Look for mulled images matching tool dependencies."""

    resolver_type = "build_mulled"
+    container_type = "docker"

    def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds):
-        super(BuildMulledContainerResolver, self).__init__(app_info)
+        super(BuildMulledDockerContainerResolver, self).__init__(app_info)
        self._involucro_context_kwds = {
            'involucro_bin': self._get_config_option("involucro_path", None)
        }
@@ -275,7 +342,7 @@ class BuildMulledContainerResolver(ContainerResolver):
            hash_func=self.hash_func,
            **self._mulled_kwds
        )
-        return cached_container_description(targets, self.namespace, hash_func=self.hash_func)
+        return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func)

    def _get_involucro_context(self):
        involucro_context = InvolucroContext(**self._involucro_context_kwds)
@@ -283,7 +350,7 @@ class BuildMulledContainerResolver(ContainerResolver):
        return involucro_context

    def __str__(self):
-        return "BuildContainerResolver[namespace=%s]" % self.namespace
+        return "BuildDockerContainerResolver[namespace=%s]" % self.namespace


 def mulled_targets(tool_info):
@@ -291,7 +358,8 @@ def mulled_targets(tool_info):


 __all__ = (
-    "CachedMulledContainerResolver",
-    "MulledContainerResolver",
-    "BuildMulledContainerResolver",
+    "CachedMulledDockerContainerResolver",
+    "CachedMulledSingularityContainerResolver",
+    "MulledDockerContainerResolver",
+    "BuildMulledDockerContainerResolver",
 )
@@ -14,9 +14,10 @@ from galaxy.util import plugin_config

 from .container_resolvers.explicit import ExplicitContainerResolver
 from .container_resolvers.mulled import (
-    BuildMulledContainerResolver,
-    CachedMulledContainerResolver,
-    MulledContainerResolver,
+    BuildMulledDockerContainerResolver,
+    CachedMulledDockerContainerResolver,
+    CachedMulledSingularityContainerResolver,
+    MulledDockerContainerResolver,
 )
 from .requirements import ContainerDescription
 from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL
@@ -218,9 +219,10 @@ class ContainerRegistry(object):
        ]
        if self.enable_beta_mulled_containers:
            default_resolvers.extend([
-                CachedMulledContainerResolver(self.app_info),
-                MulledContainerResolver(self.app_info, namespace="biocontainers"),
-                BuildMulledContainerResolver(self.app_info),
+                CachedMulledDockerContainerResolver(self.app_info),
+                MulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
+                BuildMulledDockerContainerResolver(self.app_info),
+                CachedMulledSingularityContainerResolver(self.app_info),
            ])
        return default_resolvers

@@ -231,6 +233,9 @@ class ContainerRegistry(object):
    def find_best_container_description(self, enabled_container_types, tool_info):
        """Yield best container description of supplied types matching tool info."""
        for container_resolver in self.container_resolvers:
+            if hasattr(container_resolver, "container_type"):
+                if container_resolver.container_type not in enabled_container_types:
+                    continue
            container_description = container_resolver.resolve(enabled_container_types, tool_info)
            log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description))
            if container_description: