opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix pulling of singularity images

Browse Source 
That broke in https://github.com/galaxyproject/galaxy/pull/11134, since
the singularity container resolve() inherits from docker container
resolution it'd also check to see if `docker` was on path.
This commit is contained in:
mvdbeek
2021-01-20 12:06:32 +01:00
parent 17ec062514
commit e428e4298e
3 changed files with 37 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_util/deps/container_resolvers/mulled.py
+30 -17
View File
@@ -323,26 +323,27 @@ def targets_to_mulled_name(targets, hash_func, namespace, resolution_cache=None,
return name
class DockerContainerResolver(ContainerResolver):
class CliContainerResolver(ContainerResolver):
container_type = 'docker'
cli = 'docker'
def __init__(self, *args, **kwargs):
self._docker_cli_available = bool(which('docker'))
self._cli_available = bool(which(self.cli))
super().__init__(*args, **kwargs)
@property
def docker_cli_available(self):
return self._docker_cli_available
def cli_available(self):
return self._cli_available
@docker_cli_available.setter
def docker_cli_available(self, value):
@cli_available.setter
def cli_available(self, value):
if not value:
log.info('Docker CLI not available, cannot list or pull images in Galaxy process. Does not impact kubernetes.')
self._docker_cli_available = value
log.info('{} CLI not available, cannot list or pull images in Galaxy process. Does not impact kubernetes.'.format(self.cli))
self._cli_available = value
class CachedMulledDockerContainerResolver(DockerContainerResolver):
class CachedMulledDockerContainerResolver(CliContainerResolver):
resolver_type = "cached_mulled"
shell = '/bin/bash'
@@ -353,7 +354,7 @@ class CachedMulledDockerContainerResolver(DockerContainerResolver):
self.hash_func = hash_func
def resolve(self, enabled_container_types, tool_info, **kwds):
if not self.docker_cli_available or tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
if not self.cli_available or tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
targets = mulled_targets(tool_info)
@@ -364,9 +365,10 @@ class CachedMulledDockerContainerResolver(DockerContainerResolver):
return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace
class CachedMulledSingularityContainerResolver(ContainerResolver):
class CachedMulledSingularityContainerResolver(CliContainerResolver):
resolver_type = "cached_mulled_singularity"
cli = "singularity"
container_type = "singularity"
shell = '/bin/bash'
@@ -386,7 +388,7 @@ class CachedMulledSingularityContainerResolver(ContainerResolver):
return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory
class MulledDockerContainerResolver(DockerContainerResolver):
class MulledDockerContainerResolver(CliContainerResolver):
"""Look for mulled images matching tool dependencies."""
resolver_type = "mulled"
@@ -409,10 +411,14 @@ class MulledDockerContainerResolver(DockerContainerResolver):
return None
def pull(self, container):
if self.docker_cli_available:
if self.cli_available:
command = container.build_pull_command()
shell(command)
@property
def can_list_containers(self):
return self.cli_available
def resolve(self, enabled_container_types, tool_info, install=False, session=None, **kwds):
resolution_cache = kwds.get("resolution_cache")
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
@@ -432,7 +438,7 @@ class MulledDockerContainerResolver(DockerContainerResolver):
type=self.container_type,
shell=self.shell,
)
if self.docker_cli_available:
if self.can_list_containers:
if install and not self.cached_container_description(
targets,
namespace=self.namespace,
@@ -467,6 +473,7 @@ class MulledSingularityContainerResolver(MulledDockerContainerResolver):
resolver_type = "mulled_singularity"
container_type = "singularity"
cli = "singularity"
protocol = 'docker://'
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
@@ -481,15 +488,21 @@ class MulledSingularityContainerResolver(MulledDockerContainerResolver):
cache_directory=self.cache_directory,
hash_func=hash_func)
@property
def can_list_containers(self):
# Only needs access to path, doesn't require CLI
return True
def pull(self, container):
cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace)
shell(cmds=cmds)
if self.cli_available:
cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace)
shell(cmds=cmds)
def __str__(self):
return "MulledSingularityContainerResolver[namespace=%s]" % self.namespace
class BuildMulledDockerContainerResolver(DockerContainerResolver):
class BuildMulledDockerContainerResolver(CliContainerResolver):
"""Build for Docker mulled images matching tool dependencies."""
resolver_type = "build_mulled"
lib/galaxy/tool_util/deps/containers.py
+1 -1
View File
@@ -226,7 +226,7 @@ class ContainerRegistry:
# BuildMulledDockerContainerResolver and BuildMulledSingularityContainerResolver both need the docker daemon to build images.
# If docker is not available, we don't load them.
build_mulled_docker_container_resolver = BuildMulledDockerContainerResolver(self.app_info)
if build_mulled_docker_container_resolver.docker_cli_available:
if build_mulled_docker_container_resolver.cli_available:
default_resolvers.extend([
build_mulled_docker_container_resolver,
BuildMulledSingularityContainerResolver(self.app_info),
test/unit/tool_util/test_container_resolution.py
+6 -6
View File
@@ -11,26 +11,26 @@ from galaxy.tool_util.deps.requirements import ToolRequirement
def test_docker_container_resolver_detects_docker_cli_absent(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None)
resolver = CachedMulledDockerContainerResolver()
assert resolver.docker_cli_available is False
assert resolver._cli_available is False
def test_docker_container_resolver_detects_docker_cli(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled', return_value='/bin/docker')
resolver = CachedMulledDockerContainerResolver()
assert resolver.docker_cli_available
assert resolver.cli_available
def test_cached_docker_container_docker_cli_absent_resolve(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None)
resolver = CachedMulledDockerContainerResolver()
assert resolver.docker_cli_available is False
assert resolver.cli_available is False
assert resolver.resolve(enabled_container_types=[], tool_info={}) is None
def test_docker_container_docker_cli_absent_resolve(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value=None)
resolver = MulledDockerContainerResolver()
assert resolver.docker_cli_available is False
assert resolver.cli_available is False
requirement = ToolRequirement(name="samtools", version="1.10", type="package")
tool_info = ToolInfo(requirements=[requirement])
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3')
@@ -42,12 +42,12 @@ def test_docker_container_docker_cli_absent_resolve(mocker):
def test_docker_container_docker_cli_exception_resolve(mocker):
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.which', return_value='/bin/docker')
resolver = MulledDockerContainerResolver()
assert resolver.docker_cli_available is True
assert resolver.cli_available is True
requirement = ToolRequirement(name="samtools", version="1.10", type="package")
tool_info = ToolInfo(requirements=[requirement])
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.targets_to_mulled_name', return_value='samtools:1.10--h2e538c0_3')
mocker.patch('galaxy.tool_util.deps.container_resolvers.mulled.docker_cached_container_description', side_effect=CalledProcessError(1, 'bla'))
container_description = resolver.resolve(enabled_container_types=['docker'], tool_info=tool_info, install=True)
assert resolver.docker_cli_available is True
assert resolver.cli_available is True
assert container_description.type == 'docker'
assert container_description.identifier == 'quay.io/biocontainers/samtools:1.10--h2e538c0_3'