opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7125 from mvdbeek/MulledSingularityContainerResolver

Browse Source 
Use singularity build / exec with biocontainers
This commit is contained in:
John Chilton
2019-01-07 08:35:44 -05:00
committed by GitHub
parent 952af6b6fd 8a34101089
commit 03b363a1ef
37 changed files with 1079 additions and 676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/jenkins/integration-py3/run_tests.sh
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/bash
./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@"
DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@"
.ci/jenkins/integration/run_tests.sh
+1 -1
View File
@@ -1,3 +1,3 @@
#!/bin/bash
./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@"
DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@"
config/container_resolvers_conf.xml.sample
+33 -10
View File
@@ -3,8 +3,8 @@
<!-- explicit: resolves container URI for a job through explict container
tags in the tool XML wrapper. -->
<!-- All mulled flavors below only work if enable_beta_mulled_containers is
set to true in the galaxy.yaml config file.
<!-- All mulled flavors below only work if enable_beta_mulled_containers is
set to true in the galaxy.yaml config file. -->
<!-- <cached_mulled /> -->
<!-- cached_mulled: resolves container URI through bioconda to mulled
@@ -18,19 +18,42 @@
enable_beta_mulled_containers set to true in the galaxy.yaml config file.
-->
<!-- <mulled /> -->
<!-- <mulled auto_install="True"/> -->
<!-- mulled: only resolves container URI through bioconda to mulled automatic
mapping. Should be the method of choice if mulled resolution is required
but the container execution is handled by a container orchestration
instead of docker directly (ie. Galaxy has no access to a docker client
but the container execution is handled by a container orchestration
instead of docker directly (ie. Galaxy has no access to a docker client
binary). It works of course with direct docker engine tool executions.
Set auto_install to False if Galaxy should pull container images
through the admin interface or API, but not automatically when
a tool is run.
-->
<!-- <build_mulled /> -->
<!-- <mulled_singularity auto_install="True"/> -->
<!-- mulled_singularity: only resolves container URI through bioconda to mulled automatic
mapping. Should be the method of choice if mulled resolution is required
but the container execution is handled by a container orchestration
instead of singularity directly (ie. Galaxy has no access to a singularity client
binary). It works of course with direct singularity tool executions.
Set auto_install to False if Galaxy should pull container images
through the admin interface or API, but not automatically when
a tool is run.
-->
<!-- <build_mulled auto_install="True"/> -->
<!-- build_mulled: builds a docker image locally for one or more bioconda
packages listed in the tools definition. -->
<!-- <build_mulled_singularity /> -->
packages listed in the tools definition.
Set auto_install to False if Galaxy should build container images
through the admin interface or API, but not automatically when
a tool is run.
-->
<!-- <build_mulled_singularity auto_install="True"/> -->
<!-- build_mulled_singularity: builds a singularity image locally for one
or more package requirements listed as part of the tool's definition. -->
or more package requirements listed as part of the tool's definition.
Set auto_install to False if Galaxy should build container images
through the admin interface or API, but not automatically when
a tool is run.
-->
</containers_resolvers>
config/galaxy.yml.sample
+13 -7
View File
@@ -330,17 +330,23 @@ galaxy:
# by enable_beta_mulled_containers.
#containers_resolvers_config_file: ''
# involucro is a tool used to build Docker containers for tools from
# Conda dependencies referenced in tools as `requirement`s. The
# following path is the location of involucro on the Galaxy host. This
# is ignored if the relevant container resolver isn't enabled, and
# will install on demand unless involucro_auto_init is set to False.
# involucro is a tool used to build Docker or Singularity containers
# for tools from Conda dependencies referenced in tools as
# `requirement`s. The following path is the location of involucro on
# the Galaxy host. This is ignored if the relevant container resolver
# isn't enabled, and will install on demand unless involucro_auto_init
# is set to False.
#involucro_path: database/dependencies/involucro
# Install involucro as needed to build Docker containers for tools.
# Ignored if relevant container resolver is not used.
# Install involucro as needed to build Docker or Singularity
# containers for tools. Ignored if relevant container resolver is not
# used.
#involucro_auto_init: true
# Conda channels to use when building Docker or Singularity containers
# using involucro.
#mulled_channels: 'conda-forge,bioconda'
# Enable automatic polling of relative tool sheds to see if any
# updates are available for installed repositories. Ideally only one
# Galaxy server process should be able to check for repository
doc/source/admin/galaxy_options.rst
+20 -8
View File
@@ -521,12 +521,12 @@
~~~~~~~~~~~~~~~~~~
:Description:
involucro is a tool used to build Docker containers for tools from
Conda dependencies referenced in tools as `requirement`s. The
following path is the location of involucro on the Galaxy host.
This is ignored if the relevant container resolver isn't enabled,
and will install on demand unless involucro_auto_init is set to
False.
involucro is a tool used to build Docker or Singularity containers
for tools from Conda dependencies referenced in tools as
`requirement`s. The following path is the location of involucro on
the Galaxy host. This is ignored if the relevant container
resolver isn't enabled, and will install on demand unless
involucro_auto_init is set to False.
:Default: ``database/dependencies/involucro``
:Type: str
@@ -536,12 +536,24 @@
~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Install involucro as needed to build Docker containers for tools.
Ignored if relevant container resolver is not used.
Install involucro as needed to build Docker or Singularity
containers for tools. Ignored if relevant container resolver is
not used.
:Default: ``true``
:Type: bool
~~~~~~~~~~~~~~~~~~~
``mulled_channels``
~~~~~~~~~~~~~~~~~~~
:Description:
Conda channels to use when building Docker or Singularity
containers using involucro.
:Default: ``conda-forge,bioconda``
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~~~~
``enable_tool_shed_check``
~~~~~~~~~~~~~~~~~~~~~~~~~~
lib/galaxy/config.py
+21 -1
View File
@@ -4,6 +4,7 @@ Universe configuration builder.
# absolute_import needed for tool_shed package.
from __future__ import absolute_import
import collections
import ipaddress
import logging
import logging.config
@@ -23,6 +24,7 @@ from six.moves import configparser
from galaxy.containers import parse_containers_config
from galaxy.exceptions import ConfigurationError
from galaxy.tools.deps.container_resolvers.mulled import DEFAULT_CHANNELS
from galaxy.util import ExecutionTimer
from galaxy.util import listify
from galaxy.util import string_as_bool
@@ -518,6 +520,11 @@ class Configuration(object):
involucro_path = os.path.join(tool_dependency_dir or "database", "involucro")
self.involucro_path = resolve_path(involucro_path, self.root)
self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True))
mulled_channels = kwargs.get('mulled_channels')
if mulled_channels:
self.mulled_channels = [c.strip() for c in mulled_channels.split(',')]
else:
self.mulled_channels = DEFAULT_CHANNELS
default_job_resubmission_condition = kwargs.get('default_job_resubmission_condition', '')
if not default_job_resubmission_condition.strip():
@@ -1035,6 +1042,7 @@ class ConfiguresGalaxyMixin(object):
from galaxy import tools
from galaxy.managers.citations import CitationsManager
from galaxy.tools.deps import containers
from galaxy.tools.deps.dependencies import AppInfo
import galaxy.tools.search
self.citations_manager = CitationsManager(self)
@@ -1047,7 +1055,7 @@ class ConfiguresGalaxyMixin(object):
self.toolbox = tools.ToolBox(tool_configs, self.config.tool_path, self)
galaxy_root_dir = os.path.abspath(self.config.root)
file_path = os.path.abspath(getattr(self.config, "file_path"))
app_info = containers.AppInfo(
app_info = AppInfo(
galaxy_root_dir=galaxy_root_dir,
default_file_path=file_path,
outputs_to_working_directory=self.config.outputs_to_working_directory,
@@ -1057,8 +1065,10 @@ class ConfiguresGalaxyMixin(object):
containers_resolvers_config_file=self.config.containers_resolvers_config_file,
involucro_path=self.config.involucro_path,
involucro_auto_init=self.config.involucro_auto_init,
mulled_channels=self.config.mulled_channels,
)
self.container_finder = containers.ContainerFinder(app_info)
self._set_enabled_container_types()
index_help = getattr(self.config, "index_tool_help", True)
self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_help)
self.reindex_tool_search()
@@ -1068,6 +1078,16 @@ class ConfiguresGalaxyMixin(object):
self.toolbox_search.build_index(tool_cache=self.tool_cache)
self.tool_cache.reset_status()
def _set_enabled_container_types(self):
container_types_to_destinations = collections.defaultdict(list)
for destinations in self.job_config.destinations.values():
for destination in destinations:
for enabled_container_type in self.container_finder._enabled_container_types(destination.params):
container_types_to_destinations[enabled_container_type].append(destination)
self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations)
self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes)
self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers)
def _configure_tool_data_tables(self, from_shed_config):
from galaxy.tools.data import ToolDataTableManager
lib/galaxy/jobs/__init__.py
+11 -4
View File
@@ -685,6 +685,7 @@ class JobWrapper(HasResourceParameters):
self.extra_filenames = []
self.command_line = None
self.dependencies = []
self._dependency_shell_commands = None
# Tool versioning variables
self.write_version_cmd = None
self.version_string = ""
@@ -728,6 +729,15 @@ class JobWrapper(HasResourceParameters):
def dataset_path_rewriter(self):
return self._job_dataset_path_rewriter
@property
def dependency_shell_commands(self):
"""Shell fragment to inject dependencies."""
if self._dependency_shell_commands is None:
self._dependency_shell_commands = self.tool.build_dependency_shell_commands(
job_directory=self.working_directory
)
return self._dependency_shell_commands
@property
def cleanup_job(self):
""" Remove the job after it is complete, should return "always", "onsuccess", or "never".
@@ -860,8 +870,6 @@ class JobWrapper(HasResourceParameters):
self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build()
# Ensure galaxy_lib_dir is set in case there are any later chdirs
self.galaxy_lib_dir
# Shell fragment to inject dependencies
self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory)
if self.tool.requires_galaxy_python_environment:
# These tools (upload, metadata, data_source) may need access to the datatypes registry.
self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml'))
@@ -1959,8 +1967,7 @@ class TaskWrapper(JobWrapper):
# Ensure galaxy_lib_dir is set in case there are any later chdirs
self.galaxy_lib_dir
# Shell fragment to inject dependencies
self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory)
# We need command_line persisted to the db in order for Galaxy to re-queue the job
# if the server was stopped and restarted before the job finished
task.command_line = self.command_line
lib/galaxy/jobs/command_factory.py
+1 -2
View File
@@ -158,9 +158,8 @@ def __handle_task_splitting(commands_builder, job_wrapper):
def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params):
local_dependency_resolution = remote_command_params.get("dependency_resolution", "local") == "local"
# Prepend dependency injection
if job_wrapper.dependency_shell_commands and local_dependency_resolution:
if local_dependency_resolution and job_wrapper.dependency_shell_commands:
commands_builder.prepend_commands(job_wrapper.dependency_shell_commands)
lib/galaxy/jobs/rule_helper.py
+2 -3
View File
@@ -9,6 +9,7 @@ from galaxy import (
model,
util
)
from galaxy.tools.deps.dependencies import ToolInfo
log = logging.getLogger(__name__)
@@ -42,9 +43,7 @@ class RuleHelper(object):
else:
# Have a Job object.
tool = self.app.toolbox.get_tool(job_or_tool.tool_id, tool_version=job_or_tool.tool_version)
# Can't import at top because circular import between galaxy.tools and galaxy.jobs.
import galaxy.tools.deps.containers
tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
container_description = self.app.container_finder.find_best_container_description(["docker"], tool_info)
return container_description is not None
lib/galaxy/jobs/runners/__init__.py
+6 -3
View File
@@ -25,6 +25,10 @@ from galaxy.jobs.runners.util.job_script import (
job_script,
write_script
)
from galaxy.tools.deps.dependencies import (
JobInfo,
ToolInfo
)
from galaxy.util import (
DATABASE_MAX_STRING_SIZE,
ExecutionTimer,
@@ -394,9 +398,8 @@ class BaseJobRunner(object):
compute_tmp_directory = job_wrapper.tmp_directory()
tool = job_wrapper.tool
from galaxy.tools.deps import containers
tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
job_info = containers.JobInfo(
tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
job_info = JobInfo(
compute_working_directory,
compute_tool_directory,
compute_job_directory,
lib/galaxy/tools/deps/__init__.py
+99 -34
View File
@@ -13,11 +13,17 @@ from galaxy.util import (
plugin_config
)
from galaxy.util.oset import OrderedSet
from .container_resolvers import ContainerResolver
from .dependencies import ToolInfo
from .requirements import (
ContainerDescription,
ToolRequirement,
ToolRequirements
)
from .resolvers import NullDependency
from .resolvers import (
ContainerDependency,
NullDependency,
)
from .resolvers.conda import CondaDependencyResolver
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
@@ -44,19 +50,6 @@ def build_dependency_manager(config):
return dependency_manager
class NullDependencyManager(object):
dependency_resolvers = []
def uses_tool_shed_dependencies(self):
return False
def dependency_shell_commands(self, requirements, **kwds):
return []
def find_dep(self, name, version=None, type='package', **kwds):
return NullDependency(version=version, name=name)
class DependencyManager(object):
"""
A DependencyManager attempts to resolve named and versioned dependencies by
@@ -82,6 +75,27 @@ class DependencyManager(object):
self.default_base_path = os.path.abspath(default_base_path)
self.resolver_classes = self.__resolvers_dict()
self.dependency_resolvers = self.__build_dependency_resolvers(conf_file)
self._enabled_container_types = []
self._destination_for_container_type = {}
def set_enabled_container_types(self, container_types_to_destinations):
"""Set the union of all enabled container types."""
self._enabled_container_types = [container_type for container_type in container_types_to_destinations.keys()]
# Just pick first enabled destination for a container type, probably covers the most common deployment scenarios
self._destination_for_container_type = container_types_to_destinations
def get_destination_info_for_container_type(self, container_type, destination_id=None):
if destination_id is None:
return next(iter(self._destination_for_container_type[container_type])).params
else:
for destination in self._destination_for_container_type[container_type]:
if destination.id == destination_id:
return destination.params
@property
def enabled_container_types(self):
"""Returns the union of enabled container types."""
return self._enabled_container_types
def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
"""Look in resolver-specific settings for option and then fallback to global settings.
@@ -110,7 +124,7 @@ class DependencyManager(object):
def dependency_shell_commands(self, requirements, **kwds):
requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds)
ordered_dependencies = OrderedSet(requirements_to_dependencies.values())
return [dependency.shell_commands() for dependency in ordered_dependencies]
return [dependency.shell_commands() for dependency in ordered_dependencies if not isinstance(dependency, ContainerDependency)]
def requirements_to_dependencies(self, requirements, **kwds):
"""
@@ -125,29 +139,58 @@ class DependencyManager(object):
return requirement_to_dependency
def _requirements_to_dependencies_dict(self, requirements, **kwds):
def _requirements_to_dependencies_dict(self, requirements, search=False, **kwds):
"""Build simple requirements to dependencies dict for resolution."""
requirement_to_dependency = OrderedDict()
index = kwds.get('index', None)
index = kwds.get('index')
install = kwds.get('install', False)
resolver_type = kwds.get('resolver_type')
require_exact = kwds.get('exact', False)
return_null_dependencies = kwds.get('return_null', False)
resolvable_requirements = requirements.resolvable
tool_info = ToolInfo(requirements=resolvable_requirements)
for i, resolver in enumerate(self.dependency_resolvers):
if index is not None and i != index:
continue
if len(requirement_to_dependency) == len(resolvable_requirements):
if resolver_type is not None and resolver.resolver_type != resolver_type:
continue
_requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)])
if len(_requirement_to_dependency) == len(resolvable_requirements):
# Shortcut - resolution complete.
break
if resolver.resolver_type.startswith('build_mulled') and not install:
# don't want to build images here
continue
# Check requirements all at once
all_unmet = len(requirement_to_dependency) == 0
if all_unmet and hasattr(resolver, "resolve_all"):
all_unmet = len(_requirement_to_dependency) == 0
if hasattr(resolver, "resolve_all"):
resolve = resolver.resolve_all
elif isinstance(resolver, ContainerResolver):
if not resolver.resolver_type.startswith(('cached', 'explicit')) and not (search or install):
# These would look up available containers using the quay API,
# we only want to do this if we search for containers
continue
resolve = resolver.resolve
else:
resolve = None
if all_unmet and resolve is not None:
# TODO: Handle specs.
dependencies = resolver.resolve_all(resolvable_requirements, **kwds)
dependencies = resolve(requirements=resolvable_requirements,
enabled_container_types=self.enabled_container_types,
destination_for_container_type=self.get_destination_info_for_container_type,
tool_info=tool_info,
**kwds)
if dependencies:
if isinstance(dependencies, ContainerDescription):
dependencies = [ContainerDependency(dependencies, name=r.name, version=r.version) for r in resolvable_requirements]
assert len(dependencies) == len(resolvable_requirements)
for requirement, dependency in zip(resolvable_requirements, dependencies):
log.debug(dependency.resolver_msg)
@@ -156,21 +199,24 @@ class DependencyManager(object):
# Shortcut - resolution complete.
break
# Check individual requirements
for requirement in resolvable_requirements:
if requirement in requirement_to_dependency:
continue
if not isinstance(resolver, ContainerResolver):
dependency = resolver.resolve(requirement, **kwds)
if require_exact and not dependency.exact:
continue
# Check individual requirements
for requirement in resolvable_requirements:
if requirement in _requirement_to_dependency:
continue
if not isinstance(dependency, NullDependency):
log.debug(dependency.resolver_msg)
requirement_to_dependency[requirement] = dependency
elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index):
log.debug(dependency.resolver_msg)
requirement_to_dependency[requirement] = dependency
dependency = resolver.resolve(requirement, **kwds)
if require_exact and not dependency.exact:
continue
if not isinstance(dependency, NullDependency):
log.debug(dependency.resolver_msg)
requirement_to_dependency[requirement] = dependency
elif return_null_dependencies:
log.debug(dependency.resolver_msg)
dependency.version = requirement.version
requirement_to_dependency[requirement] = dependency
return requirement_to_dependency
@@ -273,3 +319,22 @@ class CachedDependencyManager(DependencyManager):
"""
req_hashes = self.hash_dependencies(resolved_dependencies)
return os.path.abspath(os.path.join(self.tool_dependency_cache_dir, req_hashes))
class NullDependencyManager(DependencyManager):
def __init__(self, default_base_path=None, conf_file=None, app_config={}):
self.__app_config = app_config
self.resolver_classes = set()
self.dependency_resolvers = []
self._enabled_container_types = []
self._destination_for_container_type = {}
def uses_tool_shed_dependencies(self):
return False
def dependency_shell_commands(self, requirements, **kwds):
return []
def find_dep(self, name, version=None, type='package', **kwds):
return NullDependency(version=version, name=name)
lib/galaxy/tools/deps/container_classes.py
+357
View File
@@ -0,0 +1,357 @@
import os
import string
from abc import (
ABCMeta,
abstractmethod
)
import six
from galaxy.containers.docker_model import DockerVolume
from galaxy.util import (
asbool,
in_directory
)
from . import (
docker_util,
singularity_util
)
from .requirements import (
DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES,
DEFAULT_CONTAINER_SHELL,
)
DOCKER_CONTAINER_TYPE = "docker"
SINGULARITY_CONTAINER_TYPE = "singularity"
LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r'''
python << EOF
from __future__ import print_function
import json
import re
import subprocess
import tarfile
t = tarfile.TarFile("${cached_image_file}")
meta_str = t.extractfile('repositories').read()
meta = json.loads(meta_str)
tag, tag_value = meta.items()[0]
rev, rev_value = tag_value.items()[0]
cmd = "${images_cmd}"
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
stdo, stde = proc.communicate()
found = False
for line in stdo.split("\n"):
tmp = re.split(r'\s+', line)
if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value:
found = True
if not found:
print("Loading image")
cmd = "cat ${cached_image_file} | ${load_cmd}"
subprocess.check_call(cmd, shell=True)
EOF
'''
@six.add_metaclass(ABCMeta)
class Container(object):
def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description):
self.container_id = container_id
self.app_info = app_info
self.tool_info = tool_info
self.destination_info = destination_info
self.job_info = job_info
self.container_description = container_description
def prop(self, name, default):
destination_name = "docker_%s" % name
return self.destination_info.get(destination_name, default)
@property
def resolve_dependencies(self):
return DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES if not self.container_description else self.container_description.resolve_dependencies
@property
def shell(self):
return DEFAULT_CONTAINER_SHELL if not self.container_description else self.container_description.shell
@abstractmethod
def containerize_command(self, command):
"""
Use destination supplied container configuration parameters,
container_id, and command to build a new command that runs
input command in container.
"""
def preprocess_volumes(volumes_raw_str, container_type):
"""Process Galaxy volume specification string to either Docker or Singularity specification.
Galaxy allows the mount try "default_ro" which translates to ro for Docker and
ro for Singularity iff no subdirectories are rw (Singularity does not allow ro
parent directories with rw subdirectories).
>>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE)
['/a/b:rw']
>>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE)
['/a/b:ro', '/a/b/c:rw']
>>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE)
['/a/b:ro', '/a/b/c:rw']
>>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE)
['/a/b:rw', '/a/b/c:rw']
"""
volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")]
volumes = []
rw_paths = []
for volume_raw_str in volumes_raw_strs:
volume_parts = volume_raw_str.split(":")
if len(volume_parts) > 2:
raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str)
if len(volume_parts) == 1:
volume_parts.append("rw")
volumes.append(volume_parts)
if volume_parts[1] == "rw":
rw_paths.append(volume_parts[0])
for volume in volumes:
path = volume[0]
how = volume[1]
if how == "default_ro":
how = "ro"
if container_type == SINGULARITY_CONTAINER_TYPE:
for rw_path in rw_paths:
if in_directory(rw_path, path):
how = "rw"
volume[1] = how
return [":".join(v) for v in volumes]
class HasDockerLikeVolumes(object):
"""Mixin to share functionality related to Docker volume handling.
Singularity seems to have a fairly compatible syntax for volume handling.
"""
def _expand_volume_str(self, value):
if not value:
return value
template = string.Template(value)
variables = dict()
def add_var(name, value):
if value:
if not value.startswith("$"):
value = os.path.abspath(value)
variables[name] = value
add_var("working_directory", self.job_info.working_directory)
add_var("tmp_directory", self.job_info.tmp_directory)
add_var("job_directory", self.job_info.job_directory)
add_var("tool_directory", self.job_info.tool_directory)
add_var("galaxy_root", self.app_info.galaxy_root_dir)
add_var("default_file_path", self.app_info.default_file_path)
add_var("library_import_dir", self.app_info.library_import_dir)
if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar":
# We have a Pulsar job directory, so everything needed (excluding index
# files) should be available in job_directory...
defaults = "$job_directory:default_ro,$tool_directory:default_ro,$job_directory/outputs:rw,$working_directory:rw"
else:
defaults = "$galaxy_root:default_ro,$tool_directory:default_ro"
if self.job_info.job_directory:
defaults += ",$job_directory:default_ro"
if self.job_info.tmp_directory is not None:
defaults += ",$tmp_directory:rw"
if self.app_info.outputs_to_working_directory:
# Should need default_file_path (which is of course an estimate given
# object stores anyway).
defaults += ",$working_directory:rw,$default_file_path:default_ro"
else:
defaults += ",$working_directory:rw,$default_file_path:rw"
if self.app_info.library_import_dir:
defaults += ",$library_import_dir:default_ro"
# Define $defaults that can easily be extended with external library and
# index data without deployer worrying about above details.
variables["defaults"] = string.Template(defaults).safe_substitute(variables)
return template.safe_substitute(variables)
class DockerContainer(Container, HasDockerLikeVolumes):
container_type = DOCKER_CONTAINER_TYPE
@property
def docker_host_props(self):
docker_host_props = dict(
docker_cmd=self.prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND),
sudo=asbool(self.prop("sudo", docker_util.DEFAULT_SUDO)),
sudo_cmd=self.prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND),
host=self.prop("host", docker_util.DEFAULT_HOST),
)
return docker_host_props
def build_pull_command(self):
return docker_util.build_pull_command(self.container_id, **self.docker_host_props)
def containerize_command(self, command):
env_directives = []
for pass_through_var in self.tool_info.env_pass_through:
env_directives.append('"%s=$%s"' % (pass_through_var, pass_through_var))
# Allow destinations to explicitly set environment variables just for
# docker container. Better approach is to set for destination and then
# pass through only what tool needs however. (See todo in ToolInfo.)
for key, value in six.iteritems(self.destination_info):
if key.startswith("docker_env_"):
env = key[len("docker_env_"):]
env_directives.append('"%s=%s"' % (env, value))
working_directory = self.job_info.working_directory
if not working_directory:
raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory)
volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults"))
preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type)
# TODO: Remove redundant volumes...
volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list]
# If a tool definitely has a temp directory available set it to /tmp in container for compat.
# with CWL. This is part of that spec and should make it easier to share containers between CWL
# and Galaxy.
if self.job_info.tmp_directory is not None:
volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory))
volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM)
docker_host_props = self.docker_host_props
cached_image_file = self.__get_cached_image_file()
if not cached_image_file:
# TODO: Add option to cache it once here and create cached_image_file.
cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props)
else:
cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props)
run_command = docker_util.build_docker_run_command(
command,
self.container_id,
volumes=volumes,
volumes_from=volumes_from,
env_directives=env_directives,
working_directory=working_directory,
net=self.prop("net", "none"), # By default, docker instance has networking disabled
auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)),
set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER),
run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
**docker_host_props
)
return "%s\n%s" % (cache_command, run_command)
def __cache_from_file_command(self, cached_image_file, docker_host_props):
images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props)
load_cmd = docker_util.build_docker_load_command(**docker_host_props)
return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute(
cached_image_file=cached_image_file,
images_cmd=images_cmd,
load_cmd=load_cmd
)
def __get_cached_image_file(self):
container_id = self.container_id
cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path"))
cache_path = docker_cache_path(cache_directory, container_id)
return cache_path if os.path.exists(cache_path) else None
def __get_destination_overridable_property(self, name):
prop_name = "docker_%s" % name
if prop_name in self.destination_info:
return self.destination_info[prop_name]
else:
return getattr(self.app_info, name)
def docker_cache_path(cache_directory, container_id):
file_container_id = container_id.replace("/", "_slash_")
cache_file_name = "docker_%s.tar" % file_container_id
return os.path.join(cache_directory, cache_file_name)
class SingularityContainer(Container, HasDockerLikeVolumes):
container_type = SINGULARITY_CONTAINER_TYPE
def get_singularity_target_kwds(self):
return dict(
singularity_cmd=self.prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND),
sudo=asbool(self.prop("sudo", singularity_util.DEFAULT_SUDO)),
sudo_cmd=self.prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND),
)
def build_mulled_singularity_pull_command(self, cache_directory, namespace="biocontainers"):
return singularity_util.pull_mulled_singularity_command(
docker_image_identifier=self.container_id,
cache_directory=cache_directory,
namespace=namespace,
**self.get_singularity_target_kwds()
)
def containerize_command(self, command):
env = []
for pass_through_var in self.tool_info.env_pass_through:
env.append((pass_through_var, "$%s" % pass_through_var))
# Allow destinations to explicitly set environment variables just for
# docker container. Better approach is to set for destination and then
# pass through only what tool needs however. (See todo in ToolInfo.)
for key, value in six.iteritems(self.destination_info):
if key.startswith("singularity_env_"):
real_key = key[len("singularity_env_"):]
env.append((real_key, value))
working_directory = self.job_info.working_directory
if not working_directory:
raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory)
volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults"))
preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type)
volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list]
run_command = singularity_util.build_singularity_run_command(
command,
self.container_id,
volumes=volumes,
env=env,
working_directory=working_directory,
run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
**self.get_singularity_target_kwds()
)
return run_command
CONTAINER_CLASSES = dict(
docker=DockerContainer,
singularity=SingularityContainer,
)
class NullContainer(object):
def __init__(self):
pass
def __bool__(self):
return False
__nonzero__ = __bool__
NULL_CONTAINER = NullContainer()
lib/galaxy/tools/deps/container_resolvers/__init__.py
+3 -2
View File
@@ -16,7 +16,8 @@ class ContainerResolver(Dictifiable):
"""Description of a technique for resolving container images for tool execution."""
# Keys for dictification.
dict_collection_visible_keys = ['resolver_type']
dict_collection_visible_keys = ['resolver_type', 'can_uninstall_dependencies']
can_uninstall_dependencies = False
def __init__(self, app_info=None, **kwds):
"""Default initializer for ``ContainerResolver`` subclasses."""
@@ -33,7 +34,7 @@ class ContainerResolver(Dictifiable):
return default
@abstractmethod
def resolve(self, tool_info):
def resolve(self, enabled_container_types, tool_info, **kwds):
"""Find a container matching all supplied requirements for tool.
The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description
lib/galaxy/tools/deps/container_resolvers/explicit.py
+26 -2
View File
@@ -13,7 +13,7 @@ class ExplicitContainerResolver(ContainerResolver):
resolver_type = "explicit"
def resolve(self, enabled_container_types, tool_info):
def resolve(self, enabled_container_types, tool_info, **kwds):
"""Find a container explicitly mentioned in tool description.
This ignores the tool requirements and assumes the tool author crafted
@@ -26,4 +26,28 @@ class ExplicitContainerResolver(ContainerResolver):
return None
__all__ = ("ExplicitContainerResolver", )
class ExplicitSingularityContainerResolver(ExplicitContainerResolver):
resolver_type = 'explicit_singularity'
container_type = 'singularity'
def resolve(self, enabled_container_types, tool_info, **kwds):
"""Find a container explicitly mentioned in tool description.
This ignores the tool requirements and assumes the tool author crafted
a correct container. We use singularity here to fetch docker containers,
hence the container_description hack here.
"""
for container_description in tool_info.container_descriptions:
if container_description.type == 'docker':
desc_dict = container_description.to_dict()
desc_dict['type'] = self.container_type
desc_dict['identifier'] = "docker://%s" % container_description.identifier
container_description = container_description.from_dict(desc_dict)
if self._container_type_enabled(container_description, enabled_container_types):
return container_description
return None
__all__ = ("ExplicitContainerResolver", "ExplicitSingularityContainerResolver")
lib/galaxy/tools/deps/container_resolvers/mulled.py
+173 -84
View File
@@ -7,7 +7,12 @@ import subprocess
import six
from galaxy.util import unicodify
from galaxy.util import (
string_as_bool,
unicodify,
)
from ..commands import shell
from ..container_classes import CONTAINER_CLASSES
from ..container_resolvers import (
ContainerResolver,
)
@@ -25,7 +30,10 @@ from ..mulled.util import (
v1_image_name,
v2_image_name,
)
from ..requirements import ContainerDescription
from ..requirements import (
ContainerDescription,
DEFAULT_CONTAINER_SHELL,
)
log = logging.getLogger(__name__)
@@ -112,7 +120,8 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None):
build = None
if version and "--" in version:
version, build = split_tag(version)
if prefix and image_name.startswith(prefix):
image_name = image_name[len(prefix):]
image = CachedMulledImageSingleTarget(image_name, version, build, identifier)
return image
@@ -177,7 +186,7 @@ def find_best_matching_cached_image(targets, cached_images, hash_func):
return image
def docker_cached_container_description(targets, namespace, hash_func="v2"):
def docker_cached_container_description(targets, namespace, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL):
if len(targets) == 0:
return None
@@ -189,12 +198,13 @@ def docker_cached_container_description(targets, namespace, hash_func="v2"):
container = ContainerDescription(
image.image_identifier,
type="docker",
shell=shell,
)
return container
def singularity_cached_container_description(targets, cache_directory, hash_func="v2"):
def singularity_cached_container_description(targets, cache_directory, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL):
if len(targets) == 0:
return None
@@ -209,28 +219,85 @@ def singularity_cached_container_description(targets, cache_directory, hash_func
container = ContainerDescription(
os.path.abspath(os.path.join(cache_directory, image.image_identifier)),
type="singularity",
shell=shell,
)
return container
def targets_to_mulled_name(targets, hash_func, namespace):
name = None
if len(targets) == 1:
target = targets[0]
target_version = target.version
tags = mulled_tags_for(namespace, target.package_name)
if not tags:
return None
if target_version:
for tag in tags:
if '--' in tag:
version, build = split_tag(tag)
else:
version = tag
build = None
if version == target_version:
name = "%s:%s" % (target.package_name, version)
if build:
name = "%s--%s" % (name, build)
break
else:
version, build = split_tag(tags[0])
name = "%s:%s--%s" % (target.package_name, version, build)
else:
def tags_if_available(image_name):
if ":" in image_name:
repo_name, tag_prefix = image_name.split(":", 2)
else:
repo_name = image_name
tag_prefix = None
tags = mulled_tags_for(namespace, repo_name, tag_prefix=tag_prefix)
return tags
if hash_func == "v2":
base_image_name = v2_image_name(targets)
tags = tags_if_available(base_image_name)
if tags:
if ":" in base_image_name:
# base_image_name of form <package_hash>:<version_hash>, expand tag
# to include build number in tag.
name = "%s:%s" % (base_image_name.split(":")[0], tags[0])
else:
# base_image_name of form <package_hash>, simply add build number
# as tag to fully qualify image.
name = "%s:%s" % (base_image_name, tags[0])
elif hash_func == "v1":
base_image_name = v1_image_name(targets)
tags = tags_if_available(base_image_name)
if tags:
name = "%s:%s" % (base_image_name, tags[0])
return name
@six.python_2_unicode_compatible
class CachedMulledDockerContainerResolver(ContainerResolver):
resolver_type = "cached_mulled"
container_type = "docker"
shell = '/bin/bash'
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"):
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds):
super(CachedMulledDockerContainerResolver, self).__init__(app_info)
self.namespace = namespace
self.hash_func = hash_func
def resolve(self, enabled_container_types, tool_info):
if tool_info.requires_galaxy_python_environment:
def resolve(self, enabled_container_types, tool_info, **kwds):
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
targets = mulled_targets(tool_info)
return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func)
return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell)
def __str__(self):
return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace
@@ -241,18 +308,19 @@ class CachedMulledSingularityContainerResolver(ContainerResolver):
resolver_type = "cached_mulled_singularity"
container_type = "singularity"
shell = '/bin/bash'
def __init__(self, app_info=None, hash_func="v2", **kwds):
super(CachedMulledSingularityContainerResolver, self).__init__(app_info)
self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled"))
self.hash_func = hash_func
def resolve(self, enabled_container_types, tool_info):
if tool_info.requires_galaxy_python_environment:
def resolve(self, enabled_container_types, tool_info, **kwds):
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
targets = mulled_targets(tool_info)
return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func)
return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell)
def __str__(self):
return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory
@@ -264,91 +332,108 @@ class MulledDockerContainerResolver(ContainerResolver):
resolver_type = "mulled"
container_type = "docker"
shell = '/bin/bash'
protocol = None
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"):
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
super(MulledDockerContainerResolver, self).__init__(app_info)
self.namespace = namespace
self.hash_func = hash_func
self.auto_install = string_as_bool(auto_install)
def resolve(self, enabled_container_types, tool_info):
if tool_info.requires_galaxy_python_environment:
def cached_container_description(self, targets, namespace, hash_func):
return docker_cached_container_description(targets, namespace, hash_func)
def pull(self, container):
command = container.build_pull_command()
shell(command)
def resolve(self, enabled_container_types, tool_info, install=False, **kwds):
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
targets = mulled_targets(tool_info)
if len(targets) == 0:
return None
name = None
if len(targets) == 1:
target = targets[0]
target_version = target.version
tags = mulled_tags_for(self.namespace, target.package_name)
if not tags:
return None
if target_version:
for tag in tags:
version, build = split_tag(tag)
if version == target_version:
name = "%s:%s--%s" % (target.package_name, version, build)
break
else:
version, build = split_tag(tags[0])
name = "%s:%s--%s" % (target.package_name, version, build)
else:
def tags_if_available(image_name):
if ":" in image_name:
repo_name, tag_prefix = image_name.split(":", 2)
else:
repo_name = image_name
tag_prefix = None
tags = mulled_tags_for(self.namespace, repo_name, tag_prefix=tag_prefix)
return tags
if self.hash_func == "v2":
base_image_name = v2_image_name(targets)
tags = tags_if_available(base_image_name)
if tags:
if ":" in base_image_name:
# base_image_name of form <package_hash>:<version_hash>, expand tag
# to include build number in tag.
name = "%s:%s" % (base_image_name.split(":")[0], tags[0])
else:
# base_image_name of form <package_hash>, simply add build number
# as tag to fully qualify image.
name = "%s:%s" % (base_image_name, tags[0])
elif self.hash_func == "v1":
base_image_name = v1_image_name(targets)
tags = tags_if_available(base_image_name)
if tags:
name = "%s:%s" % (base_image_name, tags[0])
name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace)
if name:
return ContainerDescription(
"quay.io/%s/%s" % (self.namespace, name),
container_id = "quay.io/%s/%s" % (self.namespace, name)
if self.protocol:
container_id = "%s%s" % (self.protocol, container_id)
container_description = ContainerDescription(
container_id,
type=self.container_type,
shell=self.shell,
)
destination_for_container_type = kwds.get('destination_for_container_type')
if install and destination_for_container_type and not self.cached_container_description(
targets,
namespace=self.namespace,
hash_func=self.hash_func
):
container = CONTAINER_CLASSES[self.container_type](container_description.identifier,
self.app_info,
tool_info,
destination_for_container_type(self.container_type),
{},
container_description)
self.pull(container)
if not self.auto_install:
container_description = self.cached_container_description(
targets,
namespace=self.namespace,
hash_func=self.hash_func
)
return container_description
def __str__(self):
return "MulledDockerContainerResolver[namespace=%s]" % self.namespace
@six.python_2_unicode_compatible
class MulledSingularityContainerResolver(MulledDockerContainerResolver):
resolver_type = "mulled_singularity"
container_type = "singularity"
protocol = 'docker://'
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
super(MulledSingularityContainerResolver, self).__init__(app_info)
self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled"))
self.namespace = namespace
self.hash_func = hash_func
self.auto_install = string_as_bool(auto_install)
def cached_container_description(self, targets, namespace, hash_func):
return singularity_cached_container_description(targets,
cache_directory=self.cache_directory,
hash_func=hash_func)
def pull(self, container):
cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace)
shell(cmds=cmds)
def __str__(self):
return "MulledSingularityContainerResolver[namespace=%s]" % self.namespace
@six.python_2_unicode_compatible
class BuildMulledDockerContainerResolver(ContainerResolver):
"""Build for Docker mulled images matching tool dependencies."""
resolver_type = "build_mulled"
container_type = "docker"
shell = '/bin/bash'
def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds):
def __init__(self, app_info=None, namespace="local", hash_func="v2", auto_install=True, **kwds):
super(BuildMulledDockerContainerResolver, self).__init__(app_info)
self._involucro_context_kwds = {
'involucro_bin': self._get_config_option("involucro_path", None)
}
self.namespace = namespace
self.hash_func = hash_func
self.auto_install = string_as_bool(auto_install)
self._mulled_kwds = {
'namespace': namespace,
'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS),
@@ -357,20 +442,20 @@ class BuildMulledDockerContainerResolver(ContainerResolver):
}
self.auto_init = self._get_config_option("involucro_auto_init", True)
def resolve(self, enabled_container_types, tool_info):
if tool_info.requires_galaxy_python_environment:
def resolve(self, enabled_container_types, tool_info, install=False, **kwds):
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
targets = mulled_targets(tool_info)
if len(targets) == 0:
return None
mull_targets(
targets,
involucro_context=self._get_involucro_context(),
**self._mulled_kwds
)
return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func)
if self.auto_install or install:
mull_targets(
targets,
involucro_context=self._get_involucro_context(),
**self._mulled_kwds
)
return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell)
def _get_involucro_context(self):
involucro_context = InvolucroContext(**self._involucro_context_kwds)
@@ -387,14 +472,16 @@ class BuildMulledSingularityContainerResolver(ContainerResolver):
resolver_type = "build_mulled_singularity"
container_type = "singularity"
shell = '/bin/bash'
def __init__(self, app_info=None, hash_func="v2", **kwds):
def __init__(self, app_info=None, hash_func="v2", auto_install=True, **kwds):
super(BuildMulledSingularityContainerResolver, self).__init__(app_info)
self._involucro_context_kwds = {
'involucro_bin': self._get_config_option("involucro_path", None)
}
self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled"))
self.hash_func = hash_func
self.auto_install = string_as_bool(auto_install)
self._mulled_kwds = {
'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS),
'hash_func': self.hash_func,
@@ -404,20 +491,21 @@ class BuildMulledSingularityContainerResolver(ContainerResolver):
}
self.auto_init = self._get_config_option("involucro_auto_init", True)
def resolve(self, enabled_container_types, tool_info):
if tool_info.requires_galaxy_python_environment:
def resolve(self, enabled_container_types, tool_info, install=False, **kwds):
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
targets = mulled_targets(tool_info)
if len(targets) == 0:
return None
mull_targets(
targets,
involucro_context=self._get_involucro_context(),
**self._mulled_kwds
)
return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func)
if self.auto_install or install:
mull_targets(
targets,
involucro_context=self._get_involucro_context(),
**self._mulled_kwds
)
return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell)
def _get_involucro_context(self):
involucro_context = InvolucroContext(**self._involucro_context_kwds)
@@ -425,7 +513,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver):
return involucro_context
def __str__(self):
return "BuildDockerContainerResolver[cache_directory=%s]" % self.cache_directory
return "BuildSingularityContainerResolver[cache_directory=%s]" % self.cache_directory
def mulled_targets(tool_info):
@@ -436,6 +524,7 @@ __all__ = (
"CachedMulledDockerContainerResolver",
"CachedMulledSingularityContainerResolver",
"MulledDockerContainerResolver",
"MulledSingularityContainerResolver",
"BuildMulledDockerContainerResolver",
"BuildMulledSingularityContainerResolver",
)
lib/galaxy/tools/deps/containers.py
+16 -389
View File
@@ -1,73 +1,38 @@
import logging
import os
import string
from abc import (
ABCMeta,
abstractmethod
)
import six
from galaxy.containers.docker_model import DockerVolume
from galaxy.util import (
asbool,
in_directory,
plugin_config
)
from . import (
docker_util,
singularity_util
from .container_classes import (
CONTAINER_CLASSES,
DOCKER_CONTAINER_TYPE,
NULL_CONTAINER,
SINGULARITY_CONTAINER_TYPE,
)
from .container_resolvers.explicit import (
ExplicitContainerResolver,
ExplicitSingularityContainerResolver,
)
from .container_resolvers.explicit import ExplicitContainerResolver
from .container_resolvers.mulled import (
BuildMulledDockerContainerResolver,
BuildMulledSingularityContainerResolver,
CachedMulledDockerContainerResolver,
CachedMulledSingularityContainerResolver,
MulledDockerContainerResolver,
MulledSingularityContainerResolver,
)
from .requirements import (
ContainerDescription,
DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES,
DEFAULT_CONTAINER_SHELL
)
log = logging.getLogger(__name__)
DOCKER_CONTAINER_TYPE = "docker"
SINGULARITY_CONTAINER_TYPE = "singularity"
DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE
ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE]
LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r'''
python << EOF
from __future__ import print_function
import json
import re
import subprocess
import tarfile
t = tarfile.TarFile("${cached_image_file}")
meta_str = t.extractfile('repositories').read()
meta = json.loads(meta_str)
tag, tag_value = meta.items()[0]
rev, rev_value = tag_value.items()[0]
cmd = "${images_cmd}"
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
stdo, stde = proc.communicate()
found = False
for line in stdo.split("\n"):
tmp = re.split(r'\s+', line)
if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value:
found = True
if not found:
print("Loading image")
cmd = "cat ${cached_image_file} | ${load_cmd}"
subprocess.check_call(cmd, shell=True)
EOF
'''
class ContainerFinder(object):
@@ -75,7 +40,7 @@ class ContainerFinder(object):
self.app_info = app_info
self.container_registry = ContainerRegistry(app_info)
def __enabled_container_types(self, destination_info):
def _enabled_container_types(self, destination_info):
return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)]
def find_best_container_description(self, enabled_container_types, tool_info):
@@ -86,7 +51,7 @@ class ContainerFinder(object):
return container_description
def find_container(self, tool_info, destination_info, job_info):
enabled_container_types = self.__enabled_container_types(destination_info)
enabled_container_types = self._enabled_container_types(destination_info)
# Short-cut everything else and just skip checks if no container type is enabled.
if not enabled_container_types:
@@ -236,13 +201,15 @@ class ContainerRegistry(object):
def __default_containers_resolvers(self):
default_resolvers = [
ExplicitContainerResolver(self.app_info),
ExplicitSingularityContainerResolver(self.app_info),
]
if self.enable_beta_mulled_containers:
default_resolvers.extend([
CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
MulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"),
BuildMulledDockerContainerResolver(self.app_info),
CachedMulledSingularityContainerResolver(self.app_info),
BuildMulledSingularityContainerResolver(self.app_info),
])
return default_resolvers
@@ -264,343 +231,3 @@ class ContainerRegistry(object):
return container_description
return None
class AppInfo(object):
def __init__(
self,
galaxy_root_dir=None,
default_file_path=None,
outputs_to_working_directory=False,
container_image_cache_path=None,
library_import_dir=None,
enable_beta_mulled_containers=False,
containers_resolvers_config_file=None,
involucro_path=None,
involucro_auto_init=True,
):
self.galaxy_root_dir = galaxy_root_dir
self.default_file_path = default_file_path
# TODO: Vary default value for docker_volumes based on this...
self.outputs_to_working_directory = outputs_to_working_directory
self.container_image_cache_path = container_image_cache_path
self.library_import_dir = library_import_dir
self.enable_beta_mulled_containers = enable_beta_mulled_containers
self.containers_resolvers_config_file = containers_resolvers_config_file
self.involucro_path = involucro_path
self.involucro_auto_init = involucro_auto_init
class ToolInfo(object):
# TODO: Introduce tool XML syntax to annotate the optional environment
# variables they can consume (e.g. JVM options, license keys, etc..)
# and add these to env_path_through
def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]):
self.container_descriptions = container_descriptions
self.requirements = requirements
self.requires_galaxy_python_environment = requires_galaxy_python_environment
self.env_pass_through = env_pass_through
class JobInfo(object):
def __init__(
self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type
):
self.working_directory = working_directory
self.job_directory = job_directory
# Tool files may be remote staged - so this is unintuitively a property
# of the job not of the tool.
self.tool_directory = tool_directory
self.tmp_directory = tmp_directory
self.job_directory_type = job_directory_type # "galaxy" or "pulsar"
@six.add_metaclass(ABCMeta)
class Container(object):
def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description):
self.container_id = container_id
self.app_info = app_info
self.tool_info = tool_info
self.destination_info = destination_info
self.job_info = job_info
self.container_description = container_description
@property
def resolve_dependencies(self):
return DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES if not self.container_description else self.container_description.resolve_dependencies
@property
def shell(self):
return DEFAULT_CONTAINER_SHELL if not self.container_description else self.container_description.shell
@abstractmethod
def containerize_command(self, command):
"""
Use destination supplied container configuration parameters,
container_id, and command to build a new command that runs
input command in container.
"""
def preprocess_volumes(volumes_raw_str, container_type):
"""Process Galaxy volume specification string to either Docker or Singularity specification.
Galaxy allows the mount try "default_ro" which translates to ro for Docker and
ro for Singularity iff no subdirectories are rw (Singularity does not allow ro
parent directories with rw subdirectories).
>>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE)
['/a/b:rw']
>>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE)
['/a/b:ro', '/a/b/c:rw']
>>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE)
['/a/b:ro', '/a/b/c:rw']
>>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE)
['/a/b:rw', '/a/b/c:rw']
"""
volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")]
volumes = []
rw_paths = []
for volume_raw_str in volumes_raw_strs:
volume_parts = volume_raw_str.split(":")
if len(volume_parts) > 2:
raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str)
if len(volume_parts) == 1:
volume_parts.append("rw")
volumes.append(volume_parts)
if volume_parts[1] == "rw":
rw_paths.append(volume_parts[0])
for volume in volumes:
path = volume[0]
how = volume[1]
if how == "default_ro":
how = "ro"
if container_type == SINGULARITY_CONTAINER_TYPE:
for rw_path in rw_paths:
if in_directory(rw_path, path):
how = "rw"
volume[1] = how
return [":".join(v) for v in volumes]
class HasDockerLikeVolumes(object):
"""Mixin to share functionality related to Docker volume handling.
Singularity seems to have a fairly compatible syntax for volume handling.
"""
def _expand_volume_str(self, value):
if not value:
return value
template = string.Template(value)
variables = dict()
def add_var(name, value):
if value:
if not value.startswith("$"):
value = os.path.abspath(value)
variables[name] = value
add_var("working_directory", self.job_info.working_directory)
add_var("tmp_directory", self.job_info.tmp_directory)
add_var("job_directory", self.job_info.job_directory)
add_var("tool_directory", self.job_info.tool_directory)
add_var("galaxy_root", self.app_info.galaxy_root_dir)
add_var("default_file_path", self.app_info.default_file_path)
add_var("library_import_dir", self.app_info.library_import_dir)
if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar":
# We have a Pulsar job directory, so everything needed (excluding index
# files) should be available in job_directory...
defaults = "$job_directory:default_ro,$tool_directory:default_ro,$job_directory/outputs:rw,$working_directory:rw"
else:
defaults = "$galaxy_root:default_ro,$tool_directory:default_ro"
if self.job_info.job_directory:
defaults += ",$job_directory:default_ro"
if self.job_info.tmp_directory is not None:
defaults += ",$tmp_directory:rw"
if self.app_info.outputs_to_working_directory:
# Should need default_file_path (which is of course an estimate given
# object stores anyway).
defaults += ",$working_directory:rw,$default_file_path:default_ro"
else:
defaults += ",$working_directory:rw,$default_file_path:rw"
if self.app_info.library_import_dir:
defaults += ",$library_import_dir:default_ro"
# Define $defaults that can easily be extended with external library and
# index data without deployer worrying about above details.
variables["defaults"] = string.Template(defaults).safe_substitute(variables)
return template.safe_substitute(variables)
class DockerContainer(Container, HasDockerLikeVolumes):
container_type = DOCKER_CONTAINER_TYPE
def containerize_command(self, command):
def prop(name, default):
destination_name = "docker_%s" % name
return self.destination_info.get(destination_name, default)
env_directives = []
for pass_through_var in self.tool_info.env_pass_through:
env_directives.append('"%s=$%s"' % (pass_through_var, pass_through_var))
# Allow destinations to explicitly set environment variables just for
# docker container. Better approach is to set for destination and then
# pass through only what tool needs however. (See todo in ToolInfo.)
for key, value in six.iteritems(self.destination_info):
if key.startswith("docker_env_"):
env = key[len("docker_env_"):]
env_directives.append('"%s=%s"' % (env, value))
working_directory = self.job_info.working_directory
if not working_directory:
raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory)
volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults"))
preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type)
# TODO: Remove redundant volumes...
volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list]
# If a tool definitely has a temp directory available set it to /tmp in container for compat.
# with CWL. This is part of that spec and should make it easier to share containers between CWL
# and Galaxy.
if self.job_info.tmp_directory is not None:
volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory))
volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM)
docker_host_props = dict(
docker_cmd=prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND),
sudo=asbool(prop("sudo", docker_util.DEFAULT_SUDO)),
sudo_cmd=prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND),
host=prop("host", docker_util.DEFAULT_HOST),
)
cached_image_file = self.__get_cached_image_file()
if not cached_image_file:
# TODO: Add option to cache it once here and create cached_image_file.
cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props)
else:
cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props)
run_command = docker_util.build_docker_run_command(
command,
self.container_id,
volumes=volumes,
volumes_from=volumes_from,
env_directives=env_directives,
working_directory=working_directory,
net=prop("net", "none"), # By default, docker instance has networking disabled
auto_rm=asbool(prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)),
set_user=prop("set_user", docker_util.DEFAULT_SET_USER),
run_extra_arguments=prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
**docker_host_props
)
return "%s\n%s" % (cache_command, run_command)
def __cache_from_file_command(self, cached_image_file, docker_host_props):
images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props)
load_cmd = docker_util.build_docker_load_command(**docker_host_props)
return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute(
cached_image_file=cached_image_file,
images_cmd=images_cmd,
load_cmd=load_cmd
)
def __get_cached_image_file(self):
container_id = self.container_id
cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path"))
cache_path = docker_cache_path(cache_directory, container_id)
return cache_path if os.path.exists(cache_path) else None
def __get_destination_overridable_property(self, name):
prop_name = "docker_%s" % name
if prop_name in self.destination_info:
return self.destination_info[prop_name]
else:
return getattr(self.app_info, name)
def docker_cache_path(cache_directory, container_id):
file_container_id = container_id.replace("/", "_slash_")
cache_file_name = "docker_%s.tar" % file_container_id
return os.path.join(cache_directory, cache_file_name)
class SingularityContainer(Container, HasDockerLikeVolumes):
container_type = SINGULARITY_CONTAINER_TYPE
def containerize_command(self, command):
def prop(name, default):
destination_name = "singularity_%s" % name
return self.destination_info.get(destination_name, default)
env = []
for pass_through_var in self.tool_info.env_pass_through:
env.append((pass_through_var, "$%s" % pass_through_var))
# Allow destinations to explicitly set environment variables just for
# docker container. Better approach is to set for destination and then
# pass through only what tool needs however. (See todo in ToolInfo.)
for key, value in six.iteritems(self.destination_info):
if key.startswith("singularity_env_"):
real_key = key[len("singularity_env_"):]
env.append((real_key, value))
working_directory = self.job_info.working_directory
if not working_directory:
raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory)
volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults"))
preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type)
volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list]
singularity_target_kwds = dict(
singularity_cmd=prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND),
sudo=asbool(prop("sudo", singularity_util.DEFAULT_SUDO)),
sudo_cmd=prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND),
)
run_command = singularity_util.build_singularity_run_command(
command,
self.container_id,
volumes=volumes,
env=env,
working_directory=working_directory,
run_extra_arguments=prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
**singularity_target_kwds
)
return run_command
CONTAINER_CLASSES = dict(
docker=DockerContainer,
singularity=SingularityContainer,
)
class NullContainer(object):
def __init__(self):
pass
def __bool__(self):
return False
__nonzero__ = __bool__
NULL_CONTAINER = NullContainer()
lib/galaxy/tools/deps/dependencies.py
+55
View File
@@ -1,5 +1,60 @@
from galaxy.tools.deps.requirements import ToolRequirements
from galaxy.util import bunch
from .mulled.mulled_build import DEFAULT_CHANNELS
class AppInfo(object):
def __init__(
self,
galaxy_root_dir=None,
default_file_path=None,
outputs_to_working_directory=False,
container_image_cache_path=None,
library_import_dir=None,
enable_beta_mulled_containers=False,
containers_resolvers_config_file=None,
involucro_path=None,
involucro_auto_init=True,
mulled_channels=DEFAULT_CHANNELS,
):
self.galaxy_root_dir = galaxy_root_dir
self.default_file_path = default_file_path
# TODO: Vary default value for docker_volumes based on this...
self.outputs_to_working_directory = outputs_to_working_directory
self.container_image_cache_path = container_image_cache_path
self.library_import_dir = library_import_dir
self.enable_beta_mulled_containers = enable_beta_mulled_containers
self.containers_resolvers_config_file = containers_resolvers_config_file
self.involucro_path = involucro_path
self.involucro_auto_init = involucro_auto_init
self.mulled_channels = mulled_channels
class ToolInfo(object):
# TODO: Introduce tool XML syntax to annotate the optional environment
# variables they can consume (e.g. JVM options, license keys, etc..)
# and add these to env_path_through
def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]):
self.container_descriptions = container_descriptions
self.requirements = requirements
self.requires_galaxy_python_environment = requires_galaxy_python_environment
self.env_pass_through = env_pass_through
class JobInfo(object):
def __init__(
self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type
):
self.working_directory = working_directory
self.job_directory = job_directory
# Tool files may be remote staged - so this is unintuitively a property
# of the job not of the tool.
self.tool_directory = tool_directory
self.tmp_directory = tmp_directory
self.job_directory_type = job_directory_type # "galaxy" or "pulsar"
class DependenciesDescription(object):
lib/galaxy/tools/deps/dockerfiles.py
+1 -1
View File
@@ -3,7 +3,7 @@ import os
from ..deps import commands
from ..deps import docker_util
from ..deps.containers import docker_cache_path
from ..deps.container_classes import docker_cache_path
from ..deps.requirements import parse_requirements_from_xml
from ...tools import loader_directory
lib/galaxy/tools/deps/mulled/invfile.lua
+2 -5
View File
@@ -48,7 +48,7 @@ end
local singularity_image = VAR.SINGULARITY_IMAGE
if singularity_image == '' then
singularity_image = 'quay.io/biocontainers/singularity:2.3--0'
singularity_image = 'quay.io/biocontainers/singularity:2.4.6--0'
end
local singularity_image_dir = VAR.SINGULARITY_IMAGE_DIR
@@ -103,10 +103,7 @@ if VAR.SINGULARITY ~= '' then
.using(singularity_image)
.withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true})
.withConfig({entrypoint = {'/bin/sh', '-c'}})
-- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size
.run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi")
.run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME)
.run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity')
.run('mkdir -p /usr/local/var/singularity/mnt/container && singularity build /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity.def')
.run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME)
end
lib/galaxy/tools/deps/mulled/mulled_build.py
+8 -4
View File
@@ -68,7 +68,7 @@ From: bgruening/busybox-bash:0.1
def involucro_link():
if IS_OS_X:
url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION
url = "https://github.com/mvdbeek/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION
else:
url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION
return url
@@ -236,7 +236,7 @@ def mull_targets(
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def:
with open(os.path.join(singularity_image_dir, 'Singularity.def'), 'w+') as sin_def:
fill_template = SINGULARITY_TEMPLATE % {'container_test': test}
sin_def.write(fill_template)
with PrintProgress():
@@ -275,12 +275,16 @@ class InvolucroContext(installable.InstallableContext):
def exec_command(self, involucro_args):
cmd = self.build_command(involucro_args)
# Create ./build dir manually, otherwise Docker will do it as root
os.mkdir('./build')
created_build_dir = False
if not os.path.exists('build'):
created_build_dir = True
os.mkdir('./build')
try:
res = self.shell_exec(" ".join(cmd))
finally:
# delete build directory in any case
shutil.rmtree('./build')
if created_build_dir:
shutil.rmtree('./build')
return res
def is_installed(self):
lib/galaxy/tools/deps/mulled/util.py
+15 -1
View File
@@ -13,6 +13,8 @@ try:
except ImportError:
requests = None
MULLED_TAG_CACHE = collections.defaultdict(dict)
def create_repository(namespace, repo_name, oauth_token):
assert oauth_token
@@ -56,7 +58,19 @@ def mulled_tags_for(namespace, image, tag_prefix=None):
The result will be sorted so newest tags are first.
"""
tags = quay_versions(namespace, image)
tags_cached = False
if namespace in MULLED_TAG_CACHE:
if image in MULLED_TAG_CACHE[namespace]:
tags, last_checked = MULLED_TAG_CACHE[namespace][image]
if not tags and time.time() - last_checked < 300:
# it's possible we haven't seen the tags before, we check every 5 minutes
tags_cached = False
else:
tags_cached = True
if not tags_cached:
tags = quay_versions(namespace, image)
last_checked = time.time()
MULLED_TAG_CACHE[namespace][image] = (tags, last_checked)
if tag_prefix is not None:
tags = [t for t in tags if t.startswith(tag_prefix)]
tags = version_sorted(tags)
lib/galaxy/tools/deps/requirements.py
+1 -1
View File
@@ -240,7 +240,7 @@ def parse_requirements_from_xml(xml_root):
if requirements_elem is not None:
container_elems = requirements_elem.findall('container')
containers = map(container_from_element, container_elems)
containers = [container_from_element(c) for c in container_elems]
return requirements, containers
lib/galaxy/tools/deps/resolvers/__init__.py
+27
View File
@@ -257,6 +257,33 @@ class Dependency(Dictifiable):
return "Using dependency %s version %s of type %s" % (self.name, self.version, self.dependency_type)
class ContainerDependency(Dependency):
dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path']
def __init__(self, container_description, name, version):
self.dependency_type = container_description.type
self._name = name
self._version = version
self.environment_path = container_description.identifier
@property
def name(self):
return self._name
@property
def version(self):
return self._version
@property
def exact(self):
return True
@property
def shell_commands(self):
return None
class NullDependency(Dependency):
dependency_type = None
exact = True
lib/galaxy/tools/deps/singularity_util.py
+29 -2
View File
@@ -1,5 +1,6 @@
from six.moves import shlex_quote
import os
from six.moves import shlex_quote
DEFAULT_WORKING_DIRECTORY = None
DEFAULT_SINGULARITY_COMMAND = "singularity"
@@ -8,6 +9,27 @@ DEFAULT_SUDO_COMMAND = "sudo"
DEFAULT_RUN_EXTRA_ARGUMENTS = None
def pull_mulled_singularity_command(docker_image_identifier,
cache_directory,
namespace=None,
singularity_cmd=DEFAULT_SINGULARITY_COMMAND,
sudo=DEFAULT_SUDO,
sudo_cmd=DEFAULT_SUDO_COMMAND):
command_parts = []
command_parts += _singularity_prefix(
singularity_cmd=singularity_cmd,
sudo=sudo,
sudo_cmd=sudo_cmd,
)
save_path = docker_image_identifier
if namespace:
prefix = "docker://quay.io/%s/" % namespace
if docker_image_identifier.startswith(prefix):
save_path = docker_image_identifier[len(prefix):]
command_parts.extend(["build", os.path.join(cache_directory, save_path), docker_image_identifier])
return command_parts
def build_singularity_run_command(
container_command,
image,
@@ -21,7 +43,10 @@ def build_singularity_run_command(
):
command_parts = []
# http://singularity.lbl.gov/docs-environment-metadata
home = None
for (key, value) in env:
if key == 'HOME':
home = value
command_parts.extend(["SINGULARITYENV_%s=%s" % (key, value)])
command_parts += _singularity_prefix(
singularity_cmd=singularity_cmd,
@@ -34,6 +59,8 @@ def build_singularity_run_command(
command_parts.extend(["-B", shlex_quote(str(volume))])
if working_directory:
command_parts.extend(["--pwd", shlex_quote(working_directory)])
if home is not None:
command_parts.extend(["--home", "%s:%s" % (home, home)])
if run_extra_arguments:
command_parts.append(run_extra_arguments)
full_image = image
@@ -56,4 +83,4 @@ def _singularity_prefix(
return command_parts
__all__ = ("build_singularity_run_command",)
__all__ = ("build_singularity_run_command", "pull_mulled_singularity_command")
lib/galaxy/tools/deps/views.py
+10 -4
View File
@@ -49,10 +49,12 @@ class DependencyResolversView(object):
kwds = {'install': False,
'return_null': True,
'installed_tool_dependencies': installed_tool_dependencies}
dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()}
dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements,
**kwds)
for tool, requirements in tool_requirements_d.items()}
return dependencies_per_tool
def uninstall_dependencies(self, index=None, **payload):
def uninstall_dependencies(self, index=None, resolver_type=None, **payload):
"""Attempt to uninstall requirements. Returns 0 if successfull, else None."""
requirements = payload.get('requirements')
if not requirements:
@@ -61,6 +63,10 @@ class DependencyResolversView(object):
resolver = self._dependency_resolvers[index]
if resolver.can_uninstall_dependencies:
return resolver.uninstall(requirements)
elif resolver_type:
for resolver in self._dependency_resolvers:
if resolver.resolver_type == resolver_type and resolver.can_uninstall_dependencies:
return resolver.uninstall(requirements)
else:
for index in self.uninstallable_resolvers:
return_code = self._dependency_resolvers[index].uninstall(requirements)
@@ -97,8 +103,8 @@ class DependencyResolversView(object):
envs_to_remove = envs_to_remove.difference(can_remove)
return list(removed_environments)
def install_dependencies(self, requirements):
return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True})
def install_dependencies(self, requirements, **kwds):
return self._dependency_manager._requirements_to_dependencies_dict(requirements, **kwds)
def install_dependency(self, index=None, **payload):
"""
lib/galaxy/webapps/galaxy/api/tools.py
+13 -5
View File
@@ -234,7 +234,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
Return the resolver status for a specific tool id.
[{"status": "installed", "name": "hisat2", "versionless": false, "resolver_type": "conda", "version": "2.0.3", "type": "package"}]
"""
tool = self._get_tool(id)
tool = self._get_tool(id, user=trans.user)
return tool.tool_requirements_status
@expose_api
@@ -249,11 +249,15 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
Attempts to install requirements via the dependency resolver
parameters:
index: index of dependency resolver to use when installing dependency.
Defaults to using the highest ranking resolver
resolver_type: Use the dependency resolver of this resolver_type to install dependency.
build_dependency_cache: If true, attempts to cache dependencies for this tool
force_rebuild: If true and cache dir exists, attempts to delete cache dir
"""
tool = self._get_tool(id)
tool._view.install_dependencies(tool.requirements)
tool = self._get_tool(id, user=trans.user)
kwds['install'] = True
tool._view.install_dependencies(tool.requirements, **kwds)
if kwds.get('build_dependency_cache'):
tool.build_dependency_cache(**kwds)
# TODO: rework resolver install system to log and report what has been done.
@@ -267,9 +271,13 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
DELETE /api/tools/{tool_id}/dependencies
Attempts to uninstall requirements via the dependency resolver
parameters:
index: index of dependency resolver to use when installing dependency.
Defaults to using the highest ranking resolver
resolver_type: Use the dependency resolver of this resolver_type to install dependency
"""
tool = self._get_tool(id)
tool._view.uninstall_dependencies(index=None, requirements=tool.requirements)
tool = self._get_tool(id, user=trans.user)
tool._view.uninstall_dependencies(requirements=tool.requirements, **kwds)
# TODO: rework resolver install system to log and report what has been done.
return tool.tool_requirements_status
lib/galaxy/webapps/galaxy/config_schema.yml
+9 -2
View File
@@ -414,7 +414,7 @@ mapping:
default: database/dependencies/involucro
required: false
desc: |
involucro is a tool used to build Docker containers for tools from Conda
involucro is a tool used to build Docker or Singularity containers for tools from Conda
dependencies referenced in tools as `requirement`s. The following path is
the location of involucro on the Galaxy host. This is ignored if the relevant
container resolver isn't enabled, and will install on demand unless
@@ -425,9 +425,16 @@ mapping:
default: true
required: false
desc: |
Install involucro as needed to build Docker containers for tools. Ignored if
Install involucro as needed to build Docker or Singularity containers for tools. Ignored if
relevant container resolver is not used.
mulled_channels:
type: str
default: 'conda-forge,bioconda'
required: false
desc: |
Conda channels to use when building Docker or Singularity containers using involucro.
enable_tool_shed_check:
type: bool
default: false
run_tests.sh
+3 -11
View File
@@ -270,7 +270,7 @@ exists() {
type "$1" >/dev/null 2>/dev/null
}
DOCKER_DEFAULT_IMAGE='galaxy/testing-base:18.09.0'
DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.01.3'
test_script="./scripts/functional_tests.py"
report_file="run_functional_tests.html"
@@ -295,20 +295,12 @@ then
else
db_type="sqlite"
fi
if [ "$1" = "--external_tmp" ]; then
# If /tmp is a tmpfs there may be better performance by reusing
# the parent's temp file system. Also, it seems to decrease the
# frequency or errors such as the following:
# /bin/sh: 1: /tmp/tmpiWU3kJ/tmp_8zLxx/job_working_directory_mwwDmg/000/274/galaxy_274.sh: Text file busy
tmp=$(mktemp -d)
chmod 1777 $tmp
DOCKER_RUN_EXTRA_ARGS="-v ${tmp}:/tmp ${DOCKER_RUN_EXTRA_ARGS}"
shift
fi
MY_UID=$(id -u)
# Skip client build process in the Docker container for all tests, the Jenkins task builds the client
# locally before testing - you will need to do this also if using this script for Selenium testing.
DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}"
echo "Docker version:"
docker --version
echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..."
docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS \
-e "BUILD_NUMBER=$BUILD_NUMBER" \
test/base/integration_util.py
+4
View File
@@ -33,6 +33,10 @@ def skip_unless_docker():
return skip_unless_executable("docker")
def skip_unless_singularity():
return skip_unless_executable("singularity")
class IntegrationTestCase(TestCase, UsesApiTestCaseMixin):
"""Unit test case with utilities for spinning up Galaxy."""
test/docker/base/Dockerfile
+21 -24
View File
@@ -21,27 +21,18 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans
curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
curl -sL https://deb.nodesource.com/setup_8.x | bash - && \
apt-key adv --keyserver pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5 && \
echo "deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list && \
echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \
{ \
echo mysql-community-server mysql-community-server/data-dir select ''; \
echo mysql-community-server mysql-community-server/root-pass password ''; \
echo mysql-community-server mysql-community-server/re-root-pass password ''; \
echo mysql-community-server mysql-community-server/remove-test-db select false; \
} | debconf-set-selections && \
apt-get update -y && \
apt-get install -y libpq-dev postgresql postgresql-client \
ansible wget mysql-server="${MYSQL_MAJOR}"* libmysqlclient-dev="${MYSQL_MAJOR}"* \
apt-get install -y libpq-dev postgresql postgresql-client ansible wget \
slurm-llnl libmunge-dev slurm-drmaa-dev ant cmake curl \
g++ gcc gfortran git-core libffi-dev liblapack-dev \
libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \
mercurial nginx-extras patch postgresql postgresql \
postgresql-client python-boto python-dev python3-dev \
patch postgresql postgresql \
postgresql-client python-dev python3-dev \
python-prettytable python-psycopg2 python-virtualenv python-pip \
rsync slurm-drmaa-dev swig sysstat unzip \
autoconf automake build-essential libatlas-base-dev libblas-dev openssl \
bzip2 \
bzip2 uuid-dev libssl-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \
ca-certificates \
openjdk-8-jre-headless \
tzdata \
@@ -53,14 +44,14 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans
libnss3 libgconf-2-4 && \
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN npm install -g grunt grunt-cli
# Install golang
RUN export VERSION=1.11 OS=linux ARCH=amd64 && curl -sL https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz \
| tar -C /usr/local -xzf -
RUN mkdir -p /tmp/ansible && \
mkdir -p /opt/galaxy/db && \
chown -R postgres:postgres /opt/galaxy/db && \
sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf
chown -R postgres:postgres /opt/galaxy/db
ADD start_mysql.sh /opt/galaxy/start_mysql.sh
ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml
ADD provision.yml /tmp/ansible/provision.yml
@@ -93,7 +84,6 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \
export GALAXY_VIRTUAL_ENV=$VENV && \
. $GALAXY_VIRTUAL_ENV/bin/activate && \
pip install psycopg2; done && \
pip install mysql && \
cd $GALAXY_ROOT && \
echo "Prepopulating postgres database" && \
su -c '/usr/lib/postgresql/${POSTGRES_MAJOR}/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \
@@ -101,15 +91,10 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \
GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" bash create_db.sh && \
echo "Prepopulating sqlite database" && \
GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" bash create_db.sh && \
sh /opt/galaxy/start_mysql.sh && \
echo "Prepopulating mysql database" && \
GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" bash create_db.sh && \
echo "Prepopulating toolshed postgres database" && \
TOOL_SHED_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" bash create_db.sh tool_shed && \
echo "Prepopulating toolshed sqlite database" && \
TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed && \
echo "Prepopulating toolshed mysql database" && \
TOOL_SHED_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" bash create_db.sh tool_shed
TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed
# bcftools for Galaxy.
RUN mkdir -p /tmp/install && \
@@ -141,6 +126,18 @@ RUN sudo mkdir -p /opt/selenium \
&& wget --no-verbose https://selenium-release.storage.googleapis.com/3.6/selenium-server-standalone-3.6.0.jar \
-O /opt/selenium/selenium-server-standalone.jar
USER seluser
RUN export GOPATH=${HOME}/go && export PATH=/usr/local/go/bin:${PATH}:${GOPATH}/bin && \
mkdir -p $GOPATH/src/github.com/sylabs && \
cd $GOPATH/src/github.com/sylabs && \
git clone https://github.com/sylabs/singularity.git && \
cd singularity && \
git checkout v3.0.1 && \
./mconfig && \
cd builddir && make && sudo make install && \
rm -Rf ${HOME}/go
USER root
#==============================
test/docker/base/run_test_wrapper.sh
+4 -8
View File
@@ -16,9 +16,10 @@ echo "Creating galaxy group with gid $GALAXY_TEST_UID - it may already exist and
groupadd -r galaxy -g "$GALAXY_TEST_UID" | true
echo "Creating galaxy user with uid $GALAXY_TEST_UID - it may already exist and this is fine."
useradd -u $GALAXY_TEST_UID -r -g galaxy -d /home/galaxy -c "Galaxy User" galaxy -s /bin/bash | true
mkdir -p /home/galaxy/.singularity
echo "Setting galaxy user password - the operation may fail."
echo "galaxy:galaxy" | chpasswd | true
chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}"
chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" /home/galaxy/
: ${GALAXY_TEST_DATABASE_TYPE:-"postgres"}
if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ];
@@ -30,12 +31,6 @@ then
GALAXY_TEST_DBURI="postgres://root@localhost:5930/galaxy?client_encoding=utf8"
TOOL_SHED_TEST_DBURI="postgres://root@localhost:5930/toolshed?client_encoding=utf8"
export GALAXY_CONFIG_OVERRIDE_DATABASE_ENCODING="UTF-8"
elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ];
then
sh /opt/galaxy/start_mysql.sh
GALAXY_TEST_INSTALL_DB_MERGED="true"
GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock"
TOOL_SHED_TEST_DBURI="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock"
elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ];
then
GALAXY_TEST_INSTALL_DB_MERGED="true"
@@ -54,7 +49,8 @@ cd /galaxy
: ${GALAXY_VIRTUAL_ENV:=.venv}
HOME=/galaxy
HOME=/home/galaxy
echo "Running common startup for updated dependencies (if any)"
sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; }
test/docker/base/start_mysql.sh
-47
View File
@@ -1,47 +0,0 @@
#!/bin/bash
set -e
MYSQL_USER=galaxy
MYSQL_PASSWORD=galaxy
MYSQL_DATABASE=galaxy
SHED_MYSQL_DATABASE=toolshed
mkdir -p /var/lib/mysql
chown -R mysql:mysql /var/lib/mysql
# Derived from
# https://github.com/docker-library/mysql/blob/master/5.7/docker-entrypoint.sh
DATADIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
tempSqlFile='/tmp/mysql-first-time.sql'
cat > "$tempSqlFile" <<-EOSQL
DELETE FROM mysql.user ;
CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
EOSQL
if [ "$MYSQL_DATABASE" ]; then
echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile"
fi
if [ "$SHED_MYSQL_DATABASE" ]; then
echo "CREATE DATABASE IF NOT EXISTS \`$SHED_MYSQL_DATABASE\` ;" >> "$tempSqlFile"
fi
if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile"
if [ "$MYSQL_DATABASE" ]; then
echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile"
fi
if [ "$SHED_MYSQL_DATABASE" ]; then
echo "GRANT ALL ON \`$SHED_MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile"
fi
fi
echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile"
mysqld_safe --init-file="$tempSqlFile" &
sleep 5
test/functional/tools/job_environment_default.xml
+1 -1
View File
@@ -2,7 +2,7 @@
<requirements>
<container type="docker">busybox:ubuntu-14.04</container>
</requirements>
<command><![CDATA[
<command detect_errors="exit_code"><![CDATA[
echo `id -u` > '$user_id';
echo `id -g` > '$group_id';
echo `pwd` > '$pwd';
test/functional/tools/job_environment_default_legacy.xml
+1 -1
View File
@@ -2,7 +2,7 @@
<requirements>
<container type="docker">busybox:ubuntu-14.04</container>
</requirements>
<command><![CDATA[
<command detect_errors="exit_code"><![CDATA[
echo `id -u` > '$user_id';
echo `id -g` > '$group_id';
echo `pwd` > '$pwd';
test/integration/singularity_job_conf.xml
+25
View File
@@ -0,0 +1,25 @@
<?xml version="1.0"?>
<job_conf>
<plugins>
<plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
</plugins>
<handlers>
<handler id="main"/>
</handlers>
<destinations default="local_singularity">
<destination id="local_singularity" runner="local">
<param id="singularity_enabled">true</param>
</destination>
<destination id="local_upload" runner="local">
</destination>
</destinations>
<tools>
<tool id="upload1" destination="local_upload" />
</tools>
</job_conf>
test/integration/test_dockerized_jobs.py → test/integration/test_containerized_jobs.py
+43 -7
View File
@@ -1,32 +1,47 @@
"""Integration tests for running tools in Docker containers."""
import os
import unittest
from base import integration_util
from base.populators import (
DatasetPopulator,
)
from galaxy.tools.deps.commands import which
from .test_job_environments import RunsEnvironmentJobs
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml")
SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.xml")
EXTENDED_TIMEOUT = 120
@integration_util.skip_unless_docker()
class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs):
framework_tool_and_types = True
job_config_file = DOCKERIZED_JOB_CONFIG_FILE
build_mulled_resolver = 'build_mulled'
container_type = 'docker'
default_container_home_dir = '/'
@classmethod
def handle_galaxy_config_kwds(cls, config):
cls.jobs_directory = cls._test_driver.mkdtemp()
config["jobs_directory"] = cls.jobs_directory
config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE
config["job_config_file"] = cls.job_config_file
# Disable tool dependency resolution.
config["tool_dependency_dir"] = "none"
config["conda_auto_init"] = False
config["conda_auto_install"] = False
config["enable_beta_mulled_containers"] = "true"
@classmethod
def setUpClass(cls):
if not which(cls.container_type):
raise unittest.SkipTest("Executable '%s' not found on PATH" % cls.container_type)
super(DockerizedJobsIntegrationTestCase, cls).setUpClass()
def setUp(self):
super(DockerizedJobsIntegrationTestCase, self).setUp()
self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
@@ -35,16 +50,16 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru
def test_explicit(self):
self.dataset_populator.run_tool("mulled_example_explicit", {}, self.history_id)
self.dataset_populator.wait_for_history(self.history_id, assert_ok=True)
output = self.dataset_populator.get_history_dataset_content(self.history_id)
output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT)
assert "0.7.15-r1140" in output
def test_mulled_simple(self):
self.dataset_populator.run_tool("mulled_example_simple", {}, self.history_id)
self.dataset_populator.wait_for_history(self.history_id, assert_ok=True)
output = self.dataset_populator.get_history_dataset_content(self.history_id)
output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT)
assert "0.7.15-r1140" in output
def test_docker_job_enviornment(self):
def test_container_job_enviornment(self):
job_env = self._run_and_get_environment_properties("job_environment_default")
euid = os.geteuid()
@@ -57,7 +72,7 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru
assert job_env.home.startswith(self.jobs_directory)
assert job_env.home.endswith("/home")
def test_docker_job_environment_legacy(self):
def test_container_job_environment_legacy(self):
job_env = self._run_and_get_environment_properties("job_environment_default_legacy")
euid = os.geteuid()
@@ -69,4 +84,25 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru
assert job_env.pwd.endswith("/working")
# Should we change env_pass_through to just always include TMP and HOME for docker?
# I'm not sure, if yes this would change.
assert job_env.home == "/", job_env.home
assert job_env.home == self.default_container_home_dir, job_env.home
def test_build_mulled(self):
if not which('docker'):
raise unittest.SkipTest("Docker not found on PATH, required for building images via involucro")
resolver_type = self.build_mulled_resolver
tool_id = 'mulled_example_multi_1'
endpoint = "tools/%s/dependencies" % tool_id
data = {'id': tool_id, 'resolver_type': resolver_type}
create_response = self._post(endpoint, data=data, admin=True)
self._assert_status_code_is(create_response, 200)
response = create_response.json()
assert any([True for d in response if d['dependency_type'] == self.container_type])
class SingularityJobsIntegrationTestCase(DockerizedJobsIntegrationTestCase):
job_config_file = SINGULARITY_JOB_CONFIG_FILE
build_mulled_resolver = 'build_mulled_singularity'
container_type = 'singularity'
# singularity passes $HOME by default
default_container_home_dir = os.environ.get('HOME', '/')
test/integration/test_resolvers.py
+24 -1
View File
@@ -1,4 +1,4 @@
"""Integration tests for conda dependency resolution."""
"""Integration tests for dependency resolution."""
import os
from tempfile import mkdtemp
@@ -125,6 +125,29 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase):
response = create_response.json()
assert not [True for d in response if d['dependency_type'] == 'conda']
def _uninstall_mulled_example_multi_1(self, resolver_type=None):
tool_id = 'mulled_example_multi_1'
endpoint = "tools/%s/dependencies" % tool_id
data = {'id': tool_id, 'resolver_type': resolver_type}
create_response = self._delete(endpoint, data=data, admin=True)
self._assert_status_code_is(create_response, 200)
response = create_response.json()
assert not [True for d in response if d['dependency_type'] == 'conda']
def test_conda_install_with_resolver_type_via_tools_api(self):
# Makes sure dependency is not already installed
self._uninstall_mulled_example_multi_1(resolver_type='conda')
# Now do the actual test
tool_id = 'mulled_example_multi_1'
endpoint = "tools/%s/dependencies" % tool_id
data = {'id': tool_id, 'resolver_type': 'conda'}
create_response = self._post(endpoint, data=data, admin=True)
self._assert_status_code_is(create_response, 200)
response = create_response.json()
assert any([True for d in response if d['dependency_type'] == 'conda'])
# Now that we know install was successfullt we can also doube check that the uninstall works
self._uninstall_mulled_example_multi_1(resolver_type='conda')
def test_conda_clean(self):
endpoint = 'dependency_resolvers/clean'
create_response = self._post(endpoint, data={}, admin=True)