diff --git a/.ci/jenkins/integration-py3/run_tests.sh b/.ci/jenkins/integration-py3/run_tests.sh index b45d68d67b5..ffc6c87e98c 100755 --- a/.ci/jenkins/integration-py3/run_tests.sh +++ b/.ci/jenkins/integration-py3/run_tests.sh @@ -1,3 +1,3 @@ #!/bin/bash -./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@" +DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --python3 --db postgres --clean_pyc --integration "$@" diff --git a/.ci/jenkins/integration/run_tests.sh b/.ci/jenkins/integration/run_tests.sh index f5b8f320f8e..4119aef60b2 100755 --- a/.ci/jenkins/integration/run_tests.sh +++ b/.ci/jenkins/integration/run_tests.sh @@ -1,3 +1,3 @@ #!/bin/bash -./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@" +DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@" diff --git a/config/container_resolvers_conf.xml.sample b/config/container_resolvers_conf.xml.sample index 40cfd3b1fbd..039f2f5bfe1 100644 --- a/config/container_resolvers_conf.xml.sample +++ b/config/container_resolvers_conf.xml.sample @@ -3,8 +3,8 @@ - - + - + + + + - - + packages listed in the tools definition. + Set auto_install to False if Galaxy should build container images + through the admin interface or API, but not automatically when + a tool is run. + --> + + + or more package requirements listed as part of the tool's definition. + Set auto_install to False if Galaxy should build container images + through the admin interface or API, but not automatically when + a tool is run. + + --> diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index a885309a562..f5c7b50ecba 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -330,17 +330,23 @@ galaxy: # by enable_beta_mulled_containers. #containers_resolvers_config_file: '' - # involucro is a tool used to build Docker containers for tools from - # Conda dependencies referenced in tools as `requirement`s. The - # following path is the location of involucro on the Galaxy host. This - # is ignored if the relevant container resolver isn't enabled, and - # will install on demand unless involucro_auto_init is set to False. + # involucro is a tool used to build Docker or Singularity containers + # for tools from Conda dependencies referenced in tools as + # `requirement`s. The following path is the location of involucro on + # the Galaxy host. This is ignored if the relevant container resolver + # isn't enabled, and will install on demand unless involucro_auto_init + # is set to False. #involucro_path: database/dependencies/involucro - # Install involucro as needed to build Docker containers for tools. - # Ignored if relevant container resolver is not used. + # Install involucro as needed to build Docker or Singularity + # containers for tools. Ignored if relevant container resolver is not + # used. #involucro_auto_init: true + # Conda channels to use when building Docker or Singularity containers + # using involucro. + #mulled_channels: 'conda-forge,bioconda' + # Enable automatic polling of relative tool sheds to see if any # updates are available for installed repositories. Ideally only one # Galaxy server process should be able to check for repository diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index ac5e1da47fe..c7604360086 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -521,12 +521,12 @@ ~~~~~~~~~~~~~~~~~~ :Description: - involucro is a tool used to build Docker containers for tools from - Conda dependencies referenced in tools as `requirement`s. The - following path is the location of involucro on the Galaxy host. - This is ignored if the relevant container resolver isn't enabled, - and will install on demand unless involucro_auto_init is set to - False. + involucro is a tool used to build Docker or Singularity containers + for tools from Conda dependencies referenced in tools as + `requirement`s. The following path is the location of involucro on + the Galaxy host. This is ignored if the relevant container + resolver isn't enabled, and will install on demand unless + involucro_auto_init is set to False. :Default: ``database/dependencies/involucro`` :Type: str @@ -536,12 +536,24 @@ ~~~~~~~~~~~~~~~~~~~~~~~ :Description: - Install involucro as needed to build Docker containers for tools. - Ignored if relevant container resolver is not used. + Install involucro as needed to build Docker or Singularity + containers for tools. Ignored if relevant container resolver is + not used. :Default: ``true`` :Type: bool +~~~~~~~~~~~~~~~~~~~ +``mulled_channels`` +~~~~~~~~~~~~~~~~~~~ + +:Description: + Conda channels to use when building Docker or Singularity + containers using involucro. +:Default: ``conda-forge,bioconda`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~ ``enable_tool_shed_check`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index e502bc4279a..c7355929ad6 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -4,6 +4,7 @@ Universe configuration builder. # absolute_import needed for tool_shed package. from __future__ import absolute_import +import collections import ipaddress import logging import logging.config @@ -23,6 +24,7 @@ from six.moves import configparser from galaxy.containers import parse_containers_config from galaxy.exceptions import ConfigurationError +from galaxy.tools.deps.container_resolvers.mulled import DEFAULT_CHANNELS from galaxy.util import ExecutionTimer from galaxy.util import listify from galaxy.util import string_as_bool @@ -518,6 +520,11 @@ class Configuration(object): involucro_path = os.path.join(tool_dependency_dir or "database", "involucro") self.involucro_path = resolve_path(involucro_path, self.root) self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True)) + mulled_channels = kwargs.get('mulled_channels') + if mulled_channels: + self.mulled_channels = [c.strip() for c in mulled_channels.split(',')] + else: + self.mulled_channels = DEFAULT_CHANNELS default_job_resubmission_condition = kwargs.get('default_job_resubmission_condition', '') if not default_job_resubmission_condition.strip(): @@ -1035,6 +1042,7 @@ class ConfiguresGalaxyMixin(object): from galaxy import tools from galaxy.managers.citations import CitationsManager from galaxy.tools.deps import containers + from galaxy.tools.deps.dependencies import AppInfo import galaxy.tools.search self.citations_manager = CitationsManager(self) @@ -1047,7 +1055,7 @@ class ConfiguresGalaxyMixin(object): self.toolbox = tools.ToolBox(tool_configs, self.config.tool_path, self) galaxy_root_dir = os.path.abspath(self.config.root) file_path = os.path.abspath(getattr(self.config, "file_path")) - app_info = containers.AppInfo( + app_info = AppInfo( galaxy_root_dir=galaxy_root_dir, default_file_path=file_path, outputs_to_working_directory=self.config.outputs_to_working_directory, @@ -1057,8 +1065,10 @@ class ConfiguresGalaxyMixin(object): containers_resolvers_config_file=self.config.containers_resolvers_config_file, involucro_path=self.config.involucro_path, involucro_auto_init=self.config.involucro_auto_init, + mulled_channels=self.config.mulled_channels, ) self.container_finder = containers.ContainerFinder(app_info) + self._set_enabled_container_types() index_help = getattr(self.config, "index_tool_help", True) self.toolbox_search = galaxy.tools.search.ToolBoxSearch(self.toolbox, index_help) self.reindex_tool_search() @@ -1068,6 +1078,16 @@ class ConfiguresGalaxyMixin(object): self.toolbox_search.build_index(tool_cache=self.tool_cache) self.tool_cache.reset_status() + def _set_enabled_container_types(self): + container_types_to_destinations = collections.defaultdict(list) + for destinations in self.job_config.destinations.values(): + for destination in destinations: + for enabled_container_type in self.container_finder._enabled_container_types(destination.params): + container_types_to_destinations[enabled_container_type].append(destination) + self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) + self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes) + self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers) + def _configure_tool_data_tables(self, from_shed_config): from galaxy.tools.data import ToolDataTableManager diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 16fb348e72c..5418dfeba3d 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -685,6 +685,7 @@ class JobWrapper(HasResourceParameters): self.extra_filenames = [] self.command_line = None self.dependencies = [] + self._dependency_shell_commands = None # Tool versioning variables self.write_version_cmd = None self.version_string = "" @@ -728,6 +729,15 @@ class JobWrapper(HasResourceParameters): def dataset_path_rewriter(self): return self._job_dataset_path_rewriter + @property + def dependency_shell_commands(self): + """Shell fragment to inject dependencies.""" + if self._dependency_shell_commands is None: + self._dependency_shell_commands = self.tool.build_dependency_shell_commands( + job_directory=self.working_directory + ) + return self._dependency_shell_commands + @property def cleanup_job(self): """ Remove the job after it is complete, should return "always", "onsuccess", or "never". @@ -860,8 +870,6 @@ class JobWrapper(HasResourceParameters): self.command_line, self.extra_filenames, self.environment_variables = tool_evaluator.build() # Ensure galaxy_lib_dir is set in case there are any later chdirs self.galaxy_lib_dir - # Shell fragment to inject dependencies - self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory) if self.tool.requires_galaxy_python_environment: # These tools (upload, metadata, data_source) may need access to the datatypes registry. self.app.datatypes_registry.to_xml_file(os.path.join(self.working_directory, 'registry.xml')) @@ -1959,8 +1967,7 @@ class TaskWrapper(JobWrapper): # Ensure galaxy_lib_dir is set in case there are any later chdirs self.galaxy_lib_dir - # Shell fragment to inject dependencies - self.dependency_shell_commands = self.tool.build_dependency_shell_commands(job_directory=self.working_directory) + # We need command_line persisted to the db in order for Galaxy to re-queue the job # if the server was stopped and restarted before the job finished task.command_line = self.command_line diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 7922531ee8b..ebeaa4bc94a 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -158,9 +158,8 @@ def __handle_task_splitting(commands_builder, job_wrapper): def __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params): local_dependency_resolution = remote_command_params.get("dependency_resolution", "local") == "local" - # Prepend dependency injection - if job_wrapper.dependency_shell_commands and local_dependency_resolution: + if local_dependency_resolution and job_wrapper.dependency_shell_commands: commands_builder.prepend_commands(job_wrapper.dependency_shell_commands) diff --git a/lib/galaxy/jobs/rule_helper.py b/lib/galaxy/jobs/rule_helper.py index b45a6f57bf6..2dd339d6ae4 100644 --- a/lib/galaxy/jobs/rule_helper.py +++ b/lib/galaxy/jobs/rule_helper.py @@ -9,6 +9,7 @@ from galaxy import ( model, util ) +from galaxy.tools.deps.dependencies import ToolInfo log = logging.getLogger(__name__) @@ -42,9 +43,7 @@ class RuleHelper(object): else: # Have a Job object. tool = self.app.toolbox.get_tool(job_or_tool.tool_id, tool_version=job_or_tool.tool_version) - # Can't import at top because circular import between galaxy.tools and galaxy.jobs. - import galaxy.tools.deps.containers - tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) + tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) container_description = self.app.container_finder.find_best_container_description(["docker"], tool_info) return container_description is not None diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 3849b2299d8..3ddec1e070e 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -25,6 +25,10 @@ from galaxy.jobs.runners.util.job_script import ( job_script, write_script ) +from galaxy.tools.deps.dependencies import ( + JobInfo, + ToolInfo +) from galaxy.util import ( DATABASE_MAX_STRING_SIZE, ExecutionTimer, @@ -394,9 +398,8 @@ class BaseJobRunner(object): compute_tmp_directory = job_wrapper.tmp_directory() tool = job_wrapper.tool - from galaxy.tools.deps import containers - tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) - job_info = containers.JobInfo( + tool_info = ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through) + job_info = JobInfo( compute_working_directory, compute_tool_directory, compute_job_directory, diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 0e68d8c068d..116193ddc7e 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -13,11 +13,17 @@ from galaxy.util import ( plugin_config ) from galaxy.util.oset import OrderedSet +from .container_resolvers import ContainerResolver +from .dependencies import ToolInfo from .requirements import ( + ContainerDescription, ToolRequirement, ToolRequirements ) -from .resolvers import NullDependency +from .resolvers import ( + ContainerDependency, + NullDependency, +) from .resolvers.conda import CondaDependencyResolver from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver @@ -44,19 +50,6 @@ def build_dependency_manager(config): return dependency_manager -class NullDependencyManager(object): - dependency_resolvers = [] - - def uses_tool_shed_dependencies(self): - return False - - def dependency_shell_commands(self, requirements, **kwds): - return [] - - def find_dep(self, name, version=None, type='package', **kwds): - return NullDependency(version=version, name=name) - - class DependencyManager(object): """ A DependencyManager attempts to resolve named and versioned dependencies by @@ -82,6 +75,27 @@ class DependencyManager(object): self.default_base_path = os.path.abspath(default_base_path) self.resolver_classes = self.__resolvers_dict() self.dependency_resolvers = self.__build_dependency_resolvers(conf_file) + self._enabled_container_types = [] + self._destination_for_container_type = {} + + def set_enabled_container_types(self, container_types_to_destinations): + """Set the union of all enabled container types.""" + self._enabled_container_types = [container_type for container_type in container_types_to_destinations.keys()] + # Just pick first enabled destination for a container type, probably covers the most common deployment scenarios + self._destination_for_container_type = container_types_to_destinations + + def get_destination_info_for_container_type(self, container_type, destination_id=None): + if destination_id is None: + return next(iter(self._destination_for_container_type[container_type])).params + else: + for destination in self._destination_for_container_type[container_type]: + if destination.id == destination_id: + return destination.params + + @property + def enabled_container_types(self): + """Returns the union of enabled container types.""" + return self._enabled_container_types def get_resolver_option(self, resolver, key, explicit_resolver_options={}): """Look in resolver-specific settings for option and then fallback to global settings. @@ -110,7 +124,7 @@ class DependencyManager(object): def dependency_shell_commands(self, requirements, **kwds): requirements_to_dependencies = self.requirements_to_dependencies(requirements, **kwds) ordered_dependencies = OrderedSet(requirements_to_dependencies.values()) - return [dependency.shell_commands() for dependency in ordered_dependencies] + return [dependency.shell_commands() for dependency in ordered_dependencies if not isinstance(dependency, ContainerDependency)] def requirements_to_dependencies(self, requirements, **kwds): """ @@ -125,29 +139,58 @@ class DependencyManager(object): return requirement_to_dependency - def _requirements_to_dependencies_dict(self, requirements, **kwds): + def _requirements_to_dependencies_dict(self, requirements, search=False, **kwds): """Build simple requirements to dependencies dict for resolution.""" requirement_to_dependency = OrderedDict() - index = kwds.get('index', None) + index = kwds.get('index') + install = kwds.get('install', False) + resolver_type = kwds.get('resolver_type') require_exact = kwds.get('exact', False) return_null_dependencies = kwds.get('return_null', False) resolvable_requirements = requirements.resolvable + tool_info = ToolInfo(requirements=resolvable_requirements) for i, resolver in enumerate(self.dependency_resolvers): + if index is not None and i != index: continue - if len(requirement_to_dependency) == len(resolvable_requirements): + if resolver_type is not None and resolver.resolver_type != resolver_type: + continue + + _requirement_to_dependency = OrderedDict([(k, v) for k, v in requirement_to_dependency.items() if not isinstance(v, NullDependency)]) + + if len(_requirement_to_dependency) == len(resolvable_requirements): # Shortcut - resolution complete. break + if resolver.resolver_type.startswith('build_mulled') and not install: + # don't want to build images here + continue + # Check requirements all at once - all_unmet = len(requirement_to_dependency) == 0 - if all_unmet and hasattr(resolver, "resolve_all"): + all_unmet = len(_requirement_to_dependency) == 0 + if hasattr(resolver, "resolve_all"): + resolve = resolver.resolve_all + elif isinstance(resolver, ContainerResolver): + if not resolver.resolver_type.startswith(('cached', 'explicit')) and not (search or install): + # These would look up available containers using the quay API, + # we only want to do this if we search for containers + continue + resolve = resolver.resolve + else: + resolve = None + if all_unmet and resolve is not None: # TODO: Handle specs. - dependencies = resolver.resolve_all(resolvable_requirements, **kwds) + dependencies = resolve(requirements=resolvable_requirements, + enabled_container_types=self.enabled_container_types, + destination_for_container_type=self.get_destination_info_for_container_type, + tool_info=tool_info, + **kwds) if dependencies: + if isinstance(dependencies, ContainerDescription): + dependencies = [ContainerDependency(dependencies, name=r.name, version=r.version) for r in resolvable_requirements] assert len(dependencies) == len(resolvable_requirements) for requirement, dependency in zip(resolvable_requirements, dependencies): log.debug(dependency.resolver_msg) @@ -156,21 +199,24 @@ class DependencyManager(object): # Shortcut - resolution complete. break - # Check individual requirements - for requirement in resolvable_requirements: - if requirement in requirement_to_dependency: - continue + if not isinstance(resolver, ContainerResolver): - dependency = resolver.resolve(requirement, **kwds) - if require_exact and not dependency.exact: - continue + # Check individual requirements + for requirement in resolvable_requirements: + if requirement in _requirement_to_dependency: + continue - if not isinstance(dependency, NullDependency): - log.debug(dependency.resolver_msg) - requirement_to_dependency[requirement] = dependency - elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index): - log.debug(dependency.resolver_msg) - requirement_to_dependency[requirement] = dependency + dependency = resolver.resolve(requirement, **kwds) + if require_exact and not dependency.exact: + continue + + if not isinstance(dependency, NullDependency): + log.debug(dependency.resolver_msg) + requirement_to_dependency[requirement] = dependency + elif return_null_dependencies: + log.debug(dependency.resolver_msg) + dependency.version = requirement.version + requirement_to_dependency[requirement] = dependency return requirement_to_dependency @@ -273,3 +319,22 @@ class CachedDependencyManager(DependencyManager): """ req_hashes = self.hash_dependencies(resolved_dependencies) return os.path.abspath(os.path.join(self.tool_dependency_cache_dir, req_hashes)) + + +class NullDependencyManager(DependencyManager): + + def __init__(self, default_base_path=None, conf_file=None, app_config={}): + self.__app_config = app_config + self.resolver_classes = set() + self.dependency_resolvers = [] + self._enabled_container_types = [] + self._destination_for_container_type = {} + + def uses_tool_shed_dependencies(self): + return False + + def dependency_shell_commands(self, requirements, **kwds): + return [] + + def find_dep(self, name, version=None, type='package', **kwds): + return NullDependency(version=version, name=name) diff --git a/lib/galaxy/tools/deps/container_classes.py b/lib/galaxy/tools/deps/container_classes.py new file mode 100644 index 00000000000..ecb10f46362 --- /dev/null +++ b/lib/galaxy/tools/deps/container_classes.py @@ -0,0 +1,357 @@ +import os +import string +from abc import ( + ABCMeta, + abstractmethod +) + +import six + +from galaxy.containers.docker_model import DockerVolume +from galaxy.util import ( + asbool, + in_directory +) +from . import ( + docker_util, + singularity_util +) +from .requirements import ( + DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, + DEFAULT_CONTAINER_SHELL, +) + +DOCKER_CONTAINER_TYPE = "docker" +SINGULARITY_CONTAINER_TYPE = "singularity" + +LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r''' +python << EOF +from __future__ import print_function + +import json +import re +import subprocess +import tarfile + +t = tarfile.TarFile("${cached_image_file}") +meta_str = t.extractfile('repositories').read() +meta = json.loads(meta_str) +tag, tag_value = meta.items()[0] +rev, rev_value = tag_value.items()[0] +cmd = "${images_cmd}" +proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) +stdo, stde = proc.communicate() +found = False +for line in stdo.split("\n"): + tmp = re.split(r'\s+', line) + if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value: + found = True +if not found: + print("Loading image") + cmd = "cat ${cached_image_file} | ${load_cmd}" + subprocess.check_call(cmd, shell=True) +EOF +''' + + +@six.add_metaclass(ABCMeta) +class Container(object): + + def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description): + self.container_id = container_id + self.app_info = app_info + self.tool_info = tool_info + self.destination_info = destination_info + self.job_info = job_info + self.container_description = container_description + + def prop(self, name, default): + destination_name = "docker_%s" % name + return self.destination_info.get(destination_name, default) + + @property + def resolve_dependencies(self): + return DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES if not self.container_description else self.container_description.resolve_dependencies + + @property + def shell(self): + return DEFAULT_CONTAINER_SHELL if not self.container_description else self.container_description.shell + + @abstractmethod + def containerize_command(self, command): + """ + Use destination supplied container configuration parameters, + container_id, and command to build a new command that runs + input command in container. + """ + + +def preprocess_volumes(volumes_raw_str, container_type): + """Process Galaxy volume specification string to either Docker or Singularity specification. + + Galaxy allows the mount try "default_ro" which translates to ro for Docker and + ro for Singularity iff no subdirectories are rw (Singularity does not allow ro + parent directories with rw subdirectories). + + >>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE) + ['/a/b:rw'] + >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) + ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) + ['/a/b:ro', '/a/b/c:rw'] + >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) + ['/a/b:rw', '/a/b/c:rw'] + """ + + volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")] + volumes = [] + rw_paths = [] + + for volume_raw_str in volumes_raw_strs: + volume_parts = volume_raw_str.split(":") + if len(volume_parts) > 2: + raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str) + if len(volume_parts) == 1: + volume_parts.append("rw") + volumes.append(volume_parts) + if volume_parts[1] == "rw": + rw_paths.append(volume_parts[0]) + + for volume in volumes: + path = volume[0] + how = volume[1] + + if how == "default_ro": + how = "ro" + if container_type == SINGULARITY_CONTAINER_TYPE: + for rw_path in rw_paths: + if in_directory(rw_path, path): + how = "rw" + + volume[1] = how + + return [":".join(v) for v in volumes] + + +class HasDockerLikeVolumes(object): + """Mixin to share functionality related to Docker volume handling. + + Singularity seems to have a fairly compatible syntax for volume handling. + """ + + def _expand_volume_str(self, value): + if not value: + return value + + template = string.Template(value) + variables = dict() + + def add_var(name, value): + if value: + if not value.startswith("$"): + value = os.path.abspath(value) + variables[name] = value + + add_var("working_directory", self.job_info.working_directory) + add_var("tmp_directory", self.job_info.tmp_directory) + add_var("job_directory", self.job_info.job_directory) + add_var("tool_directory", self.job_info.tool_directory) + add_var("galaxy_root", self.app_info.galaxy_root_dir) + add_var("default_file_path", self.app_info.default_file_path) + add_var("library_import_dir", self.app_info.library_import_dir) + + if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar": + # We have a Pulsar job directory, so everything needed (excluding index + # files) should be available in job_directory... + defaults = "$job_directory:default_ro,$tool_directory:default_ro,$job_directory/outputs:rw,$working_directory:rw" + else: + defaults = "$galaxy_root:default_ro,$tool_directory:default_ro" + if self.job_info.job_directory: + defaults += ",$job_directory:default_ro" + if self.job_info.tmp_directory is not None: + defaults += ",$tmp_directory:rw" + if self.app_info.outputs_to_working_directory: + # Should need default_file_path (which is of course an estimate given + # object stores anyway). + defaults += ",$working_directory:rw,$default_file_path:default_ro" + else: + defaults += ",$working_directory:rw,$default_file_path:rw" + + if self.app_info.library_import_dir: + defaults += ",$library_import_dir:default_ro" + + # Define $defaults that can easily be extended with external library and + # index data without deployer worrying about above details. + variables["defaults"] = string.Template(defaults).safe_substitute(variables) + + return template.safe_substitute(variables) + + +class DockerContainer(Container, HasDockerLikeVolumes): + + container_type = DOCKER_CONTAINER_TYPE + + @property + def docker_host_props(self): + docker_host_props = dict( + docker_cmd=self.prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND), + sudo=asbool(self.prop("sudo", docker_util.DEFAULT_SUDO)), + sudo_cmd=self.prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND), + host=self.prop("host", docker_util.DEFAULT_HOST), + ) + return docker_host_props + + def build_pull_command(self): + return docker_util.build_pull_command(self.container_id, **self.docker_host_props) + + def containerize_command(self, command): + env_directives = [] + for pass_through_var in self.tool_info.env_pass_through: + env_directives.append('"%s=$%s"' % (pass_through_var, pass_through_var)) + + # Allow destinations to explicitly set environment variables just for + # docker container. Better approach is to set for destination and then + # pass through only what tool needs however. (See todo in ToolInfo.) + for key, value in six.iteritems(self.destination_info): + if key.startswith("docker_env_"): + env = key[len("docker_env_"):] + env_directives.append('"%s=%s"' % (env, value)) + + working_directory = self.job_info.working_directory + if not working_directory: + raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) + + volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults")) + preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) + # TODO: Remove redundant volumes... + volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] + # If a tool definitely has a temp directory available set it to /tmp in container for compat. + # with CWL. This is part of that spec and should make it easier to share containers between CWL + # and Galaxy. + if self.job_info.tmp_directory is not None: + volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory)) + volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM) + + docker_host_props = self.docker_host_props + + cached_image_file = self.__get_cached_image_file() + if not cached_image_file: + # TODO: Add option to cache it once here and create cached_image_file. + cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) + else: + cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props) + run_command = docker_util.build_docker_run_command( + command, + self.container_id, + volumes=volumes, + volumes_from=volumes_from, + env_directives=env_directives, + working_directory=working_directory, + net=self.prop("net", "none"), # By default, docker instance has networking disabled + auto_rm=asbool(self.prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)), + set_user=self.prop("set_user", docker_util.DEFAULT_SET_USER), + run_extra_arguments=self.prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS), + **docker_host_props + ) + return "%s\n%s" % (cache_command, run_command) + + def __cache_from_file_command(self, cached_image_file, docker_host_props): + images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props) + load_cmd = docker_util.build_docker_load_command(**docker_host_props) + + return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute( + cached_image_file=cached_image_file, + images_cmd=images_cmd, + load_cmd=load_cmd + ) + + def __get_cached_image_file(self): + container_id = self.container_id + cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path")) + cache_path = docker_cache_path(cache_directory, container_id) + return cache_path if os.path.exists(cache_path) else None + + def __get_destination_overridable_property(self, name): + prop_name = "docker_%s" % name + if prop_name in self.destination_info: + return self.destination_info[prop_name] + else: + return getattr(self.app_info, name) + + +def docker_cache_path(cache_directory, container_id): + file_container_id = container_id.replace("/", "_slash_") + cache_file_name = "docker_%s.tar" % file_container_id + return os.path.join(cache_directory, cache_file_name) + + +class SingularityContainer(Container, HasDockerLikeVolumes): + + container_type = SINGULARITY_CONTAINER_TYPE + + def get_singularity_target_kwds(self): + return dict( + singularity_cmd=self.prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND), + sudo=asbool(self.prop("sudo", singularity_util.DEFAULT_SUDO)), + sudo_cmd=self.prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND), + ) + + def build_mulled_singularity_pull_command(self, cache_directory, namespace="biocontainers"): + return singularity_util.pull_mulled_singularity_command( + docker_image_identifier=self.container_id, + cache_directory=cache_directory, + namespace=namespace, + **self.get_singularity_target_kwds() + ) + + def containerize_command(self, command): + + env = [] + for pass_through_var in self.tool_info.env_pass_through: + env.append((pass_through_var, "$%s" % pass_through_var)) + + # Allow destinations to explicitly set environment variables just for + # docker container. Better approach is to set for destination and then + # pass through only what tool needs however. (See todo in ToolInfo.) + for key, value in six.iteritems(self.destination_info): + if key.startswith("singularity_env_"): + real_key = key[len("singularity_env_"):] + env.append((real_key, value)) + + working_directory = self.job_info.working_directory + if not working_directory: + raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) + + volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults")) + preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) + volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] + + run_command = singularity_util.build_singularity_run_command( + command, + self.container_id, + volumes=volumes, + env=env, + working_directory=working_directory, + run_extra_arguments=self.prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS), + **self.get_singularity_target_kwds() + ) + return run_command + + +CONTAINER_CLASSES = dict( + docker=DockerContainer, + singularity=SingularityContainer, +) + + +class NullContainer(object): + + def __init__(self): + pass + + def __bool__(self): + return False + __nonzero__ = __bool__ + + +NULL_CONTAINER = NullContainer() diff --git a/lib/galaxy/tools/deps/container_resolvers/__init__.py b/lib/galaxy/tools/deps/container_resolvers/__init__.py index 0e819f9458a..337b0328a4d 100644 --- a/lib/galaxy/tools/deps/container_resolvers/__init__.py +++ b/lib/galaxy/tools/deps/container_resolvers/__init__.py @@ -16,7 +16,8 @@ class ContainerResolver(Dictifiable): """Description of a technique for resolving container images for tool execution.""" # Keys for dictification. - dict_collection_visible_keys = ['resolver_type'] + dict_collection_visible_keys = ['resolver_type', 'can_uninstall_dependencies'] + can_uninstall_dependencies = False def __init__(self, app_info=None, **kwds): """Default initializer for ``ContainerResolver`` subclasses.""" @@ -33,7 +34,7 @@ class ContainerResolver(Dictifiable): return default @abstractmethod - def resolve(self, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): """Find a container matching all supplied requirements for tool. The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description diff --git a/lib/galaxy/tools/deps/container_resolvers/explicit.py b/lib/galaxy/tools/deps/container_resolvers/explicit.py index 043ac20fed6..1480f078dce 100644 --- a/lib/galaxy/tools/deps/container_resolvers/explicit.py +++ b/lib/galaxy/tools/deps/container_resolvers/explicit.py @@ -13,7 +13,7 @@ class ExplicitContainerResolver(ContainerResolver): resolver_type = "explicit" - def resolve(self, enabled_container_types, tool_info): + def resolve(self, enabled_container_types, tool_info, **kwds): """Find a container explicitly mentioned in tool description. This ignores the tool requirements and assumes the tool author crafted @@ -26,4 +26,28 @@ class ExplicitContainerResolver(ContainerResolver): return None -__all__ = ("ExplicitContainerResolver", ) +class ExplicitSingularityContainerResolver(ExplicitContainerResolver): + + resolver_type = 'explicit_singularity' + container_type = 'singularity' + + def resolve(self, enabled_container_types, tool_info, **kwds): + """Find a container explicitly mentioned in tool description. + + This ignores the tool requirements and assumes the tool author crafted + a correct container. We use singularity here to fetch docker containers, + hence the container_description hack here. + """ + for container_description in tool_info.container_descriptions: + if container_description.type == 'docker': + desc_dict = container_description.to_dict() + desc_dict['type'] = self.container_type + desc_dict['identifier'] = "docker://%s" % container_description.identifier + container_description = container_description.from_dict(desc_dict) + if self._container_type_enabled(container_description, enabled_container_types): + return container_description + + return None + + +__all__ = ("ExplicitContainerResolver", "ExplicitSingularityContainerResolver") diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py index fc53d5cf7a8..8d108a3da55 100644 --- a/lib/galaxy/tools/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -7,7 +7,12 @@ import subprocess import six -from galaxy.util import unicodify +from galaxy.util import ( + string_as_bool, + unicodify, +) +from ..commands import shell +from ..container_classes import CONTAINER_CLASSES from ..container_resolvers import ( ContainerResolver, ) @@ -25,7 +30,10 @@ from ..mulled.util import ( v1_image_name, v2_image_name, ) -from ..requirements import ContainerDescription +from ..requirements import ( + ContainerDescription, + DEFAULT_CONTAINER_SHELL, +) log = logging.getLogger(__name__) @@ -112,7 +120,8 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None): build = None if version and "--" in version: version, build = split_tag(version) - + if prefix and image_name.startswith(prefix): + image_name = image_name[len(prefix):] image = CachedMulledImageSingleTarget(image_name, version, build, identifier) return image @@ -177,7 +186,7 @@ def find_best_matching_cached_image(targets, cached_images, hash_func): return image -def docker_cached_container_description(targets, namespace, hash_func="v2"): +def docker_cached_container_description(targets, namespace, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL): if len(targets) == 0: return None @@ -189,12 +198,13 @@ def docker_cached_container_description(targets, namespace, hash_func="v2"): container = ContainerDescription( image.image_identifier, type="docker", + shell=shell, ) return container -def singularity_cached_container_description(targets, cache_directory, hash_func="v2"): +def singularity_cached_container_description(targets, cache_directory, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL): if len(targets) == 0: return None @@ -209,28 +219,85 @@ def singularity_cached_container_description(targets, cache_directory, hash_func container = ContainerDescription( os.path.abspath(os.path.join(cache_directory, image.image_identifier)), type="singularity", + shell=shell, ) return container +def targets_to_mulled_name(targets, hash_func, namespace): + name = None + if len(targets) == 1: + target = targets[0] + target_version = target.version + tags = mulled_tags_for(namespace, target.package_name) + + if not tags: + return None + + if target_version: + for tag in tags: + if '--' in tag: + version, build = split_tag(tag) + else: + version = tag + build = None + if version == target_version: + name = "%s:%s" % (target.package_name, version) + if build: + name = "%s--%s" % (name, build) + break + else: + version, build = split_tag(tags[0]) + name = "%s:%s--%s" % (target.package_name, version, build) + else: + def tags_if_available(image_name): + if ":" in image_name: + repo_name, tag_prefix = image_name.split(":", 2) + else: + repo_name = image_name + tag_prefix = None + tags = mulled_tags_for(namespace, repo_name, tag_prefix=tag_prefix) + return tags + + if hash_func == "v2": + base_image_name = v2_image_name(targets) + tags = tags_if_available(base_image_name) + if tags: + if ":" in base_image_name: + # base_image_name of form :, expand tag + # to include build number in tag. + name = "%s:%s" % (base_image_name.split(":")[0], tags[0]) + else: + # base_image_name of form , simply add build number + # as tag to fully qualify image. + name = "%s:%s" % (base_image_name, tags[0]) + elif hash_func == "v1": + base_image_name = v1_image_name(targets) + tags = tags_if_available(base_image_name) + if tags: + name = "%s:%s" % (base_image_name, tags[0]) + return name + + @six.python_2_unicode_compatible class CachedMulledDockerContainerResolver(ContainerResolver): resolver_type = "cached_mulled" container_type = "docker" + shell = '/bin/bash' - def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds): super(CachedMulledDockerContainerResolver, self).__init__(app_info) self.namespace = namespace self.hash_func = hash_func - def resolve(self, enabled_container_types, tool_info): - if tool_info.requires_galaxy_python_environment: + def resolve(self, enabled_container_types, tool_info, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) - return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) def __str__(self): return "CachedMulledDockerContainerResolver[namespace=%s]" % self.namespace @@ -241,18 +308,19 @@ class CachedMulledSingularityContainerResolver(ContainerResolver): resolver_type = "cached_mulled_singularity" container_type = "singularity" + shell = '/bin/bash' def __init__(self, app_info=None, hash_func="v2", **kwds): super(CachedMulledSingularityContainerResolver, self).__init__(app_info) self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) self.hash_func = hash_func - def resolve(self, enabled_container_types, tool_info): - if tool_info.requires_galaxy_python_environment: + def resolve(self, enabled_container_types, tool_info, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) - return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) def __str__(self): return "CachedMulledSingularityContainerResolver[cache_directory=%s]" % self.cache_directory @@ -264,91 +332,108 @@ class MulledDockerContainerResolver(ContainerResolver): resolver_type = "mulled" container_type = "docker" + shell = '/bin/bash' + protocol = None - def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"): + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): super(MulledDockerContainerResolver, self).__init__(app_info) self.namespace = namespace self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) - def resolve(self, enabled_container_types, tool_info): - if tool_info.requires_galaxy_python_environment: + def cached_container_description(self, targets, namespace, hash_func): + return docker_cached_container_description(targets, namespace, hash_func) + + def pull(self, container): + command = container.build_pull_command() + shell(command) + + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) if len(targets) == 0: return None - name = None - - if len(targets) == 1: - target = targets[0] - target_version = target.version - tags = mulled_tags_for(self.namespace, target.package_name) - - if not tags: - return None - - if target_version: - for tag in tags: - version, build = split_tag(tag) - if version == target_version: - name = "%s:%s--%s" % (target.package_name, version, build) - break - else: - version, build = split_tag(tags[0]) - name = "%s:%s--%s" % (target.package_name, version, build) - else: - def tags_if_available(image_name): - if ":" in image_name: - repo_name, tag_prefix = image_name.split(":", 2) - else: - repo_name = image_name - tag_prefix = None - tags = mulled_tags_for(self.namespace, repo_name, tag_prefix=tag_prefix) - return tags - - if self.hash_func == "v2": - base_image_name = v2_image_name(targets) - tags = tags_if_available(base_image_name) - if tags: - if ":" in base_image_name: - # base_image_name of form :, expand tag - # to include build number in tag. - name = "%s:%s" % (base_image_name.split(":")[0], tags[0]) - else: - # base_image_name of form , simply add build number - # as tag to fully qualify image. - name = "%s:%s" % (base_image_name, tags[0]) - elif self.hash_func == "v1": - base_image_name = v1_image_name(targets) - tags = tags_if_available(base_image_name) - if tags: - name = "%s:%s" % (base_image_name, tags[0]) - + name = targets_to_mulled_name(targets=targets, hash_func=self.hash_func, namespace=self.namespace) if name: - return ContainerDescription( - "quay.io/%s/%s" % (self.namespace, name), + container_id = "quay.io/%s/%s" % (self.namespace, name) + if self.protocol: + container_id = "%s%s" % (self.protocol, container_id) + container_description = ContainerDescription( + container_id, type=self.container_type, + shell=self.shell, ) + destination_for_container_type = kwds.get('destination_for_container_type') + if install and destination_for_container_type and not self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func + ): + container = CONTAINER_CLASSES[self.container_type](container_description.identifier, + self.app_info, + tool_info, + destination_for_container_type(self.container_type), + {}, + container_description) + self.pull(container) + if not self.auto_install: + container_description = self.cached_container_description( + targets, + namespace=self.namespace, + hash_func=self.hash_func + ) + return container_description def __str__(self): return "MulledDockerContainerResolver[namespace=%s]" % self.namespace +@six.python_2_unicode_compatible +class MulledSingularityContainerResolver(MulledDockerContainerResolver): + + resolver_type = "mulled_singularity" + container_type = "singularity" + protocol = 'docker://' + + def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds): + super(MulledSingularityContainerResolver, self).__init__(app_info) + self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) + self.namespace = namespace + self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) + + def cached_container_description(self, targets, namespace, hash_func): + return singularity_cached_container_description(targets, + cache_directory=self.cache_directory, + hash_func=hash_func) + + def pull(self, container): + cmds = container.build_mulled_singularity_pull_command(cache_directory=self.cache_directory, namespace=self.namespace) + shell(cmds=cmds) + + def __str__(self): + return "MulledSingularityContainerResolver[namespace=%s]" % self.namespace + + @six.python_2_unicode_compatible class BuildMulledDockerContainerResolver(ContainerResolver): """Build for Docker mulled images matching tool dependencies.""" resolver_type = "build_mulled" container_type = "docker" + shell = '/bin/bash' - def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds): + def __init__(self, app_info=None, namespace="local", hash_func="v2", auto_install=True, **kwds): super(BuildMulledDockerContainerResolver, self).__init__(app_info) self._involucro_context_kwds = { 'involucro_bin': self._get_config_option("involucro_path", None) } self.namespace = namespace self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) self._mulled_kwds = { 'namespace': namespace, 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), @@ -357,20 +442,20 @@ class BuildMulledDockerContainerResolver(ContainerResolver): } self.auto_init = self._get_config_option("involucro_auto_init", True) - def resolve(self, enabled_container_types, tool_info): - if tool_info.requires_galaxy_python_environment: + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) if len(targets) == 0: return None - - mull_targets( - targets, - involucro_context=self._get_involucro_context(), - **self._mulled_kwds - ) - return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func) + if self.auto_install or install: + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) + return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell) def _get_involucro_context(self): involucro_context = InvolucroContext(**self._involucro_context_kwds) @@ -387,14 +472,16 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): resolver_type = "build_mulled_singularity" container_type = "singularity" + shell = '/bin/bash' - def __init__(self, app_info=None, hash_func="v2", **kwds): + def __init__(self, app_info=None, hash_func="v2", auto_install=True, **kwds): super(BuildMulledSingularityContainerResolver, self).__init__(app_info) self._involucro_context_kwds = { 'involucro_bin': self._get_config_option("involucro_path", None) } self.cache_directory = kwds.get("cache_directory", os.path.join(app_info.container_image_cache_path, "singularity", "mulled")) self.hash_func = hash_func + self.auto_install = string_as_bool(auto_install) self._mulled_kwds = { 'channels': self._get_config_option("mulled_channels", DEFAULT_CHANNELS), 'hash_func': self.hash_func, @@ -404,20 +491,21 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): } self.auto_init = self._get_config_option("involucro_auto_init", True) - def resolve(self, enabled_container_types, tool_info): - if tool_info.requires_galaxy_python_environment: + def resolve(self, enabled_container_types, tool_info, install=False, **kwds): + if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types: return None targets = mulled_targets(tool_info) if len(targets) == 0: return None - mull_targets( - targets, - involucro_context=self._get_involucro_context(), - **self._mulled_kwds - ) - return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func) + if self.auto_install or install: + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) + return singularity_cached_container_description(targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell) def _get_involucro_context(self): involucro_context = InvolucroContext(**self._involucro_context_kwds) @@ -425,7 +513,7 @@ class BuildMulledSingularityContainerResolver(ContainerResolver): return involucro_context def __str__(self): - return "BuildDockerContainerResolver[cache_directory=%s]" % self.cache_directory + return "BuildSingularityContainerResolver[cache_directory=%s]" % self.cache_directory def mulled_targets(tool_info): @@ -436,6 +524,7 @@ __all__ = ( "CachedMulledDockerContainerResolver", "CachedMulledSingularityContainerResolver", "MulledDockerContainerResolver", + "MulledSingularityContainerResolver", "BuildMulledDockerContainerResolver", "BuildMulledSingularityContainerResolver", ) diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index a43516e615d..5a8234f84db 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -1,73 +1,38 @@ import logging import os -import string -from abc import ( - ABCMeta, - abstractmethod -) -import six - -from galaxy.containers.docker_model import DockerVolume from galaxy.util import ( asbool, - in_directory, plugin_config ) -from . import ( - docker_util, - singularity_util +from .container_classes import ( + CONTAINER_CLASSES, + DOCKER_CONTAINER_TYPE, + NULL_CONTAINER, + SINGULARITY_CONTAINER_TYPE, +) +from .container_resolvers.explicit import ( + ExplicitContainerResolver, + ExplicitSingularityContainerResolver, ) -from .container_resolvers.explicit import ExplicitContainerResolver from .container_resolvers.mulled import ( BuildMulledDockerContainerResolver, BuildMulledSingularityContainerResolver, CachedMulledDockerContainerResolver, CachedMulledSingularityContainerResolver, MulledDockerContainerResolver, + MulledSingularityContainerResolver, ) from .requirements import ( ContainerDescription, - DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, - DEFAULT_CONTAINER_SHELL ) log = logging.getLogger(__name__) -DOCKER_CONTAINER_TYPE = "docker" -SINGULARITY_CONTAINER_TYPE = "singularity" + DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE] -LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = r''' -python << EOF -from __future__ import print_function - -import json -import re -import subprocess -import tarfile - -t = tarfile.TarFile("${cached_image_file}") -meta_str = t.extractfile('repositories').read() -meta = json.loads(meta_str) -tag, tag_value = meta.items()[0] -rev, rev_value = tag_value.items()[0] -cmd = "${images_cmd}" -proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) -stdo, stde = proc.communicate() -found = False -for line in stdo.split("\n"): - tmp = re.split(r'\s+', line) - if tmp[0] == tag and tmp[1] == rev and tmp[2] == rev_value: - found = True -if not found: - print("Loading image") - cmd = "cat ${cached_image_file} | ${load_cmd}" - subprocess.check_call(cmd, shell=True) -EOF -''' - class ContainerFinder(object): @@ -75,7 +40,7 @@ class ContainerFinder(object): self.app_info = app_info self.container_registry = ContainerRegistry(app_info) - def __enabled_container_types(self, destination_info): + def _enabled_container_types(self, destination_info): return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)] def find_best_container_description(self, enabled_container_types, tool_info): @@ -86,7 +51,7 @@ class ContainerFinder(object): return container_description def find_container(self, tool_info, destination_info, job_info): - enabled_container_types = self.__enabled_container_types(destination_info) + enabled_container_types = self._enabled_container_types(destination_info) # Short-cut everything else and just skip checks if no container type is enabled. if not enabled_container_types: @@ -236,13 +201,15 @@ class ContainerRegistry(object): def __default_containers_resolvers(self): default_resolvers = [ ExplicitContainerResolver(self.app_info), + ExplicitSingularityContainerResolver(self.app_info), ] if self.enable_beta_mulled_containers: default_resolvers.extend([ CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + CachedMulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), MulledDockerContainerResolver(self.app_info, namespace="biocontainers"), + MulledSingularityContainerResolver(self.app_info, namespace="biocontainers"), BuildMulledDockerContainerResolver(self.app_info), - CachedMulledSingularityContainerResolver(self.app_info), BuildMulledSingularityContainerResolver(self.app_info), ]) return default_resolvers @@ -264,343 +231,3 @@ class ContainerRegistry(object): return container_description return None - - -class AppInfo(object): - - def __init__( - self, - galaxy_root_dir=None, - default_file_path=None, - outputs_to_working_directory=False, - container_image_cache_path=None, - library_import_dir=None, - enable_beta_mulled_containers=False, - containers_resolvers_config_file=None, - involucro_path=None, - involucro_auto_init=True, - ): - self.galaxy_root_dir = galaxy_root_dir - self.default_file_path = default_file_path - # TODO: Vary default value for docker_volumes based on this... - self.outputs_to_working_directory = outputs_to_working_directory - self.container_image_cache_path = container_image_cache_path - self.library_import_dir = library_import_dir - self.enable_beta_mulled_containers = enable_beta_mulled_containers - self.containers_resolvers_config_file = containers_resolvers_config_file - self.involucro_path = involucro_path - self.involucro_auto_init = involucro_auto_init - - -class ToolInfo(object): - # TODO: Introduce tool XML syntax to annotate the optional environment - # variables they can consume (e.g. JVM options, license keys, etc..) - # and add these to env_path_through - - def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]): - self.container_descriptions = container_descriptions - self.requirements = requirements - self.requires_galaxy_python_environment = requires_galaxy_python_environment - self.env_pass_through = env_pass_through - - -class JobInfo(object): - - def __init__( - self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type - ): - self.working_directory = working_directory - self.job_directory = job_directory - # Tool files may be remote staged - so this is unintuitively a property - # of the job not of the tool. - self.tool_directory = tool_directory - self.tmp_directory = tmp_directory - self.job_directory_type = job_directory_type # "galaxy" or "pulsar" - - -@six.add_metaclass(ABCMeta) -class Container(object): - - def __init__(self, container_id, app_info, tool_info, destination_info, job_info, container_description): - self.container_id = container_id - self.app_info = app_info - self.tool_info = tool_info - self.destination_info = destination_info - self.job_info = job_info - self.container_description = container_description - - @property - def resolve_dependencies(self): - return DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES if not self.container_description else self.container_description.resolve_dependencies - - @property - def shell(self): - return DEFAULT_CONTAINER_SHELL if not self.container_description else self.container_description.shell - - @abstractmethod - def containerize_command(self, command): - """ - Use destination supplied container configuration parameters, - container_id, and command to build a new command that runs - input command in container. - """ - - -def preprocess_volumes(volumes_raw_str, container_type): - """Process Galaxy volume specification string to either Docker or Singularity specification. - - Galaxy allows the mount try "default_ro" which translates to ro for Docker and - ro for Singularity iff no subdirectories are rw (Singularity does not allow ro - parent directories with rw subdirectories). - - >>> preprocess_volumes("/a/b", DOCKER_CONTAINER_TYPE) - ['/a/b:rw'] - >>> preprocess_volumes("/a/b:ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) - ['/a/b:ro', '/a/b/c:rw'] - >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", DOCKER_CONTAINER_TYPE) - ['/a/b:ro', '/a/b/c:rw'] - >>> preprocess_volumes("/a/b:default_ro,/a/b/c:rw", SINGULARITY_CONTAINER_TYPE) - ['/a/b:rw', '/a/b/c:rw'] - """ - - volumes_raw_strs = [v.strip() for v in volumes_raw_str.split(",")] - volumes = [] - rw_paths = [] - - for volume_raw_str in volumes_raw_strs: - volume_parts = volume_raw_str.split(":") - if len(volume_parts) > 2: - raise Exception("Unparsable volumes string in configuration [%s]" % volumes_raw_str) - if len(volume_parts) == 1: - volume_parts.append("rw") - volumes.append(volume_parts) - if volume_parts[1] == "rw": - rw_paths.append(volume_parts[0]) - - for volume in volumes: - path = volume[0] - how = volume[1] - - if how == "default_ro": - how = "ro" - if container_type == SINGULARITY_CONTAINER_TYPE: - for rw_path in rw_paths: - if in_directory(rw_path, path): - how = "rw" - - volume[1] = how - - return [":".join(v) for v in volumes] - - -class HasDockerLikeVolumes(object): - """Mixin to share functionality related to Docker volume handling. - - Singularity seems to have a fairly compatible syntax for volume handling. - """ - - def _expand_volume_str(self, value): - if not value: - return value - - template = string.Template(value) - variables = dict() - - def add_var(name, value): - if value: - if not value.startswith("$"): - value = os.path.abspath(value) - variables[name] = value - - add_var("working_directory", self.job_info.working_directory) - add_var("tmp_directory", self.job_info.tmp_directory) - add_var("job_directory", self.job_info.job_directory) - add_var("tool_directory", self.job_info.tool_directory) - add_var("galaxy_root", self.app_info.galaxy_root_dir) - add_var("default_file_path", self.app_info.default_file_path) - add_var("library_import_dir", self.app_info.library_import_dir) - - if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar": - # We have a Pulsar job directory, so everything needed (excluding index - # files) should be available in job_directory... - defaults = "$job_directory:default_ro,$tool_directory:default_ro,$job_directory/outputs:rw,$working_directory:rw" - else: - defaults = "$galaxy_root:default_ro,$tool_directory:default_ro" - if self.job_info.job_directory: - defaults += ",$job_directory:default_ro" - if self.job_info.tmp_directory is not None: - defaults += ",$tmp_directory:rw" - if self.app_info.outputs_to_working_directory: - # Should need default_file_path (which is of course an estimate given - # object stores anyway). - defaults += ",$working_directory:rw,$default_file_path:default_ro" - else: - defaults += ",$working_directory:rw,$default_file_path:rw" - - if self.app_info.library_import_dir: - defaults += ",$library_import_dir:default_ro" - - # Define $defaults that can easily be extended with external library and - # index data without deployer worrying about above details. - variables["defaults"] = string.Template(defaults).safe_substitute(variables) - - return template.safe_substitute(variables) - - -class DockerContainer(Container, HasDockerLikeVolumes): - - container_type = DOCKER_CONTAINER_TYPE - - def containerize_command(self, command): - def prop(name, default): - destination_name = "docker_%s" % name - return self.destination_info.get(destination_name, default) - - env_directives = [] - for pass_through_var in self.tool_info.env_pass_through: - env_directives.append('"%s=$%s"' % (pass_through_var, pass_through_var)) - - # Allow destinations to explicitly set environment variables just for - # docker container. Better approach is to set for destination and then - # pass through only what tool needs however. (See todo in ToolInfo.) - for key, value in six.iteritems(self.destination_info): - if key.startswith("docker_env_"): - env = key[len("docker_env_"):] - env_directives.append('"%s=%s"' % (env, value)) - - working_directory = self.job_info.working_directory - if not working_directory: - raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) - - volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults")) - preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) - # TODO: Remove redundant volumes... - volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] - # If a tool definitely has a temp directory available set it to /tmp in container for compat. - # with CWL. This is part of that spec and should make it easier to share containers between CWL - # and Galaxy. - if self.job_info.tmp_directory is not None: - volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory)) - volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM) - - docker_host_props = dict( - docker_cmd=prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND), - sudo=asbool(prop("sudo", docker_util.DEFAULT_SUDO)), - sudo_cmd=prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND), - host=prop("host", docker_util.DEFAULT_HOST), - ) - - cached_image_file = self.__get_cached_image_file() - if not cached_image_file: - # TODO: Add option to cache it once here and create cached_image_file. - cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) - else: - cache_command = self.__cache_from_file_command(cached_image_file, docker_host_props) - run_command = docker_util.build_docker_run_command( - command, - self.container_id, - volumes=volumes, - volumes_from=volumes_from, - env_directives=env_directives, - working_directory=working_directory, - net=prop("net", "none"), # By default, docker instance has networking disabled - auto_rm=asbool(prop("auto_rm", docker_util.DEFAULT_AUTO_REMOVE)), - set_user=prop("set_user", docker_util.DEFAULT_SET_USER), - run_extra_arguments=prop("run_extra_arguments", docker_util.DEFAULT_RUN_EXTRA_ARGUMENTS), - **docker_host_props - ) - return "%s\n%s" % (cache_command, run_command) - - def __cache_from_file_command(self, cached_image_file, docker_host_props): - images_cmd = docker_util.build_docker_images_command(truncate=False, **docker_host_props) - load_cmd = docker_util.build_docker_load_command(**docker_host_props) - - return string.Template(LOAD_CACHED_IMAGE_COMMAND_TEMPLATE).safe_substitute( - cached_image_file=cached_image_file, - images_cmd=images_cmd, - load_cmd=load_cmd - ) - - def __get_cached_image_file(self): - container_id = self.container_id - cache_directory = os.path.abspath(self.__get_destination_overridable_property("container_image_cache_path")) - cache_path = docker_cache_path(cache_directory, container_id) - return cache_path if os.path.exists(cache_path) else None - - def __get_destination_overridable_property(self, name): - prop_name = "docker_%s" % name - if prop_name in self.destination_info: - return self.destination_info[prop_name] - else: - return getattr(self.app_info, name) - - -def docker_cache_path(cache_directory, container_id): - file_container_id = container_id.replace("/", "_slash_") - cache_file_name = "docker_%s.tar" % file_container_id - return os.path.join(cache_directory, cache_file_name) - - -class SingularityContainer(Container, HasDockerLikeVolumes): - - container_type = SINGULARITY_CONTAINER_TYPE - - def containerize_command(self, command): - def prop(name, default): - destination_name = "singularity_%s" % name - return self.destination_info.get(destination_name, default) - - env = [] - for pass_through_var in self.tool_info.env_pass_through: - env.append((pass_through_var, "$%s" % pass_through_var)) - - # Allow destinations to explicitly set environment variables just for - # docker container. Better approach is to set for destination and then - # pass through only what tool needs however. (See todo in ToolInfo.) - for key, value in six.iteritems(self.destination_info): - if key.startswith("singularity_env_"): - real_key = key[len("singularity_env_"):] - env.append((real_key, value)) - - working_directory = self.job_info.working_directory - if not working_directory: - raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory) - - volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults")) - preprocessed_volumes_list = preprocess_volumes(volumes_raw, self.container_type) - volumes = [DockerVolume.from_str(v) for v in preprocessed_volumes_list] - - singularity_target_kwds = dict( - singularity_cmd=prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND), - sudo=asbool(prop("sudo", singularity_util.DEFAULT_SUDO)), - sudo_cmd=prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND), - ) - run_command = singularity_util.build_singularity_run_command( - command, - self.container_id, - volumes=volumes, - env=env, - working_directory=working_directory, - run_extra_arguments=prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS), - **singularity_target_kwds - ) - return run_command - - -CONTAINER_CLASSES = dict( - docker=DockerContainer, - singularity=SingularityContainer, -) - - -class NullContainer(object): - - def __init__(self): - pass - - def __bool__(self): - return False - __nonzero__ = __bool__ - - -NULL_CONTAINER = NullContainer() diff --git a/lib/galaxy/tools/deps/dependencies.py b/lib/galaxy/tools/deps/dependencies.py index 35aafcfb329..194cfab1f5e 100644 --- a/lib/galaxy/tools/deps/dependencies.py +++ b/lib/galaxy/tools/deps/dependencies.py @@ -1,5 +1,60 @@ from galaxy.tools.deps.requirements import ToolRequirements from galaxy.util import bunch +from .mulled.mulled_build import DEFAULT_CHANNELS + + +class AppInfo(object): + + def __init__( + self, + galaxy_root_dir=None, + default_file_path=None, + outputs_to_working_directory=False, + container_image_cache_path=None, + library_import_dir=None, + enable_beta_mulled_containers=False, + containers_resolvers_config_file=None, + involucro_path=None, + involucro_auto_init=True, + mulled_channels=DEFAULT_CHANNELS, + ): + self.galaxy_root_dir = galaxy_root_dir + self.default_file_path = default_file_path + # TODO: Vary default value for docker_volumes based on this... + self.outputs_to_working_directory = outputs_to_working_directory + self.container_image_cache_path = container_image_cache_path + self.library_import_dir = library_import_dir + self.enable_beta_mulled_containers = enable_beta_mulled_containers + self.containers_resolvers_config_file = containers_resolvers_config_file + self.involucro_path = involucro_path + self.involucro_auto_init = involucro_auto_init + self.mulled_channels = mulled_channels + + +class ToolInfo(object): + # TODO: Introduce tool XML syntax to annotate the optional environment + # variables they can consume (e.g. JVM options, license keys, etc..) + # and add these to env_path_through + + def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]): + self.container_descriptions = container_descriptions + self.requirements = requirements + self.requires_galaxy_python_environment = requires_galaxy_python_environment + self.env_pass_through = env_pass_through + + +class JobInfo(object): + + def __init__( + self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type + ): + self.working_directory = working_directory + self.job_directory = job_directory + # Tool files may be remote staged - so this is unintuitively a property + # of the job not of the tool. + self.tool_directory = tool_directory + self.tmp_directory = tmp_directory + self.job_directory_type = job_directory_type # "galaxy" or "pulsar" class DependenciesDescription(object): diff --git a/lib/galaxy/tools/deps/dockerfiles.py b/lib/galaxy/tools/deps/dockerfiles.py index bb0a292235c..67944dcbc04 100644 --- a/lib/galaxy/tools/deps/dockerfiles.py +++ b/lib/galaxy/tools/deps/dockerfiles.py @@ -3,7 +3,7 @@ import os from ..deps import commands from ..deps import docker_util -from ..deps.containers import docker_cache_path +from ..deps.container_classes import docker_cache_path from ..deps.requirements import parse_requirements_from_xml from ...tools import loader_directory diff --git a/lib/galaxy/tools/deps/mulled/invfile.lua b/lib/galaxy/tools/deps/mulled/invfile.lua index 4b771e4b8ec..6c340647fb3 100644 --- a/lib/galaxy/tools/deps/mulled/invfile.lua +++ b/lib/galaxy/tools/deps/mulled/invfile.lua @@ -48,7 +48,7 @@ end local singularity_image = VAR.SINGULARITY_IMAGE if singularity_image == '' then - singularity_image = 'quay.io/biocontainers/singularity:2.3--0' + singularity_image = 'quay.io/biocontainers/singularity:2.4.6--0' end local singularity_image_dir = VAR.SINGULARITY_IMAGE_DIR @@ -103,10 +103,7 @@ if VAR.SINGULARITY ~= '' then .using(singularity_image) .withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true}) .withConfig({entrypoint = {'/bin/sh', '-c'}}) - -- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size - .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi") - .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) - .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity') + .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity build /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity.def') .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME) end diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py index 15aa0f2affc..c3c84f5a586 100644 --- a/lib/galaxy/tools/deps/mulled/mulled_build.py +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -68,7 +68,7 @@ From: bgruening/busybox-bash:0.1 def involucro_link(): if IS_OS_X: - url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION + url = "https://github.com/mvdbeek/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION else: url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION return url @@ -236,7 +236,7 @@ def mull_targets( if singularity: if not os.path.exists(singularity_image_dir): safe_makedirs(singularity_image_dir) - with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def: + with open(os.path.join(singularity_image_dir, 'Singularity.def'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % {'container_test': test} sin_def.write(fill_template) with PrintProgress(): @@ -275,12 +275,16 @@ class InvolucroContext(installable.InstallableContext): def exec_command(self, involucro_args): cmd = self.build_command(involucro_args) # Create ./build dir manually, otherwise Docker will do it as root - os.mkdir('./build') + created_build_dir = False + if not os.path.exists('build'): + created_build_dir = True + os.mkdir('./build') try: res = self.shell_exec(" ".join(cmd)) finally: # delete build directory in any case - shutil.rmtree('./build') + if created_build_dir: + shutil.rmtree('./build') return res def is_installed(self): diff --git a/lib/galaxy/tools/deps/mulled/util.py b/lib/galaxy/tools/deps/mulled/util.py index eb7c0f18e9b..f394a907278 100644 --- a/lib/galaxy/tools/deps/mulled/util.py +++ b/lib/galaxy/tools/deps/mulled/util.py @@ -13,6 +13,8 @@ try: except ImportError: requests = None +MULLED_TAG_CACHE = collections.defaultdict(dict) + def create_repository(namespace, repo_name, oauth_token): assert oauth_token @@ -56,7 +58,19 @@ def mulled_tags_for(namespace, image, tag_prefix=None): The result will be sorted so newest tags are first. """ - tags = quay_versions(namespace, image) + tags_cached = False + if namespace in MULLED_TAG_CACHE: + if image in MULLED_TAG_CACHE[namespace]: + tags, last_checked = MULLED_TAG_CACHE[namespace][image] + if not tags and time.time() - last_checked < 300: + # it's possible we haven't seen the tags before, we check every 5 minutes + tags_cached = False + else: + tags_cached = True + if not tags_cached: + tags = quay_versions(namespace, image) + last_checked = time.time() + MULLED_TAG_CACHE[namespace][image] = (tags, last_checked) if tag_prefix is not None: tags = [t for t in tags if t.startswith(tag_prefix)] tags = version_sorted(tags) diff --git a/lib/galaxy/tools/deps/requirements.py b/lib/galaxy/tools/deps/requirements.py index ea92b7f49b2..8df34a96d11 100644 --- a/lib/galaxy/tools/deps/requirements.py +++ b/lib/galaxy/tools/deps/requirements.py @@ -240,7 +240,7 @@ def parse_requirements_from_xml(xml_root): if requirements_elem is not None: container_elems = requirements_elem.findall('container') - containers = map(container_from_element, container_elems) + containers = [container_from_element(c) for c in container_elems] return requirements, containers diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index beca0dcedf1..7fd9099acde 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -257,6 +257,33 @@ class Dependency(Dictifiable): return "Using dependency %s version %s of type %s" % (self.name, self.version, self.dependency_type) +class ContainerDependency(Dependency): + + dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path'] + + def __init__(self, container_description, name, version): + self.dependency_type = container_description.type + self._name = name + self._version = version + self.environment_path = container_description.identifier + + @property + def name(self): + return self._name + + @property + def version(self): + return self._version + + @property + def exact(self): + return True + + @property + def shell_commands(self): + return None + + class NullDependency(Dependency): dependency_type = None exact = True diff --git a/lib/galaxy/tools/deps/singularity_util.py b/lib/galaxy/tools/deps/singularity_util.py index f9ac33a551f..ba281f8880f 100644 --- a/lib/galaxy/tools/deps/singularity_util.py +++ b/lib/galaxy/tools/deps/singularity_util.py @@ -1,5 +1,6 @@ -from six.moves import shlex_quote +import os +from six.moves import shlex_quote DEFAULT_WORKING_DIRECTORY = None DEFAULT_SINGULARITY_COMMAND = "singularity" @@ -8,6 +9,27 @@ DEFAULT_SUDO_COMMAND = "sudo" DEFAULT_RUN_EXTRA_ARGUMENTS = None +def pull_mulled_singularity_command(docker_image_identifier, + cache_directory, + namespace=None, + singularity_cmd=DEFAULT_SINGULARITY_COMMAND, + sudo=DEFAULT_SUDO, + sudo_cmd=DEFAULT_SUDO_COMMAND): + command_parts = [] + command_parts += _singularity_prefix( + singularity_cmd=singularity_cmd, + sudo=sudo, + sudo_cmd=sudo_cmd, + ) + save_path = docker_image_identifier + if namespace: + prefix = "docker://quay.io/%s/" % namespace + if docker_image_identifier.startswith(prefix): + save_path = docker_image_identifier[len(prefix):] + command_parts.extend(["build", os.path.join(cache_directory, save_path), docker_image_identifier]) + return command_parts + + def build_singularity_run_command( container_command, image, @@ -21,7 +43,10 @@ def build_singularity_run_command( ): command_parts = [] # http://singularity.lbl.gov/docs-environment-metadata + home = None for (key, value) in env: + if key == 'HOME': + home = value command_parts.extend(["SINGULARITYENV_%s=%s" % (key, value)]) command_parts += _singularity_prefix( singularity_cmd=singularity_cmd, @@ -34,6 +59,8 @@ def build_singularity_run_command( command_parts.extend(["-B", shlex_quote(str(volume))]) if working_directory: command_parts.extend(["--pwd", shlex_quote(working_directory)]) + if home is not None: + command_parts.extend(["--home", "%s:%s" % (home, home)]) if run_extra_arguments: command_parts.append(run_extra_arguments) full_image = image @@ -56,4 +83,4 @@ def _singularity_prefix( return command_parts -__all__ = ("build_singularity_run_command",) +__all__ = ("build_singularity_run_command", "pull_mulled_singularity_command") diff --git a/lib/galaxy/tools/deps/views.py b/lib/galaxy/tools/deps/views.py index b1f3af0bab1..a6f41b84606 100644 --- a/lib/galaxy/tools/deps/views.py +++ b/lib/galaxy/tools/deps/views.py @@ -49,10 +49,12 @@ class DependencyResolversView(object): kwds = {'install': False, 'return_null': True, 'installed_tool_dependencies': installed_tool_dependencies} - dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()} + dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, + **kwds) + for tool, requirements in tool_requirements_d.items()} return dependencies_per_tool - def uninstall_dependencies(self, index=None, **payload): + def uninstall_dependencies(self, index=None, resolver_type=None, **payload): """Attempt to uninstall requirements. Returns 0 if successfull, else None.""" requirements = payload.get('requirements') if not requirements: @@ -61,6 +63,10 @@ class DependencyResolversView(object): resolver = self._dependency_resolvers[index] if resolver.can_uninstall_dependencies: return resolver.uninstall(requirements) + elif resolver_type: + for resolver in self._dependency_resolvers: + if resolver.resolver_type == resolver_type and resolver.can_uninstall_dependencies: + return resolver.uninstall(requirements) else: for index in self.uninstallable_resolvers: return_code = self._dependency_resolvers[index].uninstall(requirements) @@ -97,8 +103,8 @@ class DependencyResolversView(object): envs_to_remove = envs_to_remove.difference(can_remove) return list(removed_environments) - def install_dependencies(self, requirements): - return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True}) + def install_dependencies(self, requirements, **kwds): + return self._dependency_manager._requirements_to_dependencies_dict(requirements, **kwds) def install_dependency(self, index=None, **payload): """ diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index c439f0d2041..519af7bfb6e 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -234,7 +234,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): Return the resolver status for a specific tool id. [{"status": "installed", "name": "hisat2", "versionless": false, "resolver_type": "conda", "version": "2.0.3", "type": "package"}] """ - tool = self._get_tool(id) + tool = self._get_tool(id, user=trans.user) return tool.tool_requirements_status @expose_api @@ -249,11 +249,15 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): Attempts to install requirements via the dependency resolver parameters: + index: index of dependency resolver to use when installing dependency. + Defaults to using the highest ranking resolver + resolver_type: Use the dependency resolver of this resolver_type to install dependency. build_dependency_cache: If true, attempts to cache dependencies for this tool force_rebuild: If true and cache dir exists, attempts to delete cache dir """ - tool = self._get_tool(id) - tool._view.install_dependencies(tool.requirements) + tool = self._get_tool(id, user=trans.user) + kwds['install'] = True + tool._view.install_dependencies(tool.requirements, **kwds) if kwds.get('build_dependency_cache'): tool.build_dependency_cache(**kwds) # TODO: rework resolver install system to log and report what has been done. @@ -267,9 +271,13 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): DELETE /api/tools/{tool_id}/dependencies Attempts to uninstall requirements via the dependency resolver + parameters: + index: index of dependency resolver to use when installing dependency. + Defaults to using the highest ranking resolver + resolver_type: Use the dependency resolver of this resolver_type to install dependency """ - tool = self._get_tool(id) - tool._view.uninstall_dependencies(index=None, requirements=tool.requirements) + tool = self._get_tool(id, user=trans.user) + tool._view.uninstall_dependencies(requirements=tool.requirements, **kwds) # TODO: rework resolver install system to log and report what has been done. return tool.tool_requirements_status diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index cc395eaff70..85bf0fc663f 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -414,7 +414,7 @@ mapping: default: database/dependencies/involucro required: false desc: | - involucro is a tool used to build Docker containers for tools from Conda + involucro is a tool used to build Docker or Singularity containers for tools from Conda dependencies referenced in tools as `requirement`s. The following path is the location of involucro on the Galaxy host. This is ignored if the relevant container resolver isn't enabled, and will install on demand unless @@ -425,9 +425,16 @@ mapping: default: true required: false desc: | - Install involucro as needed to build Docker containers for tools. Ignored if + Install involucro as needed to build Docker or Singularity containers for tools. Ignored if relevant container resolver is not used. + mulled_channels: + type: str + default: 'conda-forge,bioconda' + required: false + desc: | + Conda channels to use when building Docker or Singularity containers using involucro. + enable_tool_shed_check: type: bool default: false diff --git a/run_tests.sh b/run_tests.sh index 676bf7c1d50..6b030fdd597 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -270,7 +270,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='galaxy/testing-base:18.09.0' +DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.01.3' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" @@ -295,20 +295,12 @@ then else db_type="sqlite" fi - if [ "$1" = "--external_tmp" ]; then - # If /tmp is a tmpfs there may be better performance by reusing - # the parent's temp file system. Also, it seems to decrease the - # frequency or errors such as the following: - # /bin/sh: 1: /tmp/tmpiWU3kJ/tmp_8zLxx/job_working_directory_mwwDmg/000/274/galaxy_274.sh: Text file busy - tmp=$(mktemp -d) - chmod 1777 $tmp - DOCKER_RUN_EXTRA_ARGS="-v ${tmp}:/tmp ${DOCKER_RUN_EXTRA_ARGS}" - shift - fi MY_UID=$(id -u) # Skip client build process in the Docker container for all tests, the Jenkins task builds the client # locally before testing - you will need to do this also if using this script for Selenium testing. DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" + echo "Docker version:" + docker --version echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS \ -e "BUILD_NUMBER=$BUILD_NUMBER" \ diff --git a/test/base/integration_util.py b/test/base/integration_util.py index 54b166bfe9a..185fb3c26cd 100644 --- a/test/base/integration_util.py +++ b/test/base/integration_util.py @@ -33,6 +33,10 @@ def skip_unless_docker(): return skip_unless_executable("docker") +def skip_unless_singularity(): + return skip_unless_executable("singularity") + + class IntegrationTestCase(TestCase, UsesApiTestCaseMixin): """Unit test case with utilities for spinning up Galaxy.""" diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index ac85272d5bb..2399868d3ff 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -21,27 +21,18 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ curl -sL https://deb.nodesource.com/setup_8.x | bash - && \ - apt-key adv --keyserver pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5 && \ - echo "deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-${MYSQL_MAJOR}" > /etc/apt/sources.list.d/mysql.list && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ - { \ - echo mysql-community-server mysql-community-server/data-dir select ''; \ - echo mysql-community-server mysql-community-server/root-pass password ''; \ - echo mysql-community-server mysql-community-server/re-root-pass password ''; \ - echo mysql-community-server mysql-community-server/remove-test-db select false; \ - } | debconf-set-selections && \ apt-get update -y && \ - apt-get install -y libpq-dev postgresql postgresql-client \ - ansible wget mysql-server="${MYSQL_MAJOR}"* libmysqlclient-dev="${MYSQL_MAJOR}"* \ + apt-get install -y libpq-dev postgresql postgresql-client ansible wget \ slurm-llnl libmunge-dev slurm-drmaa-dev ant cmake curl \ g++ gcc gfortran git-core libffi-dev liblapack-dev \ libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \ - mercurial nginx-extras patch postgresql postgresql \ - postgresql-client python-boto python-dev python3-dev \ + patch postgresql postgresql \ + postgresql-client python-dev python3-dev \ python-prettytable python-psycopg2 python-virtualenv python-pip \ rsync slurm-drmaa-dev swig sysstat unzip \ autoconf automake build-essential libatlas-base-dev libblas-dev openssl \ - bzip2 \ + bzip2 uuid-dev libssl-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \ ca-certificates \ openjdk-8-jre-headless \ tzdata \ @@ -53,14 +44,14 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans libnss3 libgconf-2-4 && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN npm install -g grunt grunt-cli +# Install golang +RUN export VERSION=1.11 OS=linux ARCH=amd64 && curl -sL https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz \ + | tar -C /usr/local -xzf - RUN mkdir -p /tmp/ansible && \ mkdir -p /opt/galaxy/db && \ - chown -R postgres:postgres /opt/galaxy/db && \ - sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf + chown -R postgres:postgres /opt/galaxy/db -ADD start_mysql.sh /opt/galaxy/start_mysql.sh ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml ADD provision.yml /tmp/ansible/provision.yml @@ -93,7 +84,6 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \ export GALAXY_VIRTUAL_ENV=$VENV && \ . $GALAXY_VIRTUAL_ENV/bin/activate && \ pip install psycopg2; done && \ - pip install mysql && \ cd $GALAXY_ROOT && \ echo "Prepopulating postgres database" && \ su -c '/usr/lib/postgresql/${POSTGRES_MAJOR}/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ @@ -101,15 +91,10 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \ GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/galaxy" bash create_db.sh && \ echo "Prepopulating sqlite database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" bash create_db.sh && \ - sh /opt/galaxy/start_mysql.sh && \ - echo "Prepopulating mysql database" && \ - GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" bash create_db.sh && \ echo "Prepopulating toolshed postgres database" && \ TOOL_SHED_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" bash create_db.sh tool_shed && \ echo "Prepopulating toolshed sqlite database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed && \ - echo "Prepopulating toolshed mysql database" && \ - TOOL_SHED_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" bash create_db.sh tool_shed + TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed # bcftools for Galaxy. RUN mkdir -p /tmp/install && \ @@ -141,6 +126,18 @@ RUN sudo mkdir -p /opt/selenium \ && wget --no-verbose https://selenium-release.storage.googleapis.com/3.6/selenium-server-standalone-3.6.0.jar \ -O /opt/selenium/selenium-server-standalone.jar +USER seluser + +RUN export GOPATH=${HOME}/go && export PATH=/usr/local/go/bin:${PATH}:${GOPATH}/bin && \ + mkdir -p $GOPATH/src/github.com/sylabs && \ + cd $GOPATH/src/github.com/sylabs && \ + git clone https://github.com/sylabs/singularity.git && \ + cd singularity && \ + git checkout v3.0.1 && \ + ./mconfig && \ + cd builddir && make && sudo make install && \ + rm -Rf ${HOME}/go + USER root #============================== diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index 6d2c3c70368..26b6de3df3a 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -16,9 +16,10 @@ echo "Creating galaxy group with gid $GALAXY_TEST_UID - it may already exist and groupadd -r galaxy -g "$GALAXY_TEST_UID" | true echo "Creating galaxy user with uid $GALAXY_TEST_UID - it may already exist and this is fine." useradd -u $GALAXY_TEST_UID -r -g galaxy -d /home/galaxy -c "Galaxy User" galaxy -s /bin/bash | true +mkdir -p /home/galaxy/.singularity echo "Setting galaxy user password - the operation may fail." echo "galaxy:galaxy" | chpasswd | true -chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" +chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "${GALAXY_VIRTUAL_ENV:-/galaxy_venv}" /home/galaxy/ : ${GALAXY_TEST_DATABASE_TYPE:-"postgres"} if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; @@ -30,12 +31,6 @@ then GALAXY_TEST_DBURI="postgres://root@localhost:5930/galaxy?client_encoding=utf8" TOOL_SHED_TEST_DBURI="postgres://root@localhost:5930/toolshed?client_encoding=utf8" export GALAXY_CONFIG_OVERRIDE_DATABASE_ENCODING="UTF-8" -elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ]; -then - sh /opt/galaxy/start_mysql.sh - GALAXY_TEST_INSTALL_DB_MERGED="true" - GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" - TOOL_SHED_TEST_DBURI="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ]; then GALAXY_TEST_INSTALL_DB_MERGED="true" @@ -54,7 +49,8 @@ cd /galaxy : ${GALAXY_VIRTUAL_ENV:=.venv} -HOME=/galaxy +HOME=/home/galaxy + echo "Running common startup for updated dependencies (if any)" sudo -E -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh --dev-wheels || { echo "common_startup.sh failed"; exit 1; } diff --git a/test/docker/base/start_mysql.sh b/test/docker/base/start_mysql.sh deleted file mode 100644 index cd378a056fe..00000000000 --- a/test/docker/base/start_mysql.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e - -MYSQL_USER=galaxy -MYSQL_PASSWORD=galaxy -MYSQL_DATABASE=galaxy -SHED_MYSQL_DATABASE=toolshed - -mkdir -p /var/lib/mysql -chown -R mysql:mysql /var/lib/mysql - -# Derived from -# https://github.com/docker-library/mysql/blob/master/5.7/docker-entrypoint.sh - -DATADIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')" - -tempSqlFile='/tmp/mysql-first-time.sql' -cat > "$tempSqlFile" <<-EOSQL - DELETE FROM mysql.user ; - CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; - GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ; - DROP DATABASE IF EXISTS test ; -EOSQL - -if [ "$MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" -fi - -if [ "$SHED_MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$SHED_MYSQL_DATABASE\` ;" >> "$tempSqlFile" -fi - -if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then - echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" - - if [ "$MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" - fi - if [ "$SHED_MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$SHED_MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" - fi -fi - -echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" - -mysqld_safe --init-file="$tempSqlFile" & -sleep 5 \ No newline at end of file diff --git a/test/functional/tools/job_environment_default.xml b/test/functional/tools/job_environment_default.xml index e8268469862..22c884657f1 100644 --- a/test/functional/tools/job_environment_default.xml +++ b/test/functional/tools/job_environment_default.xml @@ -2,7 +2,7 @@ busybox:ubuntu-14.04 - '$user_id'; echo `id -g` > '$group_id'; echo `pwd` > '$pwd'; diff --git a/test/functional/tools/job_environment_default_legacy.xml b/test/functional/tools/job_environment_default_legacy.xml index 07412ee774a..a00a661ebca 100644 --- a/test/functional/tools/job_environment_default_legacy.xml +++ b/test/functional/tools/job_environment_default_legacy.xml @@ -2,7 +2,7 @@ busybox:ubuntu-14.04 - '$user_id'; echo `id -g` > '$group_id'; echo `pwd` > '$pwd'; diff --git a/test/integration/singularity_job_conf.xml b/test/integration/singularity_job_conf.xml new file mode 100644 index 00000000000..8743b71ef7e --- /dev/null +++ b/test/integration/singularity_job_conf.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + true + + + + + + + + + + + + diff --git a/test/integration/test_dockerized_jobs.py b/test/integration/test_containerized_jobs.py similarity index 57% rename from test/integration/test_dockerized_jobs.py rename to test/integration/test_containerized_jobs.py index 43e56492c00..49e30430d54 100644 --- a/test/integration/test_dockerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -1,32 +1,47 @@ """Integration tests for running tools in Docker containers.""" import os +import unittest from base import integration_util from base.populators import ( DatasetPopulator, ) +from galaxy.tools.deps.commands import which from .test_job_environments import RunsEnvironmentJobs SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml") +SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.xml") +EXTENDED_TIMEOUT = 120 -@integration_util.skip_unless_docker() class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs): framework_tool_and_types = True + job_config_file = DOCKERIZED_JOB_CONFIG_FILE + build_mulled_resolver = 'build_mulled' + container_type = 'docker' + default_container_home_dir = '/' @classmethod def handle_galaxy_config_kwds(cls, config): cls.jobs_directory = cls._test_driver.mkdtemp() config["jobs_directory"] = cls.jobs_directory - config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE + config["job_config_file"] = cls.job_config_file # Disable tool dependency resolution. config["tool_dependency_dir"] = "none" + config["conda_auto_init"] = False + config["conda_auto_install"] = False config["enable_beta_mulled_containers"] = "true" + @classmethod + def setUpClass(cls): + if not which(cls.container_type): + raise unittest.SkipTest("Executable '%s' not found on PATH" % cls.container_type) + super(DockerizedJobsIntegrationTestCase, cls).setUpClass() + def setUp(self): super(DockerizedJobsIntegrationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) @@ -35,16 +50,16 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru def test_explicit(self): self.dataset_populator.run_tool("mulled_example_explicit", {}, self.history_id) self.dataset_populator.wait_for_history(self.history_id, assert_ok=True) - output = self.dataset_populator.get_history_dataset_content(self.history_id) + output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT) assert "0.7.15-r1140" in output def test_mulled_simple(self): self.dataset_populator.run_tool("mulled_example_simple", {}, self.history_id) self.dataset_populator.wait_for_history(self.history_id, assert_ok=True) - output = self.dataset_populator.get_history_dataset_content(self.history_id) + output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT) assert "0.7.15-r1140" in output - def test_docker_job_enviornment(self): + def test_container_job_enviornment(self): job_env = self._run_and_get_environment_properties("job_environment_default") euid = os.geteuid() @@ -57,7 +72,7 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru assert job_env.home.startswith(self.jobs_directory) assert job_env.home.endswith("/home") - def test_docker_job_environment_legacy(self): + def test_container_job_environment_legacy(self): job_env = self._run_and_get_environment_properties("job_environment_default_legacy") euid = os.geteuid() @@ -69,4 +84,25 @@ class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, Ru assert job_env.pwd.endswith("/working") # Should we change env_pass_through to just always include TMP and HOME for docker? # I'm not sure, if yes this would change. - assert job_env.home == "/", job_env.home + assert job_env.home == self.default_container_home_dir, job_env.home + + def test_build_mulled(self): + if not which('docker'): + raise unittest.SkipTest("Docker not found on PATH, required for building images via involucro") + resolver_type = self.build_mulled_resolver + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/dependencies" % tool_id + data = {'id': tool_id, 'resolver_type': resolver_type} + create_response = self._post(endpoint, data=data, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert any([True for d in response if d['dependency_type'] == self.container_type]) + + +class SingularityJobsIntegrationTestCase(DockerizedJobsIntegrationTestCase): + + job_config_file = SINGULARITY_JOB_CONFIG_FILE + build_mulled_resolver = 'build_mulled_singularity' + container_type = 'singularity' + # singularity passes $HOME by default + default_container_home_dir = os.environ.get('HOME', '/') diff --git a/test/integration/test_resolvers.py b/test/integration/test_resolvers.py index acca9fc1ffc..94a55562b8d 100644 --- a/test/integration/test_resolvers.py +++ b/test/integration/test_resolvers.py @@ -1,4 +1,4 @@ -"""Integration tests for conda dependency resolution.""" +"""Integration tests for dependency resolution.""" import os from tempfile import mkdtemp @@ -125,6 +125,29 @@ class CondaResolutionIntegrationTestCase(integration_util.IntegrationTestCase): response = create_response.json() assert not [True for d in response if d['dependency_type'] == 'conda'] + def _uninstall_mulled_example_multi_1(self, resolver_type=None): + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/dependencies" % tool_id + data = {'id': tool_id, 'resolver_type': resolver_type} + create_response = self._delete(endpoint, data=data, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert not [True for d in response if d['dependency_type'] == 'conda'] + + def test_conda_install_with_resolver_type_via_tools_api(self): + # Makes sure dependency is not already installed + self._uninstall_mulled_example_multi_1(resolver_type='conda') + # Now do the actual test + tool_id = 'mulled_example_multi_1' + endpoint = "tools/%s/dependencies" % tool_id + data = {'id': tool_id, 'resolver_type': 'conda'} + create_response = self._post(endpoint, data=data, admin=True) + self._assert_status_code_is(create_response, 200) + response = create_response.json() + assert any([True for d in response if d['dependency_type'] == 'conda']) + # Now that we know install was successfullt we can also doube check that the uninstall works + self._uninstall_mulled_example_multi_1(resolver_type='conda') + def test_conda_clean(self): endpoint = 'dependency_resolvers/clean' create_response = self._post(endpoint, data={}, admin=True)