diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 216379255e1..83231bbb541 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -1208,8 +1208,8 @@ class ConfiguresGalaxyMixin: for enabled_container_type in self.container_finder._enabled_container_types(destination.params): container_types_to_destinations[enabled_container_type].append(destination) self.toolbox.dependency_manager.set_enabled_container_types(container_types_to_destinations) - self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.container_registry.resolver_classes) - self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.container_registry.container_resolvers) + self.toolbox.dependency_manager.resolver_classes.update(self.container_finder.default_container_registry.resolver_classes) + self.toolbox.dependency_manager.dependency_resolvers.extend(self.container_finder.default_container_registry.container_resolvers) def _configure_tool_data_tables(self, from_shed_config): from galaxy.tools.data import ToolDataTableManager diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index 30dc9456607..2d0d74ff409 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -42,7 +42,9 @@ class ContainerFinder: def __init__(self, app_info, mulled_resolution_cache=None): self.app_info = app_info - self.container_registry = ContainerRegistry(app_info, mulled_resolution_cache=mulled_resolution_cache) + self.mulled_resolution_cache = mulled_resolution_cache + self.default_container_registry = ContainerRegistry(app_info, mulled_resolution_cache=mulled_resolution_cache) + self.destination_container_registeries = {} def _enabled_container_types(self, destination_info): return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)] @@ -51,11 +53,26 @@ class ContainerFinder: """Regardless of destination properties - find best container for tool. Given container types and container.ToolInfo description of the tool.""" - return self.container_registry.find_best_container_description(enabled_container_types, tool_info, **kwds) + return self.default_container_registry.find_best_container_description(enabled_container_types, tool_info, **kwds) def resolve(self, enabled_container_types, tool_info, **kwds): """Regardless of destination properties - find ResolvedContainerDescription for tool.""" - return self.container_registry.resolve(enabled_container_types, tool_info, **kwds) + return self.default_container_registry.resolve(enabled_container_types, tool_info, **kwds) + + def _container_registry_for_destination(self, destination_info): + destination_id = destination_info.get("id") # Probably not the way to get the ID? + destination_container_registry = None + if destination_id and destination_id not in self.destination_container_registeries: + if 'container_resolvers' in destination_info: + destination_container_registry = ContainerRegistry(self.app_info, destination_info=destination_info, mulled_resolution_cache=self.mulled_resolution_cache) + self.destination_container_registeries[destination_id] = destination_container_registry + elif not destination_id and 'container_resolvers' in destination_info: + destination_container_registry = ContainerRegistry(self.app_info, destination_info=destination_info, mulled_resolution_cache=self.mulled_resolution_cache) + + if destination_container_registry is None and destination_id and destination_id in self.destination_container_registeries: + destination_container_registry = self.destination_container_registeries[destination_id] + + return destination_container_registry or self.default_container_registry def find_container(self, tool_info, destination_info, job_info): enabled_container_types = self._enabled_container_types(destination_info) @@ -102,7 +119,8 @@ class ContainerFinder: return container # Otherwise lets see if we can find container for the tool. - container_description = self.find_best_container_description(enabled_container_types, tool_info) + container_registry = self._container_registry_for_destination(destination_info) + container_description = container_registry.find_best_container_description(enabled_container_types, tool_info) container = __destination_container(container_description) if container: return container @@ -124,7 +142,7 @@ class ContainerFinder: return NULL_CONTAINER def resolution_cache(self): - return self.container_registry.get_resolution_cache() + return self.default_container_registry.get_resolution_cache() def __overridden_container_id(self, container_type, destination_info): if not self.__container_type_enabled(container_type, destination_info): @@ -186,25 +204,33 @@ class NullContainerFinder: class ContainerRegistry: """Loop through enabled ContainerResolver plugins and find first match.""" - def __init__(self, app_info, mulled_resolution_cache=None): + def __init__(self, app_info, destination_info=None, mulled_resolution_cache=None): self.resolver_classes = self.__resolvers_dict() self.enable_mulled_containers = app_info.enable_mulled_containers self.app_info = app_info - self.container_resolvers = self.__build_container_resolvers(app_info) + self.container_resolvers = self.__build_container_resolvers(app_info, destination_info) self.mulled_resolution_cache = mulled_resolution_cache - def __build_container_resolvers(self, app_info): - conf_file = getattr(app_info, 'container_resolvers_config_file', None) - conf_dict = getattr(app_info, 'container_resolvers_config_dict', None) + def __build_container_resolvers(self, app_info, destination_info): + app_conf_file = getattr(app_info, 'container_resolvers_config_file', None) + app_conf_dict = getattr(app_info, 'container_resolvers_config_dict', None) + + if destination_info is not None: + conf_file = destination_info.get('container_resolvers_config_file', app_conf_file) + conf_dict = destination_info.get('container_resolvers', app_conf_dict) + else: + conf_file = app_conf_file + conf_dict = app_conf_dict + plugin_source = None - if conf_file and not os.path.exists(conf_file): + if conf_dict: + log.debug("Loading container resolution config inline from Galaxy or job configuration file") + plugin_source = plugin_config.plugin_source_from_dict(conf_dict) + elif conf_file and not os.path.exists(conf_file): log.warning(f"Unable to find config file '{conf_file}'") elif conf_file: log.debug("Loading container resolution config from file '{conf_file}'") plugin_source = plugin_config.plugin_source_from_path(conf_file) - elif conf_dict: - log.debug("Loading container resolution config inline from Galaxy configuration file") - plugin_source = plugin_config.plugin_source_from_dict(conf_dict) if plugin_source: return self._parse_resolver_conf(plugin_source) return self.__default_container_resolvers() diff --git a/test/functional/tools/mulled_example_broken_no_requirements_fallback.xml b/test/functional/tools/mulled_example_broken_no_requirements_fallback.xml new file mode 100644 index 00000000000..f22d820c2c0 --- /dev/null +++ b/test/functional/tools/mulled_example_broken_no_requirements_fallback.xml @@ -0,0 +1,25 @@ + + + + + + + $output_1 2>&1 + ]]> + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 4c27f1a0c8c..7c82aca504e 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -243,6 +243,7 @@ + diff --git a/test/integration/dockerized_job_conf.xml b/test/integration/dockerized_job_conf.xml deleted file mode 100644 index d4cdce9f651..00000000000 --- a/test/integration/dockerized_job_conf.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - true - false - - - - - - - - - - - - diff --git a/test/integration/dockerized_job_conf.yml b/test/integration/dockerized_job_conf.yml new file mode 100644 index 00000000000..dda06d8ae65 --- /dev/null +++ b/test/integration/dockerized_job_conf.yml @@ -0,0 +1,26 @@ +--- +runners: + local: + load: galaxy.jobs.runners.local:LocalJobRunner + workers: 1 + +execution: + default: local_docker + environments: + local_docker: + runner: local + docker_enabled: true + local_docker_inline_container_resolvers: + runner: local + docker_enabled: true + container_resolvers: + - type: fallback + identifier: 'quay.io/biocontainers/bwa:0.7.15--0' + local_upload: + runner: local + +tools: + - id: upload1 + environment: local_upload + - id: mulled_example_broken_no_requirements_fallback + environment: local_docker_inline_container_resolvers diff --git a/test/integration/singularity_job_conf.xml b/test/integration/singularity_job_conf.xml deleted file mode 100644 index 1a500dcab3f..00000000000 --- a/test/integration/singularity_job_conf.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - true - - - - - - - - - - - - diff --git a/test/integration/singularity_job_conf.yml b/test/integration/singularity_job_conf.yml new file mode 100644 index 00000000000..470a4a9685d --- /dev/null +++ b/test/integration/singularity_job_conf.yml @@ -0,0 +1,18 @@ +--- +runners: + local: + load: galaxy.jobs.runners.local:LocalJobRunner + workers: 1 + +execution: + default: local_singularity + environments: + local_singularity: + runner: local + singularity_enabled: true + local_upload: + runner: local + +tools: + - id: upload1 + environment: local_upload diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index 3887650cf55..2981e0d1e3f 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -12,8 +12,8 @@ from galaxy_test.driver import integration_util from .test_job_environments import RunsEnvironmentJobs SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) -DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml") -SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.xml") +DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.yml") +SINGULARITY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "singularity_job_conf.yml") EXTENDED_TIMEOUT = 120 @@ -202,6 +202,36 @@ class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase): config["container_resolvers"] = container_resolvers_config +class InlineJobEnvironmentContainerResolverTestCase(integration_util.IntegrationTestCase): + + framework_tool_and_types = True + container_type = 'docker' + job_config_file = DOCKERIZED_JOB_CONFIG_FILE + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.jobs_directory = cls._test_driver.mkdtemp() + config["jobs_directory"] = cls.jobs_directory + config["job_config_file"] = cls.job_config_file + disable_dependency_resolution(config) + + @classmethod + def setUpClass(cls): + skip_if_container_type_unavailable(cls) + super().setUpClass() + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + self.history_id = self.dataset_populator.new_history() + + def test_inline_environment_container_resolver_configuration(self): + self.dataset_populator.run_tool("mulled_example_broken_no_requirements_fallback", {}, self.history_id) + self.dataset_populator.wait_for_history(self.history_id, assert_ok=True) + output = self.dataset_populator.get_history_dataset_content(self.history_id, timeout=EXTENDED_TIMEOUT) + assert "0.7.15-r1140" in output + + # Singularity 2.4 in the official Vagrant issue has some problems running this test # case by default because subdirectories of /tmp don't bind correctly. Overridding # TMPDIR can fix this.