From db6944523f4cbdb6e28b2567c37959741d82020b Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 13 Aug 2021 15:23:30 -0400 Subject: [PATCH] Support setting container resolvers directly in the app config --- doc/source/admin/galaxy_options.rst | 16 ++++++++++++++++ lib/galaxy/config/__init__.py | 1 + lib/galaxy/config/sample/galaxy.yml.sample | 9 +++++++++ lib/galaxy/tool_util/deps/containers.py | 20 +++++++++++++------- lib/galaxy/tool_util/deps/dependencies.py | 2 ++ lib/galaxy/webapps/galaxy/config_schema.yml | 14 ++++++++++++++ test/integration/test_containerized_jobs.py | 19 +++++++++++++++++++ 7 files changed, 74 insertions(+), 7 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 55c452e3e78..70c10644e8a 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -692,6 +692,22 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~ +``container_resolvers`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Rather than specifying a container_resolvers_config_file, the + definition of the resolvers to enable can be embedded into + Galaxy's config with this option. This has no effect if a + container_resolvers_config_file is used. + The syntax, available resolvers, and documentation of their + options is explained in detail in the documentation: + https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html +:Default: ``None`` +:Type: seq + + ~~~~~~~~~~~~~~~~~~ ``involucro_path`` ~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index a49c3e32f9a..216379255e1 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -1178,6 +1178,7 @@ class ConfiguresGalaxyMixin: library_import_dir=self.config.library_import_dir, enable_mulled_containers=self.config.enable_mulled_containers, container_resolvers_config_file=self.config.container_resolvers_config_file, + container_resolvers_config_dict=self.config.container_resolvers, involucro_path=self.config.involucro_path, involucro_auto_init=self.config.involucro_auto_init, mulled_channels=self.config.mulled_channels, diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 89924ec739b..59df1a5a4ae 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -442,6 +442,15 @@ galaxy: # determined by enable_mulled_containers. #container_resolvers_config_file: null + # Rather than specifying a container_resolvers_config_file, the + # definition of the resolvers to enable can be embedded into Galaxy's + # config with this option. This has no effect if a + # container_resolvers_config_file is used. + # The syntax, available resolvers, and documentation of their options + # is explained in detail in the documentation: + # https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html + #container_resolvers: null + # involucro is a tool used to build Docker or Singularity containers # for tools from Conda dependencies referenced in tools as # `requirement` s. The following path is the location of involucro on diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index 5bd43e4b453..30dc9456607 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -195,13 +195,19 @@ class ContainerRegistry: def __build_container_resolvers(self, app_info): conf_file = getattr(app_info, 'container_resolvers_config_file', None) - if not conf_file: - return self.__default_container_resolvers() - if not os.path.exists(conf_file): - log.debug("Unable to find config file '%s'", conf_file) - return self.__default_container_resolvers() - plugin_source = plugin_config.plugin_source_from_path(conf_file) - return self._parse_resolver_conf(plugin_source) + conf_dict = getattr(app_info, 'container_resolvers_config_dict', None) + plugin_source = None + if conf_file and not os.path.exists(conf_file): + log.warning(f"Unable to find config file '{conf_file}'") + elif conf_file: + log.debug("Loading container resolution config from file '{conf_file}'") + plugin_source = plugin_config.plugin_source_from_path(conf_file) + elif conf_dict: + log.debug("Loading container resolution config inline from Galaxy configuration file") + plugin_source = plugin_config.plugin_source_from_dict(conf_dict) + if plugin_source: + return self._parse_resolver_conf(plugin_source) + return self.__default_container_resolvers() def _parse_resolver_conf(self, plugin_source): extra_kwds = { diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py index 8d513d01cf1..ec22c43afe1 100644 --- a/lib/galaxy/tool_util/deps/dependencies.py +++ b/lib/galaxy/tool_util/deps/dependencies.py @@ -16,6 +16,7 @@ class AppInfo: library_import_dir=None, enable_mulled_containers=False, container_resolvers_config_file=None, + container_resolvers_config_dict=None, involucro_path=None, involucro_auto_init=True, mulled_channels=DEFAULT_CHANNELS, @@ -30,6 +31,7 @@ class AppInfo: self.library_import_dir = library_import_dir self.enable_mulled_containers = enable_mulled_containers self.container_resolvers_config_file = container_resolvers_config_file + self.container_resolvers_config_dict = container_resolvers_config_dict self.involucro_path = involucro_path self.involucro_auto_init = involucro_auto_init self.mulled_channels = mulled_channels diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index ec6f109fa19..a26c08487eb 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -513,6 +513,20 @@ mapping: this is set to None, the default container resolvers loaded is determined by enable_mulled_containers. + container_resolvers: + type: seq + sequence: + - type: any + desc: | + Rather than specifying a container_resolvers_config_file, the definition of the + resolvers to enable can be embedded into Galaxy's config with this option. + This has no effect if a container_resolvers_config_file is used. + + The syntax, available resolvers, and documentation of their options is explained in detail in the + documentation: + + https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html + involucro_path: type: str default: involucro diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index d7abc533f56..3887650cf55 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -183,6 +183,25 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): assert "0.7.15-r1140" in output +class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.jobs_directory = cls._test_driver.mkdtemp() + config["jobs_directory"] = cls.jobs_directory + config["job_config_file"] = cls.job_config_file + disable_dependency_resolution(config) + container_resolvers_config = [{ + 'type': 'mapping', + 'mappings': [{ + 'container_type': 'docker', + 'tool_id': 'mulled_example_broken_no_requirements', + 'identifier': 'quay.io/biocontainers/bwa:0.7.15--0', + }], + }] + config["container_resolvers"] = container_resolvers_config + + # Singularity 2.4 in the official Vagrant issue has some problems running this test # case by default because subdirectories of /tmp don't bind correctly. Overridding # TMPDIR can fix this.