From 7c9c7f830536a0d15e5c8036618379d461743fa0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 13 Aug 2021 14:24:16 -0400 Subject: [PATCH 1/3] Rename containers_resolvers_config_file to container_resolvers_config_file --- doc/source/admin/galaxy_options.rst | 10 +++++----- lib/galaxy/config/__init__.py | 10 ++++++---- .../config/sample/container_resolvers_conf.xml.sample | 4 ++-- lib/galaxy/config/sample/galaxy.yml.sample | 8 ++++---- lib/galaxy/tool_util/deps/containers.py | 8 ++++---- lib/galaxy/tool_util/deps/dependencies.py | 4 ++-- lib/galaxy/webapps/galaxy/config_schema.yml | 6 +++--- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index ba2ba87389a..55c452e3e78 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -679,14 +679,14 @@ :Type: bool -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``containers_resolvers_config_file`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``container_resolvers_config_file`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: Container resolvers configuration (beta). Set up a file describing container resolvers to use when discovering containers for Galaxy. - If this is set to None, the default containers loaded is + If this is set to None, the default container resolvers loaded is determined by enable_mulled_containers. :Default: ``None`` :Type: str @@ -4449,7 +4449,7 @@ filtering (https://galaxyproject.org/user-defined-toolbox-filters/) functions. -:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters`` +:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters`` :Type: str diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 0eb7403b1ae..a49c3e32f9a 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -491,13 +491,15 @@ class CommonConfigurationMixin: class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file', - 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist') + 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist', + 'containers_resolvers_config_file') renamed_options = { 'blacklist_file': 'email_domain_blocklist_file', 'whitelist_file': 'email_domain_allowlist_file', 'sanitize_whitelist_file': 'sanitize_allowlist_file', 'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist', 'fetch_url_whitelist': 'fetch_url_allowlist', + 'containers_resolvers_config_file': 'container_resolvers_config_file', } default_config_file_name = 'galaxy.yml' deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'} @@ -728,8 +730,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # self, populate config_dict self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] - if self.containers_resolvers_config_file: - self.containers_resolvers_config_file = self._in_config_dir(self.containers_resolvers_config_file) + if self.container_resolvers_config_file: + self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file) # tool_dependency_dir can be "none" (in old configs). If so, set it to None if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none': @@ -1175,7 +1177,7 @@ class ConfiguresGalaxyMixin: container_image_cache_path=self.config.container_image_cache_path, library_import_dir=self.config.library_import_dir, enable_mulled_containers=self.config.enable_mulled_containers, - containers_resolvers_config_file=self.config.containers_resolvers_config_file, + container_resolvers_config_file=self.config.container_resolvers_config_file, involucro_path=self.config.involucro_path, involucro_auto_init=self.config.involucro_auto_init, mulled_channels=self.config.mulled_channels, diff --git a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample index 78051d5cb64..ce59312b097 100644 --- a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample +++ b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample @@ -1,4 +1,4 @@ - + @@ -63,4 +63,4 @@ - + diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 1184e78bfa4..89924ec739b 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -438,9 +438,9 @@ galaxy: # Container resolvers configuration (beta). Set up a file describing # container resolvers to use when discovering containers for Galaxy. - # If this is set to None, the default containers loaded is determined - # by enable_mulled_containers. - #containers_resolvers_config_file: null + # If this is set to None, the default container resolvers loaded is + # determined by enable_mulled_containers. + #container_resolvers_config_file: null # involucro is a tool used to build Docker or Singularity containers # for tools from Conda dependencies referenced in tools as @@ -2188,7 +2188,7 @@ galaxy: # The base module(s) that are searched for modules for toolbox # filtering (https://galaxyproject.org/user-defined-toolbox-filters/) # functions. - #toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters + #toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters # Galaxy uses AMQP internally for communicating between processes. # For example, when reloading the toolbox or locking job execution, diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index af49c501556..5bd43e4b453 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -194,12 +194,12 @@ class ContainerRegistry: self.mulled_resolution_cache = mulled_resolution_cache def __build_container_resolvers(self, app_info): - conf_file = getattr(app_info, 'containers_resolvers_config_file', None) + conf_file = getattr(app_info, 'container_resolvers_config_file', None) if not conf_file: - return self.__default_containers_resolvers() + return self.__default_container_resolvers() if not os.path.exists(conf_file): log.debug("Unable to find config file '%s'", conf_file) - return self.__default_containers_resolvers() + return self.__default_container_resolvers() plugin_source = plugin_config.plugin_source_from_path(conf_file) return self._parse_resolver_conf(plugin_source) @@ -209,7 +209,7 @@ class ContainerRegistry: } return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds) - def __default_containers_resolvers(self): + def __default_container_resolvers(self): default_resolvers = [ ExplicitContainerResolver(self.app_info), ExplicitSingularityContainerResolver(self.app_info), diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py index 760eac66239..8d513d01cf1 100644 --- a/lib/galaxy/tool_util/deps/dependencies.py +++ b/lib/galaxy/tool_util/deps/dependencies.py @@ -15,7 +15,7 @@ class AppInfo: container_image_cache_path=None, library_import_dir=None, enable_mulled_containers=False, - containers_resolvers_config_file=None, + container_resolvers_config_file=None, involucro_path=None, involucro_auto_init=True, mulled_channels=DEFAULT_CHANNELS, @@ -29,7 +29,7 @@ class AppInfo: self.container_image_cache_path = container_image_cache_path self.library_import_dir = library_import_dir self.enable_mulled_containers = enable_mulled_containers - self.containers_resolvers_config_file = containers_resolvers_config_file + self.container_resolvers_config_file = container_resolvers_config_file self.involucro_path = involucro_path self.involucro_auto_init = involucro_auto_init self.mulled_channels = mulled_channels diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index be85ddc24ef..ec6f109fa19 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -504,14 +504,14 @@ mapping: Container availability will vary by tool, this option will only be used for job destinations with Docker or Singularity enabled. - containers_resolvers_config_file: + container_resolvers_config_file: type: str required: false desc: | Container resolvers configuration (beta). Set up a file describing container resolvers to use when discovering containers for Galaxy. If - this is set to None, the default containers loaded is determined by - enable_mulled_containers. + this is set to None, the default container resolvers loaded is + determined by enable_mulled_containers. involucro_path: type: str From 0d3ef9d060e86f38db85675c9502662fe8f45460 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 13 Aug 2021 14:55:23 -0400 Subject: [PATCH 2/3] Update tests for config option rename --- test/integration/test_config_defaults.py | 2 +- test/integration/test_containerized_jobs.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/test_config_defaults.py b/test/integration/test_config_defaults.py index 343029b941a..bf91badb288 100644 --- a/test/integration/test_config_defaults.py +++ b/test/integration/test_config_defaults.py @@ -54,7 +54,7 @@ PATH_CONFIG_PROPERTIES = [ 'citation_cache_data_dir', 'citation_cache_lock_dir', 'cluster_files_directory', - 'containers_resolvers_config_file', + 'container_resolvers_config_file', 'data_manager_config_file', 'datatypes_config_file', 'dependency_resolvers_config_file', diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index 3ee0a3dbcd6..d7abc533f56 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -155,8 +155,8 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): config["jobs_directory"] = cls.jobs_directory config["job_config_file"] = cls.job_config_file disable_dependency_resolution(config) - containers_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml") - with open(containers_resolvers_config_path, "w") as f: + container_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml") + with open(container_resolvers_config_path, "w") as f: f.write(""" - type: mapping mappings: @@ -164,7 +164,7 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): tool_id: mulled_example_broken_no_requirements identifier: 'quay.io/biocontainers/bwa:0.7.15--0' """) - config["containers_resolvers_config_file"] = containers_resolvers_config_path + config["container_resolvers_config_file"] = container_resolvers_config_path @classmethod def setUpClass(cls): From db6944523f4cbdb6e28b2567c37959741d82020b Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 13 Aug 2021 15:23:30 -0400 Subject: [PATCH 3/3] Support setting container resolvers directly in the app config --- doc/source/admin/galaxy_options.rst | 16 ++++++++++++++++ lib/galaxy/config/__init__.py | 1 + lib/galaxy/config/sample/galaxy.yml.sample | 9 +++++++++ lib/galaxy/tool_util/deps/containers.py | 20 +++++++++++++------- lib/galaxy/tool_util/deps/dependencies.py | 2 ++ lib/galaxy/webapps/galaxy/config_schema.yml | 14 ++++++++++++++ test/integration/test_containerized_jobs.py | 19 +++++++++++++++++++ 7 files changed, 74 insertions(+), 7 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 55c452e3e78..70c10644e8a 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -692,6 +692,22 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~ +``container_resolvers`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Rather than specifying a container_resolvers_config_file, the + definition of the resolvers to enable can be embedded into + Galaxy's config with this option. This has no effect if a + container_resolvers_config_file is used. + The syntax, available resolvers, and documentation of their + options is explained in detail in the documentation: + https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html +:Default: ``None`` +:Type: seq + + ~~~~~~~~~~~~~~~~~~ ``involucro_path`` ~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index a49c3e32f9a..216379255e1 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -1178,6 +1178,7 @@ class ConfiguresGalaxyMixin: library_import_dir=self.config.library_import_dir, enable_mulled_containers=self.config.enable_mulled_containers, container_resolvers_config_file=self.config.container_resolvers_config_file, + container_resolvers_config_dict=self.config.container_resolvers, involucro_path=self.config.involucro_path, involucro_auto_init=self.config.involucro_auto_init, mulled_channels=self.config.mulled_channels, diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 89924ec739b..59df1a5a4ae 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -442,6 +442,15 @@ galaxy: # determined by enable_mulled_containers. #container_resolvers_config_file: null + # Rather than specifying a container_resolvers_config_file, the + # definition of the resolvers to enable can be embedded into Galaxy's + # config with this option. This has no effect if a + # container_resolvers_config_file is used. + # The syntax, available resolvers, and documentation of their options + # is explained in detail in the documentation: + # https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html + #container_resolvers: null + # involucro is a tool used to build Docker or Singularity containers # for tools from Conda dependencies referenced in tools as # `requirement` s. The following path is the location of involucro on diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index 5bd43e4b453..30dc9456607 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -195,13 +195,19 @@ class ContainerRegistry: def __build_container_resolvers(self, app_info): conf_file = getattr(app_info, 'container_resolvers_config_file', None) - if not conf_file: - return self.__default_container_resolvers() - if not os.path.exists(conf_file): - log.debug("Unable to find config file '%s'", conf_file) - return self.__default_container_resolvers() - plugin_source = plugin_config.plugin_source_from_path(conf_file) - return self._parse_resolver_conf(plugin_source) + conf_dict = getattr(app_info, 'container_resolvers_config_dict', None) + plugin_source = None + if conf_file and not os.path.exists(conf_file): + log.warning(f"Unable to find config file '{conf_file}'") + elif conf_file: + log.debug("Loading container resolution config from file '{conf_file}'") + plugin_source = plugin_config.plugin_source_from_path(conf_file) + elif conf_dict: + log.debug("Loading container resolution config inline from Galaxy configuration file") + plugin_source = plugin_config.plugin_source_from_dict(conf_dict) + if plugin_source: + return self._parse_resolver_conf(plugin_source) + return self.__default_container_resolvers() def _parse_resolver_conf(self, plugin_source): extra_kwds = { diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py index 8d513d01cf1..ec22c43afe1 100644 --- a/lib/galaxy/tool_util/deps/dependencies.py +++ b/lib/galaxy/tool_util/deps/dependencies.py @@ -16,6 +16,7 @@ class AppInfo: library_import_dir=None, enable_mulled_containers=False, container_resolvers_config_file=None, + container_resolvers_config_dict=None, involucro_path=None, involucro_auto_init=True, mulled_channels=DEFAULT_CHANNELS, @@ -30,6 +31,7 @@ class AppInfo: self.library_import_dir = library_import_dir self.enable_mulled_containers = enable_mulled_containers self.container_resolvers_config_file = container_resolvers_config_file + self.container_resolvers_config_dict = container_resolvers_config_dict self.involucro_path = involucro_path self.involucro_auto_init = involucro_auto_init self.mulled_channels = mulled_channels diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index ec6f109fa19..a26c08487eb 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -513,6 +513,20 @@ mapping: this is set to None, the default container resolvers loaded is determined by enable_mulled_containers. + container_resolvers: + type: seq + sequence: + - type: any + desc: | + Rather than specifying a container_resolvers_config_file, the definition of the + resolvers to enable can be embedded into Galaxy's config with this option. + This has no effect if a container_resolvers_config_file is used. + + The syntax, available resolvers, and documentation of their options is explained in detail in the + documentation: + + https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html + involucro_path: type: str default: involucro diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index d7abc533f56..3887650cf55 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -183,6 +183,25 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): assert "0.7.15-r1140" in output +class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.jobs_directory = cls._test_driver.mkdtemp() + config["jobs_directory"] = cls.jobs_directory + config["job_config_file"] = cls.job_config_file + disable_dependency_resolution(config) + container_resolvers_config = [{ + 'type': 'mapping', + 'mappings': [{ + 'container_type': 'docker', + 'tool_id': 'mulled_example_broken_no_requirements', + 'identifier': 'quay.io/biocontainers/bwa:0.7.15--0', + }], + }] + config["container_resolvers"] = container_resolvers_config + + # Singularity 2.4 in the official Vagrant issue has some problems running this test # case by default because subdirectories of /tmp don't bind correctly. Overridding # TMPDIR can fix this.