diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index ba2ba87389a..70c10644e8a 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -679,19 +679,35 @@ :Type: bool -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -``containers_resolvers_config_file`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``container_resolvers_config_file`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: Container resolvers configuration (beta). Set up a file describing container resolvers to use when discovering containers for Galaxy. - If this is set to None, the default containers loaded is + If this is set to None, the default container resolvers loaded is determined by enable_mulled_containers. :Default: ``None`` :Type: str +~~~~~~~~~~~~~~~~~~~~~~~ +``container_resolvers`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Rather than specifying a container_resolvers_config_file, the + definition of the resolvers to enable can be embedded into + Galaxy's config with this option. This has no effect if a + container_resolvers_config_file is used. + The syntax, available resolvers, and documentation of their + options is explained in detail in the documentation: + https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html +:Default: ``None`` +:Type: seq + + ~~~~~~~~~~~~~~~~~~ ``involucro_path`` ~~~~~~~~~~~~~~~~~~ @@ -4449,7 +4465,7 @@ filtering (https://galaxyproject.org/user-defined-toolbox-filters/) functions. -:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters`` +:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters`` :Type: str diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 0eb7403b1ae..216379255e1 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -491,13 +491,15 @@ class CommonConfigurationMixin: class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file', - 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist') + 'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist', + 'containers_resolvers_config_file') renamed_options = { 'blacklist_file': 'email_domain_blocklist_file', 'whitelist_file': 'email_domain_allowlist_file', 'sanitize_whitelist_file': 'sanitize_allowlist_file', 'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist', 'fetch_url_whitelist': 'fetch_url_allowlist', + 'containers_resolvers_config_file': 'container_resolvers_config_file', } default_config_file_name = 'galaxy.yml' deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'} @@ -728,8 +730,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin): # self, populate config_dict self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] - if self.containers_resolvers_config_file: - self.containers_resolvers_config_file = self._in_config_dir(self.containers_resolvers_config_file) + if self.container_resolvers_config_file: + self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file) # tool_dependency_dir can be "none" (in old configs). If so, set it to None if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none': @@ -1175,7 +1177,8 @@ class ConfiguresGalaxyMixin: container_image_cache_path=self.config.container_image_cache_path, library_import_dir=self.config.library_import_dir, enable_mulled_containers=self.config.enable_mulled_containers, - containers_resolvers_config_file=self.config.containers_resolvers_config_file, + container_resolvers_config_file=self.config.container_resolvers_config_file, + container_resolvers_config_dict=self.config.container_resolvers, involucro_path=self.config.involucro_path, involucro_auto_init=self.config.involucro_auto_init, mulled_channels=self.config.mulled_channels, diff --git a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample index 78051d5cb64..ce59312b097 100644 --- a/lib/galaxy/config/sample/container_resolvers_conf.xml.sample +++ b/lib/galaxy/config/sample/container_resolvers_conf.xml.sample @@ -1,4 +1,4 @@ - + @@ -63,4 +63,4 @@ - + diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 1184e78bfa4..59df1a5a4ae 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -438,9 +438,18 @@ galaxy: # Container resolvers configuration (beta). Set up a file describing # container resolvers to use when discovering containers for Galaxy. - # If this is set to None, the default containers loaded is determined - # by enable_mulled_containers. - #containers_resolvers_config_file: null + # If this is set to None, the default container resolvers loaded is + # determined by enable_mulled_containers. + #container_resolvers_config_file: null + + # Rather than specifying a container_resolvers_config_file, the + # definition of the resolvers to enable can be embedded into Galaxy's + # config with this option. This has no effect if a + # container_resolvers_config_file is used. + # The syntax, available resolvers, and documentation of their options + # is explained in detail in the documentation: + # https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html + #container_resolvers: null # involucro is a tool used to build Docker or Singularity containers # for tools from Conda dependencies referenced in tools as @@ -2188,7 +2197,7 @@ galaxy: # The base module(s) that are searched for modules for toolbox # filtering (https://galaxyproject.org/user-defined-toolbox-filters/) # functions. - #toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters + #toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters # Galaxy uses AMQP internally for communicating between processes. # For example, when reloading the toolbox or locking job execution, diff --git a/lib/galaxy/tool_util/deps/containers.py b/lib/galaxy/tool_util/deps/containers.py index af49c501556..30dc9456607 100644 --- a/lib/galaxy/tool_util/deps/containers.py +++ b/lib/galaxy/tool_util/deps/containers.py @@ -194,14 +194,20 @@ class ContainerRegistry: self.mulled_resolution_cache = mulled_resolution_cache def __build_container_resolvers(self, app_info): - conf_file = getattr(app_info, 'containers_resolvers_config_file', None) - if not conf_file: - return self.__default_containers_resolvers() - if not os.path.exists(conf_file): - log.debug("Unable to find config file '%s'", conf_file) - return self.__default_containers_resolvers() - plugin_source = plugin_config.plugin_source_from_path(conf_file) - return self._parse_resolver_conf(plugin_source) + conf_file = getattr(app_info, 'container_resolvers_config_file', None) + conf_dict = getattr(app_info, 'container_resolvers_config_dict', None) + plugin_source = None + if conf_file and not os.path.exists(conf_file): + log.warning(f"Unable to find config file '{conf_file}'") + elif conf_file: + log.debug("Loading container resolution config from file '{conf_file}'") + plugin_source = plugin_config.plugin_source_from_path(conf_file) + elif conf_dict: + log.debug("Loading container resolution config inline from Galaxy configuration file") + plugin_source = plugin_config.plugin_source_from_dict(conf_dict) + if plugin_source: + return self._parse_resolver_conf(plugin_source) + return self.__default_container_resolvers() def _parse_resolver_conf(self, plugin_source): extra_kwds = { @@ -209,7 +215,7 @@ class ContainerRegistry: } return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds) - def __default_containers_resolvers(self): + def __default_container_resolvers(self): default_resolvers = [ ExplicitContainerResolver(self.app_info), ExplicitSingularityContainerResolver(self.app_info), diff --git a/lib/galaxy/tool_util/deps/dependencies.py b/lib/galaxy/tool_util/deps/dependencies.py index 760eac66239..ec22c43afe1 100644 --- a/lib/galaxy/tool_util/deps/dependencies.py +++ b/lib/galaxy/tool_util/deps/dependencies.py @@ -15,7 +15,8 @@ class AppInfo: container_image_cache_path=None, library_import_dir=None, enable_mulled_containers=False, - containers_resolvers_config_file=None, + container_resolvers_config_file=None, + container_resolvers_config_dict=None, involucro_path=None, involucro_auto_init=True, mulled_channels=DEFAULT_CHANNELS, @@ -29,7 +30,8 @@ class AppInfo: self.container_image_cache_path = container_image_cache_path self.library_import_dir = library_import_dir self.enable_mulled_containers = enable_mulled_containers - self.containers_resolvers_config_file = containers_resolvers_config_file + self.container_resolvers_config_file = container_resolvers_config_file + self.container_resolvers_config_dict = container_resolvers_config_dict self.involucro_path = involucro_path self.involucro_auto_init = involucro_auto_init self.mulled_channels = mulled_channels diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index be85ddc24ef..a26c08487eb 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -504,14 +504,28 @@ mapping: Container availability will vary by tool, this option will only be used for job destinations with Docker or Singularity enabled. - containers_resolvers_config_file: + container_resolvers_config_file: type: str required: false desc: | Container resolvers configuration (beta). Set up a file describing container resolvers to use when discovering containers for Galaxy. If - this is set to None, the default containers loaded is determined by - enable_mulled_containers. + this is set to None, the default container resolvers loaded is + determined by enable_mulled_containers. + + container_resolvers: + type: seq + sequence: + - type: any + desc: | + Rather than specifying a container_resolvers_config_file, the definition of the + resolvers to enable can be embedded into Galaxy's config with this option. + This has no effect if a container_resolvers_config_file is used. + + The syntax, available resolvers, and documentation of their options is explained in detail in the + documentation: + + https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html involucro_path: type: str diff --git a/test/integration/test_config_defaults.py b/test/integration/test_config_defaults.py index 343029b941a..bf91badb288 100644 --- a/test/integration/test_config_defaults.py +++ b/test/integration/test_config_defaults.py @@ -54,7 +54,7 @@ PATH_CONFIG_PROPERTIES = [ 'citation_cache_data_dir', 'citation_cache_lock_dir', 'cluster_files_directory', - 'containers_resolvers_config_file', + 'container_resolvers_config_file', 'data_manager_config_file', 'datatypes_config_file', 'dependency_resolvers_config_file', diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index 3ee0a3dbcd6..3887650cf55 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -155,8 +155,8 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): config["jobs_directory"] = cls.jobs_directory config["job_config_file"] = cls.job_config_file disable_dependency_resolution(config) - containers_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml") - with open(containers_resolvers_config_path, "w") as f: + container_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml") + with open(container_resolvers_config_path, "w") as f: f.write(""" - type: mapping mappings: @@ -164,7 +164,7 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): tool_id: mulled_example_broken_no_requirements identifier: 'quay.io/biocontainers/bwa:0.7.15--0' """) - config["containers_resolvers_config_file"] = containers_resolvers_config_path + config["container_resolvers_config_file"] = container_resolvers_config_path @classmethod def setUpClass(cls): @@ -183,6 +183,25 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase): assert "0.7.15-r1140" in output +class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + cls.jobs_directory = cls._test_driver.mkdtemp() + config["jobs_directory"] = cls.jobs_directory + config["job_config_file"] = cls.job_config_file + disable_dependency_resolution(config) + container_resolvers_config = [{ + 'type': 'mapping', + 'mappings': [{ + 'container_type': 'docker', + 'tool_id': 'mulled_example_broken_no_requirements', + 'identifier': 'quay.io/biocontainers/bwa:0.7.15--0', + }], + }] + config["container_resolvers"] = container_resolvers_config + + # Singularity 2.4 in the official Vagrant issue has some problems running this test # case by default because subdirectories of /tmp don't bind correctly. Overridding # TMPDIR can fix this.