opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #12344 from natefoo/container-resolvers

Browse Source 
Support setting the container resolvers configuration directly in the Galaxy app config
This commit is contained in:
John Chilton
2021-08-16 10:41:32 -04:00
committed by GitHub
parent 0944b65138 db6944523f
commit f71d982a38
9 changed files with 102 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/admin/galaxy_options.rst
+21 -5
View File
@@ -679,19 +679,35 @@
:Type: bool
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``containers_resolvers_config_file``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``container_resolvers_config_file``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Container resolvers configuration (beta). Set up a file describing
container resolvers to use when discovering containers for Galaxy.
If this is set to None, the default containers loaded is
If this is set to None, the default container resolvers loaded is
determined by enable_mulled_containers.
:Default: ``None``
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~
``container_resolvers``
~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Rather than specifying a container_resolvers_config_file, the
definition of the resolvers to enable can be embedded into
Galaxy's config with this option. This has no effect if a
container_resolvers_config_file is used.
The syntax, available resolvers, and documentation of their
options is explained in detail in the documentation:
https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html
:Default: ``None``
:Type: seq
~~~~~~~~~~~~~~~~~~
``involucro_path``
~~~~~~~~~~~~~~~~~~
@@ -4449,7 +4465,7 @@
filtering
(https://galaxyproject.org/user-defined-toolbox-filters/)
functions.
:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters``
:Default: ``galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters``
:Type: str
lib/galaxy/config/__init__.py
+7 -4
View File
@@ -491,13 +491,15 @@ class CommonConfigurationMixin:
class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
deprecated_options = ('database_file', 'track_jobs_in_database', 'blacklist_file', 'whitelist_file',
'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist')
'sanitize_whitelist_file', 'user_library_import_symlink_whitelist', 'fetch_url_whitelist',
'containers_resolvers_config_file')
renamed_options = {
'blacklist_file': 'email_domain_blocklist_file',
'whitelist_file': 'email_domain_allowlist_file',
'sanitize_whitelist_file': 'sanitize_allowlist_file',
'user_library_import_symlink_whitelist': 'user_library_import_symlink_allowlist',
'fetch_url_whitelist': 'fetch_url_allowlist',
'containers_resolvers_config_file': 'container_resolvers_config_file',
}
default_config_file_name = 'galaxy.yml'
deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'}
@@ -728,8 +730,8 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
# self, populate config_dict
self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping]
if self.containers_resolvers_config_file:
self.containers_resolvers_config_file = self._in_config_dir(self.containers_resolvers_config_file)
if self.container_resolvers_config_file:
self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file)
# tool_dependency_dir can be "none" (in old configs). If so, set it to None
if self.tool_dependency_dir and self.tool_dependency_dir.lower() == 'none':
@@ -1175,7 +1177,8 @@ class ConfiguresGalaxyMixin:
container_image_cache_path=self.config.container_image_cache_path,
library_import_dir=self.config.library_import_dir,
enable_mulled_containers=self.config.enable_mulled_containers,
containers_resolvers_config_file=self.config.containers_resolvers_config_file,
container_resolvers_config_file=self.config.container_resolvers_config_file,
container_resolvers_config_dict=self.config.container_resolvers,
involucro_path=self.config.involucro_path,
involucro_auto_init=self.config.involucro_auto_init,
mulled_channels=self.config.mulled_channels,
lib/galaxy/config/sample/container_resolvers_conf.xml.sample
+2 -2
View File
@@ -1,4 +1,4 @@
<containers_resolvers>
<container_resolvers>
<explicit />
<!-- explicit: resolves container URI for a job through explict container
tags in the tool XML wrapper. -->
@@ -63,4 +63,4 @@
<!-- Specify a fallback container for tools/jobs that don't match the above
resolvers. -->
</containers_resolvers>
</container_resolvers>
lib/galaxy/config/sample/galaxy.yml.sample
+13 -4
View File
@@ -438,9 +438,18 @@ galaxy:
# Container resolvers configuration (beta). Set up a file describing
# container resolvers to use when discovering containers for Galaxy.
# If this is set to None, the default containers loaded is determined
# by enable_mulled_containers.
#containers_resolvers_config_file: null
# If this is set to None, the default container resolvers loaded is
# determined by enable_mulled_containers.
#container_resolvers_config_file: null
# Rather than specifying a container_resolvers_config_file, the
# definition of the resolvers to enable can be embedded into Galaxy's
# config with this option. This has no effect if a
# container_resolvers_config_file is used.
# The syntax, available resolvers, and documentation of their options
# is explained in detail in the documentation:
# https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html
#container_resolvers: null
# involucro is a tool used to build Docker or Singularity containers
# for tools from Conda dependencies referenced in tools as
@@ -2188,7 +2197,7 @@ galaxy:
# The base module(s) that are searched for modules for toolbox
# filtering (https://galaxyproject.org/user-defined-toolbox-filters/)
# functions.
#toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters
#toolbox_filter_base_modules: galaxy.tools.filters,galaxy.tools.toolbox.filters,galaxy.tool_util.toolbox.filters
# Galaxy uses AMQP internally for communicating between processes.
# For example, when reloading the toolbox or locking job execution,
lib/galaxy/tool_util/deps/containers.py
+15 -9
View File
@@ -194,14 +194,20 @@ class ContainerRegistry:
self.mulled_resolution_cache = mulled_resolution_cache
def __build_container_resolvers(self, app_info):
conf_file = getattr(app_info, 'containers_resolvers_config_file', None)
if not conf_file:
return self.__default_containers_resolvers()
if not os.path.exists(conf_file):
log.debug("Unable to find config file '%s'", conf_file)
return self.__default_containers_resolvers()
plugin_source = plugin_config.plugin_source_from_path(conf_file)
return self._parse_resolver_conf(plugin_source)
conf_file = getattr(app_info, 'container_resolvers_config_file', None)
conf_dict = getattr(app_info, 'container_resolvers_config_dict', None)
plugin_source = None
if conf_file and not os.path.exists(conf_file):
log.warning(f"Unable to find config file '{conf_file}'")
elif conf_file:
log.debug("Loading container resolution config from file '{conf_file}'")
plugin_source = plugin_config.plugin_source_from_path(conf_file)
elif conf_dict:
log.debug("Loading container resolution config inline from Galaxy configuration file")
plugin_source = plugin_config.plugin_source_from_dict(conf_dict)
if plugin_source:
return self._parse_resolver_conf(plugin_source)
return self.__default_container_resolvers()
def _parse_resolver_conf(self, plugin_source):
extra_kwds = {
@@ -209,7 +215,7 @@ class ContainerRegistry:
}
return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
def __default_containers_resolvers(self):
def __default_container_resolvers(self):
default_resolvers = [
ExplicitContainerResolver(self.app_info),
ExplicitSingularityContainerResolver(self.app_info),
lib/galaxy/tool_util/deps/dependencies.py
+4 -2
View File
@@ -15,7 +15,8 @@ class AppInfo:
container_image_cache_path=None,
library_import_dir=None,
enable_mulled_containers=False,
containers_resolvers_config_file=None,
container_resolvers_config_file=None,
container_resolvers_config_dict=None,
involucro_path=None,
involucro_auto_init=True,
mulled_channels=DEFAULT_CHANNELS,
@@ -29,7 +30,8 @@ class AppInfo:
self.container_image_cache_path = container_image_cache_path
self.library_import_dir = library_import_dir
self.enable_mulled_containers = enable_mulled_containers
self.containers_resolvers_config_file = containers_resolvers_config_file
self.container_resolvers_config_file = container_resolvers_config_file
self.container_resolvers_config_dict = container_resolvers_config_dict
self.involucro_path = involucro_path
self.involucro_auto_init = involucro_auto_init
self.mulled_channels = mulled_channels
lib/galaxy/webapps/galaxy/config_schema.yml
+17 -3
View File
@@ -504,14 +504,28 @@ mapping:
Container availability will vary by tool, this option will only be used
for job destinations with Docker or Singularity enabled.
containers_resolvers_config_file:
container_resolvers_config_file:
type: str
required: false
desc: |
Container resolvers configuration (beta). Set up a file describing
container resolvers to use when discovering containers for Galaxy. If
this is set to None, the default containers loaded is determined by
enable_mulled_containers.
this is set to None, the default container resolvers loaded is
determined by enable_mulled_containers.
container_resolvers:
type: seq
sequence:
- type: any
desc: |
Rather than specifying a container_resolvers_config_file, the definition of the
resolvers to enable can be embedded into Galaxy's config with this option.
This has no effect if a container_resolvers_config_file is used.
The syntax, available resolvers, and documentation of their options is explained in detail in the
documentation:
https://docs.galaxyproject.org/en/master/admin/dependency_resolvers.html
involucro_path:
type: str
test/integration/test_config_defaults.py
+1 -1
View File
@@ -54,7 +54,7 @@ PATH_CONFIG_PROPERTIES = [
'citation_cache_data_dir',
'citation_cache_lock_dir',
'cluster_files_directory',
'containers_resolvers_config_file',
'container_resolvers_config_file',
'data_manager_config_file',
'datatypes_config_file',
'dependency_resolvers_config_file',
test/integration/test_containerized_jobs.py
+22 -3
View File
@@ -155,8 +155,8 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase):
config["jobs_directory"] = cls.jobs_directory
config["job_config_file"] = cls.job_config_file
disable_dependency_resolution(config)
containers_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml")
with open(containers_resolvers_config_path, "w") as f:
container_resolvers_config_path = os.path.join(cls.jobs_directory, "container_resolvers.yml")
with open(container_resolvers_config_path, "w") as f:
f.write("""
- type: mapping
mappings:
@@ -164,7 +164,7 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase):
tool_id: mulled_example_broken_no_requirements
identifier: 'quay.io/biocontainers/bwa:0.7.15--0'
""")
config["containers_resolvers_config_file"] = containers_resolvers_config_path
config["container_resolvers_config_file"] = container_resolvers_config_path
@classmethod
def setUpClass(cls):
@@ -183,6 +183,25 @@ class MappingContainerResolverTestCase(integration_util.IntegrationTestCase):
assert "0.7.15-r1140" in output
class InlineContainerConfigurationTestCase(MappingContainerResolverTestCase):
@classmethod
def handle_galaxy_config_kwds(cls, config):
cls.jobs_directory = cls._test_driver.mkdtemp()
config["jobs_directory"] = cls.jobs_directory
config["job_config_file"] = cls.job_config_file
disable_dependency_resolution(config)
container_resolvers_config = [{
'type': 'mapping',
'mappings': [{
'container_type': 'docker',
'tool_id': 'mulled_example_broken_no_requirements',
'identifier': 'quay.io/biocontainers/bwa:0.7.15--0',
}],
}]
config["container_resolvers"] = container_resolvers_config
# Singularity 2.4 in the official Vagrant issue has some problems running this test
# case by default because subdirectories of /tmp don't bind correctly. Overridding
# TMPDIR can fix this.