opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 15:37:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10172 from ic4f/dev_config_debug2

Browse Source 
Uniformly process config files
This commit is contained in:
Marius van den Beek
2020-09-08 17:00:32 +02:00
committed by GitHub
parent 6f6671e6a3 aede1afcef
commit 4ccd314c2a
22 changed files with 555 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/source/admin/galaxy_options.rst
+61 -39
View File
@@ -267,7 +267,7 @@
config/tool_conf.xml does not exist). Can be a single file, a list
of files, or (for backwards compatibility) a comma-separated list
of files.
:Default: ``config/tool_conf.xml``
:Default: ``tool_conf.xml``
:Type: any
@@ -285,8 +285,6 @@
this option is preferable. This file will be created automatically
upon tool installation, whereas Galaxy will fail to start if any
files in tool_config_file cannot be read.
The value of this option will be resolved with respect to
<managed_config_dir>.
:Default: ``shed_tool_conf.xml``
:Type: str
@@ -316,8 +314,6 @@
migration scripts to install tools that have been migrated to the
tool shed upon a new release, they will be added to this tool
config file.
The value of this option will be resolved with respect to
<managed_config_dir>.
:Default: ``migrated_tools_conf.xml``
:Type: str
@@ -381,8 +377,6 @@
then use Conda if available. See
https://github.com/galaxyproject/galaxy/blob/dev/doc/source/admin/dependency_resolvers.rst
for more information on these options.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``dependency_resolvers_conf.xml``
:Type: str
@@ -480,6 +474,30 @@
:Type: bool
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``local_conda_mapping_file``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Path to a file that provides a mapping from abstract packages to
concrete conda packages. See
`config/local_conda_mapping.yml.sample` for examples.
:Default: ``local_conda_mapping.yml``
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~~~
``modules_mapping_files``
~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Path to a file that provides a mapping from abstract packages to
locally installed modules. See
`config/environment_modules_mapping.yml.sample` for examples.
:Default: ``environment_modules_mapping.yml``
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``use_cached_dependency_manager``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -532,8 +550,6 @@
File containing the Galaxy Tool Sheds that should be made
available to install from in the admin interface (.sample used if
default does not exist).
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``tool_sheds_conf.xml``
:Type: str
@@ -621,8 +637,6 @@
:Description:
Configured FileSource plugins.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``file_sources_conf.yml``
:Type: str
@@ -731,7 +745,7 @@
XML config file that contains data table entries for the
ToolDataTableManager. This file is manually # maintained by the
Galaxy administrator (.sample used if default does not exist).
:Default: ``config/tool_data_table_conf.xml``
:Default: ``tool_data_table_conf.xml``
:Type: str
@@ -747,8 +761,6 @@
installation, these entries are automatically added to the
following file, which is parsed and applied to the
ToolDataTableManager at server start up.
The value of this option will be resolved with respect to
<managed_config_dir>.
:Default: ``shed_tool_data_table_conf.xml``
:Type: str
@@ -814,8 +826,6 @@
:Description:
File that defines the builds (dbkeys) available at sites used by
display applications and the URL to those sites.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``build_sites.yml``
:Type: str
@@ -855,7 +865,7 @@
If a datatype appears in multiple files, the last definition is
used (though the first sniffer is used so limit sniffer
definitions to one file).
:Default: ``config/datatypes_conf.xml``
:Default: ``datatypes_conf.xml``
:Type: str
@@ -1141,8 +1151,6 @@
:Description:
Configuration file for the object store If this is set and exists,
it overrides any other objectstore settings.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``object_store_conf.xml``
:Type: str
@@ -1578,8 +1586,6 @@
:Description:
Location of the configuration file containing extra user
preferences.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``user_preferences_extra_conf.yml``
:Type: str
@@ -3207,8 +3213,6 @@
:Description:
Sets the path to OIDC configuration file.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``oidc_config.xml``
:Type: str
@@ -3219,8 +3223,6 @@
:Description:
Sets the path to OIDC backends configuration file.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``oidc_backends_config.xml``
:Type: str
@@ -3233,8 +3235,6 @@
XML config file that allows the use of different authentication
providers (e.g. LDAP) instead or in addition to local
authentication (.sample is used if default does not exist).
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``auth_conf.xml``
:Type: str
@@ -3456,8 +3456,6 @@
:Description:
XML config file that contains the job metric collection
configuration.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``job_metrics_conf.xml``
:Type: str
@@ -3500,7 +3498,7 @@
:Description:
File where Data Managers are configured (.sample used if default
does not exist).
:Default: ``config/data_manager_conf.xml``
:Default: ``data_manager_conf.xml``
:Type: str
@@ -3511,8 +3509,6 @@
:Description:
File where Tool Shed based Data Managers are configured. This file
will be created automatically upon data manager installation.
The value of this option will be resolved with respect to
<managed_config_dir>.
:Default: ``shed_data_manager_conf.xml``
:Type: str
@@ -3543,7 +3539,7 @@
Jobs are run locally on the system on which Galaxy is started.
Advanced job running capabilities can be configured through the
job configuration file.
:Default: ``config/job_conf.xml``
:Default: ``job_conf.xml``
:Type: str
@@ -3973,8 +3969,6 @@
definition. These fields will be presented to users in the tool
forms and allow them to overwrite default job resources such as
number of processors, memory and walltime.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``job_resource_params_conf.xml``
:Type: str
@@ -3989,8 +3983,6 @@
requires both a description of the fields available (which
defaults to the definitions in job_resource_params_file if not
set).
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``workflow_resource_params_conf.xml``
:Type: str
@@ -4022,8 +4014,6 @@
:Description:
Optional configuration file similar to `job_config_file` to
specify which Galaxy processes should schedule workflows.
The value of this option will be resolved with respect to
<config_dir>.
:Default: ``workflow_schedulers_conf.xml``
:Type: str
@@ -4297,4 +4287,36 @@
:Type: bool
~~~~~~~~~~~~~~~~~~~~~
``error_report_file``
~~~~~~~~~~~~~~~~~~~~~
:Description:
Path to error reports configuration file.
:Default: ``error_report.yml``
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~~~~
``containers_config_file``
~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Path to container interface configuration file. The containers
interface is only used if `enable_beta_containers_interface`
config option is set.
:Default: ``containers_conf.yml``
:Type: str
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``tool_destinations_config_file``
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:Description:
Path to dynamic tool destinations configuration file.
:Default: ``tool_destinations.yml``
:Type: str
lib/galaxy/app.py
+2 -2
View File
@@ -199,8 +199,8 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
if self.config.enable_oidc:
from galaxy.authnz import managers
self.authnz_manager = managers.AuthnzManager(self,
self.config.oidc_config,
self.config.oidc_backends_config)
self.config.oidc_config_file,
self.config.oidc_backends_config_file)
self.sentry_client = None
if self.config.sentry_dsn:
lib/galaxy/auth/__init__.py
+1 -1
View File
@@ -15,7 +15,7 @@ class AuthManager:
def __init__(self, app):
self.__app = app
self.redact_username_in_logs = app.config.redact_username_in_logs
self.authenticators = get_authenticators(app.config.auth_config_file, app.config.auth_config_file_set)
self.authenticators = get_authenticators(app.config.auth_config_file, app.config.is_set('auth_config_file'))
def check_registration_allowed(self, email, username, password):
"""Checks if the provided email/username is allowed to register."""
lib/galaxy/config/__init__.py
+101 -106
View File
@@ -110,9 +110,12 @@ class BaseAppConfiguration:
# If VALUE == first directory in a user-supplied path that resolves to KEY, it will be stripped from that path
renamed_options = None
deprecated_dirs = None
add_sample_file_to_defaults = None # for these options, add sample config files to their defaults
listify_options = None # values for these options are processed as lists of values
def __init__(self, **kwargs):
self._process_renamed_options(kwargs)
self._kwargs = kwargs # Save these as a record of explicitly set options
self.config_dict = kwargs
self.root = find_root(kwargs)
self._set_config_base(kwargs)
@@ -120,7 +123,9 @@ class BaseAppConfiguration:
self._raw_config = self.schema.defaults.copy() # Save schema defaults as initial config values (raw_config)
self._update_raw_config_from_kwargs(kwargs) # Overwrite raw_config with values passed in kwargs
self._create_attributes_from_raw_config() # Create attributes based on raw_config
self._preprocess_paths_to_resolve() # Any preprocessing steps that need to happen before paths are resolved
self._resolve_paths() # Overwrite attribute values with resolved paths
self._postprocess_paths_to_resolve() # Any steps that need to happen after paths are resolved
def _process_renamed_options(self, kwargs):
"""Update kwargs to set any unset renamed options to values of old-named options, if set.
@@ -132,6 +137,14 @@ class BaseAppConfiguration:
if old in kwargs and new not in kwargs:
kwargs[new] = kwargs[old]
def is_set(self, key):
"""Check if a configuration option has been explicitly set."""
# NOTE: This will check all supplied keyword arguments, including those not in the schema.
# To check only schema options, change the line below to `if property not in self._raw_config:`
if key not in self._raw_config:
log.warning("Configuration option does not exist: '%s'" % key)
return key in self._kwargs
def resolve_path(self, path):
"""Resolve a path relative to Galaxy's root."""
return self._in_root_dir(path)
@@ -164,7 +177,13 @@ class BaseAppConfiguration:
self.config_dir = os.path.abspath(self.config_dir)
self.data_dir = config_kwargs.get('data_dir')
if self.data_dir:
self.data_dir = os.path.abspath(self.data_dir)
self.sample_config_dir = os.path.join(os.path.dirname(__file__), 'sample')
if self.sample_config_dir:
self.sample_config_dir = os.path.abspath(self.sample_config_dir)
self.managed_config_dir = config_kwargs.get('managed_config_dir')
if self.managed_config_dir:
self.managed_config_dir = os.path.abspath(self.managed_config_dir)
@@ -198,6 +217,52 @@ class BaseAppConfiguration:
# Override in subclasses
raise Exception('Not implemented')
def _preprocess_paths_to_resolve(self):
# For these options, if option is not set, listify its defaults and add a sample config file.
if self.add_sample_file_to_defaults:
for key in self.add_sample_file_to_defaults:
if not self.is_set(key):
defaults = listify(getattr(self, key), do_strip=True)
sample = '%s.sample' % defaults[-1] # if there are multiple defaults, use last as template
sample = self._in_sample_dir(sample) # resolve w.r.t sample_dir
defaults.append(sample)
setattr(self, key, defaults)
def _postprocess_paths_to_resolve(self):
def select_one_path_from_list():
# To consider: options with a sample file added to defaults except options that can have multiple values.
# If value is not set, check each path in list; set to first path that exists; if none exist, set to last path in list.
keys = self.add_sample_file_to_defaults - self.listify_options if self.listify_options else self.add_sample_file_to_defaults
for key in keys:
if not self.is_set(key):
paths = getattr(self, key)
for path in paths:
if self._path_exists(path):
setattr(self, key, path)
break
else:
setattr(self, key, paths[-1]) # TODO: we assume it exists; but we've already checked in the loop! Raise error instead?
def select_one_or_all_paths_from_list():
# Values for these options are lists of paths. If value is not set, use defaults if all paths in list exist;
# otherwise, set to last path in list.
for key in self.listify_options:
if not self.is_set(key):
paths = getattr(self, key)
for path in paths:
if not self._path_exists(path):
setattr(self, key, [paths[-1]]) # value is a list
break
if self.add_sample_file_to_defaults: # Currently, this is the ONLY case when we need to pick one file from a list
select_one_path_from_list()
if self.listify_options:
select_one_or_all_paths_from_list()
def _path_exists(self, path): # factored out so we can mock it in tests
return os.path.exists(path)
def _update_raw_config_from_kwargs(self, kwargs):
def convert_datatype(key, value):
@@ -261,7 +326,18 @@ class BaseAppConfiguration:
_cache = {}
for key in self.schema.paths_to_resolve:
resolve(key)
value = getattr(self, key)
# Check if value is a list or should be listified; if so, listify and resolve each item separately.
if type(value) == list or (self.listify_options and key in self.listify_options):
saved_values = listify(getattr(self, key), do_strip=True) # listify and save original value
setattr(self, key, '_') # replace value with temporary placeholder
resolve(key) # resolve temporary value (`_` becomes `parent-path/_`)
resolved_base = getattr(self, key)[:-1] # get rid of placeholder in resolved path
# apply resolved base to saved values
resolved_paths = [os.path.join(resolved_base, value) for value in saved_values]
setattr(self, key, resolved_paths) # set config.key to a list of resolved paths
else:
resolve(key)
def _in_root_dir(self, path):
return self._in_dir(self.root, path)
@@ -281,48 +357,6 @@ class BaseAppConfiguration:
def _in_dir(self, _dir, path):
return os.path.join(_dir, path) if path else None
def _parse_config_file_options(self, defaults, listify_defaults, config_kwargs):
def root_join(val):
try:
return os.path.join(self.root, val)
except TypeError:
template = "Failed to set configuration variable %s, value %s of wrong type %s"
message = template % (var, val, type(val))
raise ConfigurationError(message)
for var, values in defaults.items():
if config_kwargs.get(var) is not None:
path = config_kwargs.get(var)
setattr(self, var + '_set', True)
else:
for value in values:
if os.path.exists(value):
path = value
break
else:
path = values[-1]
setattr(self, var + '_set', False)
setattr(self, var, root_join(path))
for var, values in listify_defaults.items():
paths = []
if config_kwargs.get(var) is not None:
paths = listify(config_kwargs.get(var))
setattr(self, var + '_set', True)
else:
for value in values:
for path in listify(value):
if not os.path.exists(path):
break
else:
paths = listify(value)
break
else:
paths = listify(values[-1])
setattr(self, var + '_set', False)
setattr(self, var, [root_join(x) for x in paths])
class CommonConfigurationMixin:
"""Shared configuration settings code for Galaxy and ToolShed."""
@@ -384,6 +418,19 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
default_config_file_name = 'galaxy.yml'
deprecated_dirs = {'config_dir': 'config', 'data_dir': 'database'}
add_sample_file_to_defaults = {
'build_sites_config_file',
'datatypes_config_file',
'job_metrics_config_file',
'tool_data_table_config_path',
'tool_config_file',
}
listify_options = {
'tool_data_table_config_path',
'tool_config_file',
}
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._override_tempdir(kwargs)
@@ -397,8 +444,9 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
tempfile.tempdir = self.new_file_path
def _process_config(self, kwargs):
# Resolve paths of other config files
self.parse_config_file_options(kwargs)
# Backwards compatibility for names used in too many places to fix
self.datatypes_config = self.datatypes_config_file
self.tool_configs = self.tool_config_file
# Collect the umask and primary gid from the environment
self.umask = os.umask(0o77) # get the current umask
@@ -428,9 +476,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.tool_data_path = self._in_data_dir(self.schema.defaults['tool_data_path'])
self.builds_file_path = os.path.join(self.tool_data_path, self.builds_file_path)
self.len_file_path = os.path.join(self.tool_data_path, self.len_file_path)
# Galaxy OIDC settings.
self.oidc_config = kwargs.get("oidc_config_file", self.oidc_config_file)
self.oidc_backends_config = kwargs.get("oidc_backends_config_file", self.oidc_backends_config_file)
self.oidc = {}
self.integrated_tool_panel_config = self._in_managed_config_dir(self.integrated_tool_panel_config)
integrated_tool_panel_tracking_directory = kwargs.get('integrated_tool_panel_tracking_directory')
@@ -538,7 +583,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
break
self.sanitize_allowlist_file = _sanitize_allowlist_path
self.allowed_origin_hostnames = self._parse_allowed_origin_hostnames(kwargs)
self.allowed_origin_hostnames = self._parse_allowed_origin_hostnames(self.allowed_origin_hostnames)
if "trust_jupyter_notebook_conversion" not in kwargs:
# if option not set, check IPython-named alternative, falling back to schema default if not set either
_default = self.trust_jupyter_notebook_conversion
@@ -594,7 +639,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
self.object_store_cache_path = self._in_root_dir(kwargs.get("object_store_cache_path", self._in_data_dir("object_store_cache")))
if self.object_store_store_by is None:
self.object_store_store_by = 'id'
if not self.file_path_set and self.file_path.endswith('objects'):
if not self.is_set('file_path') and self.file_path.endswith('objects'):
self.object_store_store_by = 'uuid'
assert self.object_store_store_by in ['id', 'uuid'], "Invalid value for object_store_store_by [%s]" % self.object_store_store_by
# Handle AWS-specific config options for backward compatibility
@@ -665,7 +710,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
with open(self.user_preferences_extra_conf_path) as stream:
self.user_preferences_extra = yaml.safe_load(stream)
except Exception:
if self.user_preferences_extra_conf_path_set:
if self.is_set('user_preferences_extra_conf_path'):
log.warning('Config file (%s) could not be found or is malformed.' % self.user_preferences_extra_conf_path)
self.user_preferences_extra = {'preferences': {}}
@@ -771,56 +816,6 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
'UWSGI_PORT': port
})
def parse_config_file_options(self, kwargs):
"""Backwards compatibility for config files moved to the config/ dir."""
defaults = dict(
auth_config_file=[self._in_config_dir('auth_conf.xml')],
build_sites_config_file=[self._in_config_dir('build_sites.yml'), self._in_sample_dir('build_sites.yml.sample')],
containers_config_file=[self._in_config_dir('containers_conf.yml')],
data_manager_config_file=[self._in_config_dir('data_manager_conf.xml')],
datatypes_config_file=[self._in_config_dir('datatypes_conf.xml'), self._in_sample_dir('datatypes_conf.xml.sample')],
dependency_resolvers_config_file=[self._in_config_dir('dependency_resolvers_conf.xml')],
error_report_file=[self._in_config_dir('error_report.yml')],
job_config_file=[self._in_config_dir('job_conf.xml')],
job_metrics_config_file=[self._in_config_dir('job_metrics_conf.xml'), self._in_sample_dir('job_metrics_conf.xml.sample')],
job_resource_params_file=[self._in_config_dir('job_resource_params_conf.xml')],
file_sources_config_file=[self._in_config_dir('file_sources_conf.yml')],
local_conda_mapping_file=[self._in_config_dir('local_conda_mapping.yml')],
migrated_tools_config=[self._in_managed_config_dir('migrated_tools_conf.xml')],
modules_mapping_files=[self._in_config_dir('environment_modules_mapping.yml')],
object_store_config_file=[self._in_config_dir('object_store_conf.xml')],
oidc_backends_config_file=[self._in_config_dir('oidc_backends_config.xml')],
oidc_config_file=[self._in_config_dir('oidc_config.xml')],
shed_data_manager_config_file=[self._in_managed_config_dir('shed_data_manager_conf.xml')],
shed_tool_config_file=[self._in_managed_config_dir('shed_tool_conf.xml')],
shed_tool_data_table_config=[self._in_managed_config_dir('shed_tool_data_table_conf.xml')],
tool_destinations_config_file=[self._in_config_dir('tool_destinations.yml')],
tool_sheds_config_file=[self._in_config_dir('tool_sheds_conf.xml')],
trs_servers_config_file=[self._in_config_dir('trs_servers_conf.yml')],
user_preferences_extra_conf_path=[self._in_config_dir('user_preferences_extra_conf.yml')],
workflow_resource_params_file=[self._in_config_dir('workflow_resource_params_conf.xml')],
workflow_schedulers_config_file=[self._in_config_dir('workflow_schedulers_conf.xml')],
markdown_export_css=[self._in_config_dir('markdown_export.css')],
markdown_export_css_pages=[self._in_config_dir('markdown_export_pages.css')],
markdown_export_css_invocation_reports=[self._in_config_dir('markdown_export_invocation_reports.css')],
# self.file_path set to self._in_data_dir('objects') by schema
file_path=[self._in_data_dir('files'), self.file_path],
)
listify_defaults = {
'tool_data_table_config_path': [
self._in_config_dir('tool_data_table_conf.xml'),
self._in_sample_dir('tool_data_table_conf.xml.sample')],
'tool_config_file': [
self._in_config_dir('tool_conf.xml'),
self._in_sample_dir('tool_conf.xml.sample')]
}
self._parse_config_file_options(defaults, listify_defaults, kwargs)
# Backwards compatibility for names used in too many places to fix
self.datatypes_config = self.datatypes_config_file
self.tool_configs = self.tool_config_file
def reload_sanitize_allowlist(self, explicit=True):
self.sanitize_allowlist = []
try:
@@ -865,13 +860,13 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
log.warning("Config option '%s' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file." % key)
@staticmethod
def _parse_allowed_origin_hostnames(kwargs):
def _parse_allowed_origin_hostnames(allowed_origin_hostnames):
"""
Parse a CSV list of strings/regexp of hostnames that should be allowed
to use CORS and will be sent the Access-Control-Allow-Origin header.
"""
allowed_origin_hostnames = listify(kwargs.get('allowed_origin_hostnames'))
if not allowed_origin_hostnames:
allowed_origin_hostnames_list = listify(allowed_origin_hostnames)
if not allowed_origin_hostnames_list:
return None
def parse(string):
@@ -881,7 +876,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
return re.compile(string, flags=(re.UNICODE))
return string
return [parse(v) for v in allowed_origin_hostnames if v]
return [parse(v) for v in allowed_origin_hostnames_list if v]
# legacy naming
@@ -1088,7 +1083,7 @@ class ConfiguresGalaxyMixin:
from_shed_config=from_shed_config)
except OSError as exc:
# Missing shed_tool_data_table_config is okay if it's the default
if exc.errno != errno.ENOENT or self.config.shed_tool_data_table_config_set:
if exc.errno != errno.ENOENT or self.config.is_set('shed_tool_data_table_config'):
raise
def _configure_datatypes_registry(self, installed_repository_manager=None):
lib/galaxy/config/sample/galaxy.yml.sample
+26 -39
View File
@@ -228,7 +228,7 @@ galaxy:
# config/tool_conf.xml does not exist). Can be a single file, a list
# of files, or (for backwards compatibility) a comma-separated list of
# files.
#tool_config_file: config/tool_conf.xml
#tool_config_file: tool_conf.xml
# Tool config file for tools installed from the Galaxy Tool Shed. Must
# be writable by Galaxy and generally should not be edited by hand. In
@@ -239,8 +239,6 @@ galaxy:
# preferable. This file will be created automatically upon tool
# installation, whereas Galaxy will fail to start if any files in
# tool_config_file cannot be read.
# The value of this option will be resolved with respect to
# <managed_config_dir>.
#shed_tool_config_file: shed_tool_conf.xml
# Enable / disable checking if any tools defined in the above non-shed
@@ -255,8 +253,6 @@ galaxy:
# migration scripts to install tools that have been migrated to the
# tool shed upon a new release, they will be added to this tool config
# file.
# The value of this option will be resolved with respect to
# <managed_config_dir>.
#migrated_tools_config: migrated_tools_conf.xml
# File that contains the XML section and tool tags from all tool panel
@@ -292,8 +288,6 @@ galaxy:
# Conda if available. See
# https://github.com/galaxyproject/galaxy/blob/dev/doc/source/admin/dependency_resolvers.rst
# for more information on these options.
# The value of this option will be resolved with respect to
# <config_dir>.
#dependency_resolvers_config_file: dependency_resolvers_conf.xml
# conda_prefix is the location on the filesystem where Conda packages
@@ -332,6 +326,16 @@ galaxy:
# and extra time spent copying packages.
#conda_copy_dependencies: false
# Path to a file that provides a mapping from abstract packages to
# concrete conda packages. See `config/local_conda_mapping.yml.sample`
# for examples.
#local_conda_mapping_file: local_conda_mapping.yml
# Path to a file that provides a mapping from abstract packages to
# locally installed modules. See
# `config/environment_modules_mapping.yml.sample` for examples.
#modules_mapping_files: environment_modules_mapping.yml
# Certain dependency resolvers (namely Conda) take a considerable
# amount of time to build an isolated job environment in the
# job_working_directory if the job working directory is on a network
@@ -358,8 +362,6 @@ galaxy:
# File containing the Galaxy Tool Sheds that should be made available
# to install from in the admin interface (.sample used if default does
# not exist).
# The value of this option will be resolved with respect to
# <config_dir>.
#tool_sheds_config_file: tool_sheds_conf.xml
# Monitor the tools and tool directories listed in any tool config
@@ -405,8 +407,6 @@ galaxy:
#legacy_eager_objectstore_initialization: false
# Configured FileSource plugins.
# The value of this option will be resolved with respect to
# <config_dir>.
#file_sources_config_file: file_sources_conf.yml
# Enable Galaxy to fetch containers registered with quay.io generated
@@ -459,7 +459,7 @@ galaxy:
# XML config file that contains data table entries for the
# ToolDataTableManager. This file is manually # maintained by the
# Galaxy administrator (.sample used if default does not exist).
#tool_data_table_config_path: config/tool_data_table_conf.xml
#tool_data_table_config_path: tool_data_table_conf.xml
# XML config file that contains additional data table entries for the
# ToolDataTableManager. This file is automatically generated based on
@@ -468,8 +468,6 @@ galaxy:
# these entries are automatically added to the following file, which
# is parsed and applied to the ToolDataTableManager at server start
# up.
# The value of this option will be resolved with respect to
# <managed_config_dir>.
#shed_tool_data_table_config: shed_tool_data_table_conf.xml
# Directory where data used by tools is located. See the samples in
@@ -500,8 +498,6 @@ galaxy:
# File that defines the builds (dbkeys) available at sites used by
# display applications and the URL to those sites.
# The value of this option will be resolved with respect to
# <config_dir>.
#build_sites_config_file: build_sites.yml
# File containing old-style genome builds.
@@ -520,7 +516,7 @@ galaxy:
# a datatype appears in multiple files, the last definition is used
# (though the first sniffer is used so limit sniffer definitions to
# one file).
#datatypes_config_file: config/datatypes_conf.xml
#datatypes_config_file: datatypes_conf.xml
# Enable sniffing of compressed datatypes. This can be
# configured/overridden on a per-datatype basis in the
@@ -645,8 +641,6 @@ galaxy:
# Configuration file for the object store If this is set and exists,
# it overrides any other objectstore settings.
# The value of this option will be resolved with respect to
# <config_dir>.
#object_store_config_file: object_store_conf.xml
# What Dataset attribute is used to reference files in an ObjectStore
@@ -844,8 +838,6 @@ galaxy:
# Location of the configuration file containing extra user
# preferences.
# The value of this option will be resolved with respect to
# <config_dir>.
#user_preferences_extra_conf_path: user_preferences_extra_conf.yml
# Default localization for Galaxy UI. Allowed values are listed at the
@@ -1590,20 +1582,14 @@ galaxy:
#enable_oidc: false
# Sets the path to OIDC configuration file.
# The value of this option will be resolved with respect to
# <config_dir>.
#oidc_config_file: oidc_config.xml
# Sets the path to OIDC backends configuration file.
# The value of this option will be resolved with respect to
# <config_dir>.
#oidc_backends_config_file: oidc_backends_config.xml
# XML config file that allows the use of different authentication
# providers (e.g. LDAP) instead or in addition to local authentication
# (.sample is used if default does not exist).
# The value of this option will be resolved with respect to
# <config_dir>.
#auth_config_file: auth_conf.xml
# Optional list of email addresses of API users who can make calls on
@@ -1699,8 +1685,6 @@ galaxy:
# XML config file that contains the job metric collection
# configuration.
# The value of this option will be resolved with respect to
# <config_dir>.
#job_metrics_config_file: job_metrics_conf.xml
# This option allows users to see the job metrics (except for
@@ -1715,12 +1699,10 @@ galaxy:
# File where Data Managers are configured (.sample used if default
# does not exist).
#data_manager_config_file: config/data_manager_conf.xml
#data_manager_config_file: data_manager_conf.xml
# File where Tool Shed based Data Managers are configured. This file
# will be created automatically upon data manager installation.
# The value of this option will be resolved with respect to
# <managed_config_dir>.
#shed_data_manager_config_file: shed_data_manager_conf.xml
# Directory to store Data Manager based tool-data. Defaults to the
@@ -1737,7 +1719,7 @@ galaxy:
# are run locally on the system on which Galaxy is started. Advanced
# job running capabilities can be configured through the job
# configuration file.
#job_config_file: config/job_conf.xml
#job_config_file: job_conf.xml
# Description of job running configuration, can be embedded into
# Galaxy configuration or loaded from an additional file with the
@@ -1948,16 +1930,12 @@ galaxy:
# These fields will be presented to users in the tool forms and allow
# them to overwrite default job resources such as number of
# processors, memory and walltime.
# The value of this option will be resolved with respect to
# <config_dir>.
#job_resource_params_file: job_resource_params_conf.xml
# Similar to the above parameter, workflows can describe parameters
# used to influence scheduling of jobs within the workflow. This
# requires both a description of the fields available (which defaults
# to the definitions in job_resource_params_file if not set).
# The value of this option will be resolved with respect to
# <config_dir>.
#workflow_resource_params_file: workflow_resource_params_conf.xml
# This parameter describes how to map users and workflows to a set of
@@ -1975,8 +1953,6 @@ galaxy:
# Optional configuration file similar to `job_config_file` to specify
# which Galaxy processes should schedule workflows.
# The value of this option will be resolved with respect to
# <config_dir>.
#workflow_schedulers_config_file: workflow_schedulers_conf.xml
# If using job concurrency limits (configured in job_config_file),
@@ -2098,3 +2074,14 @@ galaxy:
# deep learning model are shown.
#overwrite_model_recommendations: false
# Path to error reports configuration file.
#error_report_file: error_report.yml
# Path to container interface configuration file. The containers
# interface is only used if `enable_beta_containers_interface` config
# option is set.
#containers_config_file: containers_conf.yml
# Path to dynamic tool destinations configuration file.
#tool_destinations_config_file: tool_destinations.yml
lib/galaxy/config/sample/local_conda_mapping.yml.sample
+3 -2
View File
@@ -1,2 +1,3 @@
# See $GALAXY_ROOT/lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml for example mapping -
# additional site-specific mappings can be added to config/local_conda_mapping.yml.
# See $GALAXY_ROOT/lib/galaxy/tool_util/deps/resolvers/default_conda_mapping.yml for example mapping -
# additional site-specific mappings can be added to config/local_conda_mapping.yml (or the location
# of <local_conda_mapping_file> specified in galaxy.yml).
lib/galaxy/config/sample/tool_shed.yml.sample
+21
View File
@@ -405,3 +405,24 @@ tool_shed:
# activation emails.
#terms_url: null
# XML config file that allows the use of different authentication
# providers (e.g. LDAP) instead or in addition to local authentication
# (.sample is used if default does not exist).
#auth_config_file: auth_conf.xml
# XML config file that contains additional data table entries for the
# ToolDataTableManager. This file is automatically generated based on
# the current installed tool shed repositories that contain valid
# tool_data_table_conf.xml.sample files. At the time of installation,
# these entries are automatically added to the following file, which
# is parsed and applied to the ToolDataTableManager at server start
# up.
#shed_tool_data_table_config: shed_tool_data_table_conf.xml
# Datatypes config file(s), defines what data (file) types are
# available in Galaxy (.sample is used if default does not exist). If
# a datatype appears in multiple files, the last definition is used
# (though the first sniffer is used so limit sniffer definitions to
# one file).
#datatypes_config_file: datatypes_conf.xml
lib/galaxy/config/schema.py
+4 -4
View File
@@ -82,8 +82,8 @@ class AppSchema(Schema):
"for '{}' does not exist".format(resolves_to, key)
raise_error(message)
def check_type_is_str(option, key):
if option.get('type') != 'str':
def check_type_is_str_or_any(option, key):
if option.get('type') not in ('str', 'any'):
message = "Invalid schema: property '{}' should have type 'str'".format(key)
raise_error(message)
@@ -105,6 +105,6 @@ class AppSchema(Schema):
print(key)
parent = self.app_schema.get(resolves_to)
check_exists(parent, key)
check_type_is_str(parent, key)
check_type_is_str(self.app_schema[key], key)
check_type_is_str_or_any(parent, key)
check_type_is_str_or_any(self.app_schema[key], key)
check_is_dag() # must be called last: walks entire graph
lib/galaxy/tools/__init__.py
+2 -2
View File
@@ -262,11 +262,11 @@ class ToolBox(BaseGalaxyToolBox):
)
def can_load_config_file(self, config_filename):
if config_filename == self.app.config.shed_tool_config_file and not self.app.config.shed_tool_config_file_set:
if config_filename == self.app.config.shed_tool_config_file and not self.app.config.is_set('shed_tool_config_file'):
if self.dynamic_confs():
# Do not load or create a default shed_tool_config_file if another shed_tool_config file has already been loaded
return False
elif self.app.config.tool_config_file_set:
elif self.app.config.is_set('tool_config_file'):
log.warning(
"The default shed tool config file (%s) has been added to the tool_config_file option, if this is "
"not the desired behavior, please set shed_tool_config_file to your primary shed-enabled tool "
lib/galaxy/tools/data_manager/manager.py
+2 -2
View File
@@ -32,14 +32,14 @@ class DataManagers:
try:
self.load_from_xml(self.app.config.shed_data_manager_config_file, store_tool_path=True)
except OSError as exc:
if exc.errno != errno.ENOENT or self.app.config.shed_data_manager_config_file_set:
if exc.errno != errno.ENOENT or self.app.config.is_set('shed_data_manager_config_file'):
raise
def load_from_xml(self, xml_filename, store_tool_path=True):
try:
tree = util.parse_xml(xml_filename)
except OSError as e:
if e.errno != errno.ENOENT or self.app.config.data_manager_config_file_set:
if e.errno != errno.ENOENT or self.app.config.is_set('data_manager_config_file'):
raise
return # default config option and it doesn't exist, which is fine
except Exception as e:
lib/galaxy/webapps/galaxy/config_schema.yml
+74 -39
View File
@@ -208,7 +208,8 @@ mapping:
tool_config_file:
type: any
default: config/tool_conf.xml
default: tool_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Tool config files, defines what tools are available in Galaxy.
@@ -220,6 +221,7 @@ mapping:
shed_tool_config_file:
type: str
default: shed_tool_conf.xml
path_resolves_to: managed_config_dir
required: false
desc: |
Tool config file for tools installed from the Galaxy Tool Shed. Must
@@ -232,8 +234,6 @@ mapping:
installation, whereas Galaxy will fail to start if any files in
tool_config_file cannot be read.
The value of this option will be resolved with respect to <managed_config_dir>.
check_migrate_tools:
type: bool
default: false
@@ -248,14 +248,13 @@ mapping:
migrated_tools_config:
type: str
default: migrated_tools_conf.xml
path_resolves_to: managed_config_dir
required: false
desc: |
Tool config maintained by tool migration scripts. If you use the migration
scripts to install tools that have been migrated to the tool shed upon a new
release, they will be added to this tool config file.
The value of this option will be resolved with respect to <managed_config_dir>.
integrated_tool_panel_config:
type: str
default: integrated_tool_panel.xml
@@ -295,6 +294,7 @@ mapping:
dependency_resolvers_config_file:
type: str
default: dependency_resolvers_conf.xml
path_resolves_to: config_dir
required: false
desc: |
The dependency resolvers config file specifies an ordering and options for how
@@ -304,8 +304,6 @@ mapping:
See https://github.com/galaxyproject/galaxy/blob/dev/doc/source/admin/dependency_resolvers.rst
for more information on these options.
The value of this option will be resolved with respect to <config_dir>.
conda_prefix:
type: str
required: false
@@ -371,6 +369,24 @@ mapping:
This will prevent problems with some specific packages (perl, R), at the cost
of extra disk space usage and extra time spent copying packages.
local_conda_mapping_file:
type: str
default: 'local_conda_mapping.yml'
path_resolves_to: config_dir
required: false
desc: |
Path to a file that provides a mapping from abstract packages to concrete conda packages.
See `config/local_conda_mapping.yml.sample` for examples.
modules_mapping_files:
type: str
default: 'environment_modules_mapping.yml'
path_resolves_to: config_dir
required: false
desc: |
Path to a file that provides a mapping from abstract packages to locally installed modules.
See `config/environment_modules_mapping.yml.sample` for examples.
use_cached_dependency_manager:
type: bool
default: false
@@ -408,13 +424,12 @@ mapping:
tool_sheds_config_file:
type: str
default: tool_sheds_conf.xml
path_resolves_to: config_dir
required: false
desc: |
File containing the Galaxy Tool Sheds that should be made available to
install from in the admin interface (.sample used if default does not exist).
The value of this option will be resolved with respect to <config_dir>.
watch_tools:
type: str
default: 'false'
@@ -472,12 +487,11 @@ mapping:
file_sources_config_file:
type: str
default: file_sources_conf.yml
path_resolves_to: config_dir
required: false
desc: |
Configured FileSource plugins.
The value of this option will be resolved with respect to <config_dir>.
enable_mulled_containers:
type: bool
default: true
@@ -548,7 +562,8 @@ mapping:
tool_data_table_config_path:
type: str
default: config/tool_data_table_conf.xml
default: tool_data_table_conf.xml
path_resolves_to: config_dir
required: false
desc: |
XML config file that contains data table entries for the
@@ -558,6 +573,7 @@ mapping:
shed_tool_data_table_config:
type: str
default: shed_tool_data_table_conf.xml
path_resolves_to: managed_config_dir
required: false
desc: |
XML config file that contains additional data table entries for the
@@ -567,8 +583,6 @@ mapping:
entries are automatically added to the following file, which is parsed and
applied to the ToolDataTableManager at server start up.
The value of this option will be resolved with respect to <managed_config_dir>.
tool_data_path:
type: str
default: tool-data
@@ -607,13 +621,12 @@ mapping:
build_sites_config_file:
type: str
default: build_sites.yml
path_resolves_to: config_dir
required: false
desc: |
File that defines the builds (dbkeys) available at sites used by display applications
and the URL to those sites.
The value of this option will be resolved with respect to <config_dir>.
builds_file_path:
type: str
default: shared/ucsc/builds.txt
@@ -634,7 +647,8 @@ mapping:
datatypes_config_file:
type: str
default: config/datatypes_conf.xml
default: datatypes_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Datatypes config file(s), defines what data (file) types are available in
@@ -847,13 +861,12 @@ mapping:
object_store_config_file:
type: str
default: object_store_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Configuration file for the object store
If this is set and exists, it overrides any other objectstore settings.
The value of this option will be resolved with respect to <config_dir>.
object_store_store_by:
type: str
required: false
@@ -1158,6 +1171,7 @@ mapping:
trs_servers_config_file:
type: str
default: 'trs_servers_conf.yml'
path_resolves_to: config_dir
required: false
desc: |
Allow import of workflows from the TRS servers configured in
@@ -1171,12 +1185,11 @@ mapping:
user_preferences_extra_conf_path:
type: str
default: 'user_preferences_extra_conf.yml'
path_resolves_to: config_dir
required: false
desc: |
Location of the configuration file containing extra user preferences.
The value of this option will be resolved with respect to <config_dir>.
default_locale:
type: str
default: auto
@@ -2371,32 +2384,29 @@ mapping:
oidc_config_file:
type: str
default: oidc_config.xml
path_resolves_to: config_dir
required: false
desc: |
Sets the path to OIDC configuration file.
The value of this option will be resolved with respect to <config_dir>.
oidc_backends_config_file:
type: str
default: oidc_backends_config.xml
path_resolves_to: config_dir
required: false
desc: |
Sets the path to OIDC backends configuration file.
The value of this option will be resolved with respect to <config_dir>.
auth_config_file:
type: str
default: auth_conf.xml
path_resolves_to: config_dir
required: false
desc: |
XML config file that allows the use of different authentication providers
(e.g. LDAP) instead or in addition to local authentication (.sample is used
if default does not exist).
The value of this option will be resolved with respect to <config_dir>.
api_allow_run_as:
type: str
required: false
@@ -2551,12 +2561,11 @@ mapping:
job_metrics_config_file:
type: str
default: job_metrics_conf.xml
path_resolves_to: config_dir
required: false
desc: |
XML config file that contains the job metric collection configuration.
The value of this option will be resolved with respect to <config_dir>.
expose_potentially_sensitive_job_metrics:
type: bool
default: false
@@ -2581,7 +2590,8 @@ mapping:
data_manager_config_file:
type: str
default: config/data_manager_conf.xml
default: data_manager_conf.xml
path_resolves_to: config_dir
required: false
desc: |
File where Data Managers are configured (.sample used if default does not
@@ -2590,13 +2600,12 @@ mapping:
shed_data_manager_config_file:
type: str
default: shed_data_manager_conf.xml
path_resolves_to: managed_config_dir
required: false
desc: |
File where Tool Shed based Data Managers are configured. This file will be created
automatically upon data manager installation.
The value of this option will be resolved with respect to <managed_config_dir>.
galaxy_data_manager_data_path:
type: str
required: false
@@ -2606,7 +2615,8 @@ mapping:
job_config_file:
type: str
default: config/job_conf.xml
default: job_conf.xml
path_resolves_to: config_dir
required: false
desc: |
To increase performance of job execution and the web interface, you can
@@ -2870,6 +2880,7 @@ mapping:
markdown_export_css:
type: str
default: markdown_export.css
path_resolves_to: config_dir
required: false
desc: |
CSS file to apply to all Markdown exports to PDF - currently used by
@@ -2878,6 +2889,7 @@ mapping:
markdown_export_css_pages:
type: str
default: markdown_export_pages.css
path_resolves_to: config_dir
required: false
desc: |
CSS file to apply to "Galaxy Page" exports to PDF. Generally prefer
@@ -2887,6 +2899,7 @@ mapping:
markdown_export_css_invocation_reports:
type: str
default: markdown_export_invocation_reports.css
path_resolves_to: config_dir
required: false
desc: |
CSS file to apply to invocation report exports to PDF. Generally prefer
@@ -2936,6 +2949,7 @@ mapping:
job_resource_params_file:
type: str
default: job_resource_params_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Optional file containing job resource data entry fields definition.
@@ -2943,11 +2957,10 @@ mapping:
overwrite default job resources such as number of processors, memory and
walltime.
The value of this option will be resolved with respect to <config_dir>.
workflow_resource_params_file:
type: str
default: workflow_resource_params_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Similar to the above parameter, workflows can describe parameters used to
@@ -2955,8 +2968,6 @@ mapping:
of the fields available (which defaults to the definitions in
job_resource_params_file if not set).
The value of this option will be resolved with respect to <config_dir>.
workflow_resource_params_mapper:
type: str
required: false
@@ -2974,13 +2985,12 @@ mapping:
workflow_schedulers_config_file:
type: str
default: workflow_schedulers_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Optional configuration file similar to `job_config_file` to specify
which Galaxy processes should schedule workflows.
The value of this option will be resolved with respect to <config_dir>.
cache_user_job_count:
type: bool
default: false
@@ -3162,3 +3172,28 @@ mapping:
Overwrite or append to the tool recommendations by the deep learning model. When set to true, all the recommendations by the deep learning model
are overwritten by the recommendations set by an admin in a config file 'tool_recommendations_overwrite.yml'. When set to false, the recommended tools
by admins and predicted by the deep learning model are shown.
error_report_file:
type: str
default: 'error_report.yml'
path_resolves_to: config_dir
required: false
desc: |
Path to error reports configuration file.
containers_config_file:
type: str
default: 'containers_conf.yml'
path_resolves_to: config_dir
required: false
desc: |
Path to container interface configuration file. The containers interface is only used if
`enable_beta_containers_interface` config option is set.
tool_destinations_config_file:
type: str
default: 'tool_destinations.yml'
path_resolves_to: config_dir
required: false
desc: |
Path to dynamic tool destinations configuration file.
lib/galaxy/workflow/scheduling_manager.py
+1 -1
View File
@@ -178,7 +178,7 @@ class WorkflowSchedulingManager(ConfiguresHandlers):
def __init_schedulers(self):
config_file = self.app.config.workflow_schedulers_config_file
use_default_scheduler = False
if not config_file or (not os.path.exists(config_file) and not self.app.config.workflow_schedulers_config_file_set):
if not config_file or (not os.path.exists(config_file) and not self.app.config.is_set('workflow_schedulers_config_file')):
log.info("No workflow schedulers plugin config file defined, using default scheduler.")
use_default_scheduler = True
elif not os.path.exists(config_file):
lib/galaxy_test/driver/driver_util.py
+22 -7
View File
@@ -186,7 +186,7 @@ def setup_galaxy_config(
for data_manager_config in ['config/data_manager_conf.xml', 'data_manager_conf.xml']:
if os.path.exists(data_manager_config):
default_data_manager_config = data_manager_config
data_manager_config_file = "test/functional/tools/sample_data_manager_conf.xml"
data_manager_config_file = 'test/functional/tools/sample_data_manager_conf.xml'
if default_data_manager_config is not None:
data_manager_config_file = "{},{}".format(default_data_manager_config, data_manager_config_file)
master_api_key = get_master_api_key()
@@ -208,7 +208,11 @@ def setup_galaxy_config(
if shed_tool_conf is not None:
tool_conf = "{},{}".format(tool_conf, shed_tool_conf)
shed_tool_data_table_config = default_shed_tool_data_table_config
# Resolve these paths w.r.t. galaxy root; otherwise galaxy's config system will resolve them w.r.t.
# their parent directories, as per schema.
data_manager_config_file = _resolve_relative_config_paths(data_manager_config_file)
tool_config_file = _resolve_relative_config_paths(tool_conf)
tool_data_table_config_path = _resolve_relative_config_paths(tool_data_table_config_path)
config = dict(
admin_users='test@bx.psu.edu',
@@ -239,10 +243,9 @@ def setup_galaxy_config(
override_tempdir=False,
master_api_key=master_api_key,
running_functional_tests=True,
shed_tool_data_table_config=shed_tool_data_table_config,
template_cache_path=template_cache_path,
template_path='templates',
tool_config_file=tool_conf,
tool_config_file=tool_config_file,
tool_data_table_config_path=tool_data_table_config_path,
tool_parse_help=False,
tool_path=tool_path,
@@ -256,6 +259,8 @@ def setup_galaxy_config(
object_store_store_by="uuid",
simplified_workflow_run_ui="off",
)
if default_shed_tool_data_table_config:
config["shed_tool_data_table_config"] = default_shed_tool_data_table_config
if not use_shared_connection_for_amqp:
config["amqp_internal_connection"] = "sqlalchemy+sqlite:///%s?isolation_level=IMMEDIATE" % os.path.join(tmpdir, "control.sqlite")
@@ -305,16 +310,26 @@ backends:
return config
def _resolve_relative_config_paths(config_option):
# If option is not None, split into paths, resolve each w.r.t. root, then rebuild as csv string.
if config_option is not None:
resolved = []
for path in config_option.split(','):
resolved.append(os.path.join(galaxy_root, path.strip()))
return ','.join(resolved)
def _tool_data_table_config_path(default_tool_data_table_config_path=None):
tool_data_table_config_path = os.environ.get('GALAXY_TEST_TOOL_DATA_TABLE_CONF', default_tool_data_table_config_path)
if tool_data_table_config_path is None:
# ... otherise find whatever Galaxy would use as the default and
# the sample data for fucntional tests to that.
# ... otherwise find whatever Galaxy would use as the default and
# the sample data for functional tests to that.
default_tool_data_config = 'lib/galaxy/config/sample/tool_data_table_conf.xml.sample'
for tool_data_config in ['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml']:
if os.path.exists(tool_data_config):
default_tool_data_config = tool_data_config
tool_data_table_config_path = '%s,test/functional/tool-data/sample_tool_data_tables.xml' % default_tool_data_config
test_tool_data_config = 'test/functional/tool-data/sample_tool_data_tables.xml'
tool_data_table_config_path = '%s,%s' % (default_tool_data_config, test_tool_data_config)
return tool_data_table_config_path
lib/tool_shed/webapp/config.py
+4 -12
View File
@@ -31,6 +31,8 @@ TOOLSHED_CONFIG_SCHEMA_PATH = 'lib/tool_shed/webapp/config_schema.yml'
class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
default_config_file_name = 'tool_shed.yml'
add_sample_file_to_defaults = {'datatypes_config_file'}
def _load_schema(self):
return AppSchema(TOOLSHED_CONFIG_SCHEMA_PATH, TOOLSHED_APP_NAME)
@@ -42,16 +44,6 @@ class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
def shed_tool_data_path(self):
return self.tool_data_path
def parse_config_file_options(self, kwargs):
defaults = dict(
auth_config_file=[self._in_config_dir('config/auth_conf.xml')],
datatypes_config_file=[self._in_config_dir('datatypes_conf.xml'), self._in_sample_dir('datatypes_conf.xml.sample')],
shed_tool_data_table_config=[self._in_managed_config_dir('shed_tool_data_table_conf.xml')],
)
self._parse_config_file_options(defaults, dict(), kwargs)
# Backwards compatibility for names used in too many places to fix
self.datatypes_config = self.datatypes_config_file
def check(self):
# Check that required directories exist; attempt to create otherwise
paths_to_check = [
@@ -69,8 +61,8 @@ class ToolShedAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
raise ConfigurationError('File not found: %s' % self.datatypes_config)
def _process_config(self, kwargs):
# Resolve paths of other config files
self.parse_config_file_options(kwargs)
# Backwards compatibility for names used in too many places to fix
self.datatypes_config = self.datatypes_config_file
# Collect the umask and primary gid from the environment
self.umask = os.umask(0o77) # get the current umask
os.umask(self.umask) # can't get w/o set, so set it back
lib/tool_shed/webapp/config_schema.yml
+34
View File
@@ -565,3 +565,37 @@ mapping:
desc: |
The URL linked by the "Terms and Conditions" link in the "Help" menu, as well
as on the user registration and login forms and in the activation emails.
auth_config_file:
type: str
default: auth_conf.xml
path_resolves_to: config_dir
required: false
desc: |
XML config file that allows the use of different authentication providers
(e.g. LDAP) instead or in addition to local authentication (.sample is used
if default does not exist).
shed_tool_data_table_config:
type: str
default: shed_tool_data_table_conf.xml
path_resolves_to: managed_config_dir
required: false
desc: |
XML config file that contains additional data table entries for the
ToolDataTableManager. This file is automatically generated based on the
current installed tool shed repositories that contain valid
tool_data_table_conf.xml.sample files. At the time of installation, these
entries are automatically added to the following file, which is parsed and
applied to the ToolDataTableManager at server start up.
datatypes_config_file:
type: str
default: datatypes_conf.xml
path_resolves_to: config_dir
required: false
desc: |
Datatypes config file(s), defines what data (file) types are available in
Galaxy (.sample is used if default does not exist). If a datatype appears in
multiple files, the last definition is used (though the first sniffer is used
so limit sniffer definitions to one file).
test/unit/config/test_config_values.py
+50 -48
View File
@@ -11,6 +11,51 @@ from galaxy.web.formatting import expand_pretty_datetime_format
TestData = namedtuple('TestData', ('key', 'expected', 'loaded'))
@pytest.fixture(scope='module')
def appconfig():
return config.GalaxyAppConfiguration()
@pytest.fixture
def mock_config_file(monkeypatch):
# Patch this; otherwise tempfile.tempdir will be set, which is a global variable that
# defines the value of the default `dir` argument to the functions in Python's
# tempfile module - which breaks multiple tests.
monkeypatch.setattr(config.GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
# Set this to return None to force the creation of base config directories
# in _set_config_directories(). Used to test the values of these directories only.
monkeypatch.setattr(config, 'find_config_file', lambda x: None)
def test_root(appconfig):
assert appconfig.root == os.path.abspath('.')
def test_common_base_config(appconfig):
assert appconfig.shed_tools_dir == os.path.join(appconfig.data_dir, 'shed_tools')
assert appconfig.sample_config_dir == os.path.join(appconfig.root, 'lib', 'galaxy', 'config', 'sample')
def test_base_config_if_running_from_source(monkeypatch, mock_config_file):
# Simulated condition: running from source, config_file is None.
monkeypatch.setattr(config, 'running_from_source', True)
appconfig = config.GalaxyAppConfiguration()
assert not appconfig.config_file
assert appconfig.config_dir == os.path.join(appconfig.root, 'config')
assert appconfig.data_dir == os.path.join(appconfig.root, 'database')
assert appconfig.managed_config_dir == appconfig.config_dir
def test_base_config_if_running_not_from_source(monkeypatch, mock_config_file):
# Simulated condition: running not from source, config_file is None.
monkeypatch.setattr(config, 'running_from_source', False)
appconfig = config.GalaxyAppConfiguration()
assert not appconfig.config_file
assert appconfig.config_dir == os.getcwd()
assert appconfig.data_dir == os.path.join(appconfig.config_dir, 'data')
assert appconfig.managed_config_dir == os.path.join(appconfig.data_dir, 'config')
def listify_strip(value):
return listify(value, do_strip=True)
@@ -61,11 +106,13 @@ class ExpectedValues:
'citation_cache_lock_dir': self._in_data_dir('citations/locks'),
'cluster_files_directory': self._in_data_dir('pbs'),
'config_dir': self._in_config_dir(),
'containers_config_file': self._in_config_dir('containers_conf.yml'),
'data_dir': self._in_data_dir(),
'data_manager_config_file': self._in_config_dir('data_manager_conf.xml'),
'datatypes_config_file': self._in_sample_dir('datatypes_conf.xml.sample'),
'dependency_resolvers_config_file': self._in_config_dir('dependency_resolvers_conf.xml'),
'dynamic_proxy_session_map': self._in_data_dir('session_map.sqlite'),
'error_report_file': self._in_config_dir('error_report.yml'),
'file_path': self._in_data_dir('objects'),
'file_sources_config_file': self._in_config_dir('file_sources_conf.yml'),
'galaxy_data_manager_data_path': self._in_root_dir('tool-data'),
@@ -76,11 +123,13 @@ class ExpectedValues:
'job_metrics_config_file': self._in_sample_dir('job_metrics_conf.xml.sample'),
'job_resource_params_file': self._in_config_dir('job_resource_params_conf.xml'),
'len_file_path': self._in_root_dir('tool-data/shared/ucsc/chrom'),
'local_conda_mapping_file': self._in_config_dir('local_conda_mapping.yml'),
'managed_config_dir': self._in_managed_config_dir(),
'markdown_export_css': self._in_config_dir('markdown_export.css'),
'markdown_export_css_invocation_reports': self._in_config_dir('markdown_export_invocation_reports.css'),
'markdown_export_css_pages': self._in_config_dir('markdown_export_pages.css'),
'migrated_tools_config': self._in_managed_config_dir('migrated_tools_conf.xml'),
'modules_mapping_files': self._in_config_dir('environment_modules_mapping.yml'),
'mulled_resolution_cache_data_dir': self._in_data_dir('mulled/data'),
'mulled_resolution_cache_lock_dir': self._in_data_dir('mulled/locks'),
'new_file_path': self._in_data_dir('tmp'),
@@ -98,6 +147,7 @@ class ExpectedValues:
'tool_config_file': self._in_sample_dir('tool_conf.xml.sample'),
'tool_data_path': self._in_root_dir('tool-data'),
'tool_data_table_config_path': self._in_sample_dir('tool_data_table_conf.xml.sample'),
'tool_destinations_config_file': self._in_config_dir('tool_destinations.yml'),
'tool_path': self._in_root_dir('tools'),
'tool_search_index_dir': self._in_data_dir('tool_search_index'),
'tool_sheds_config_file': self._in_config_dir('tool_sheds_conf.xml'),
@@ -157,54 +207,6 @@ class ExpectedValues:
return 'sqlalchemy+sqlite:///{}/control.sqlite?isolation_level=IMMEDIATE'.format(self._config.data_dir)
@pytest.fixture
def mock_config_file(monkeypatch):
# Set this to return None to force the creation of base config directories
# in _set_config_directories(). Used to test the values of these directories only.
monkeypatch.setattr(config, 'find_config_file', lambda x: None)
@pytest.fixture
def mock_config_running_from_source(monkeypatch, mock_config_file):
# Simulated condition: running from source, config_file is None.
monkeypatch.setattr(config, 'running_from_source', True)
@pytest.fixture
def mock_config_running_not_from_source(monkeypatch, mock_config_file):
# Simulated condition: running not from source, config_file is None.
monkeypatch.setattr(config, 'running_from_source', False)
@pytest.fixture
def appconfig(monkeypatch):
monkeypatch.setattr(config.GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
return config.GalaxyAppConfiguration()
def test_root(appconfig):
assert appconfig.root == os.path.abspath('.')
def test_base_config_if_running_from_source(mock_config_running_from_source, appconfig):
assert not appconfig.config_file
assert appconfig.config_dir == os.path.join(appconfig.root, 'config')
assert appconfig.data_dir == os.path.join(appconfig.root, 'database')
assert appconfig.managed_config_dir == appconfig.config_dir
def test_base_config_if_running_not_from_source(mock_config_running_not_from_source, appconfig):
assert not appconfig.config_file
assert appconfig.config_dir == os.getcwd()
assert appconfig.data_dir == os.path.join(appconfig.config_dir, 'data')
assert appconfig.managed_config_dir == os.path.join(appconfig.data_dir, 'config')
def test_common_base_config(appconfig):
assert appconfig.shed_tools_dir == os.path.join(appconfig.data_dir, 'shed_tools')
assert appconfig.sample_config_dir == os.path.join(appconfig.root, 'lib', 'galaxy', 'config', 'sample')
def get_config_data():
config.GalaxyAppConfiguration._override_tempdir = lambda a, b: None # method must be mocked
configuration = config.GalaxyAppConfiguration()
test/unit/config/test_load_config.py
+43 -18
View File
@@ -1,9 +1,8 @@
import pytest
from galaxy.config import GalaxyAppConfiguration
from galaxy.config import BaseAppConfiguration
from galaxy.config.schema import AppSchema
MOCK_SCHEMA = {
'property1': {'default': 'a', 'type': 'str'}, # str
'property2': {'default': 1, 'type': 'int'}, # int
@@ -13,25 +12,19 @@ MOCK_SCHEMA = {
'property6': {'something_else': 'b'}, # no type
}
MOCK_RENAMED_OPTIONS = {
'old_property1': 'property1'
}
def get_schema(app_mapping):
return {'mapping': {'galaxy': {'mapping': app_mapping}}}
return {'mapping': {'_': {'mapping': app_mapping}}}
@pytest.fixture
def mock_init(monkeypatch):
monkeypatch.setattr(BaseAppConfiguration, '_load_schema', lambda a: AppSchema(None, '_'))
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(MOCK_SCHEMA))
monkeypatch.setattr(GalaxyAppConfiguration, '_process_config', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, 'renamed_options', MOCK_RENAMED_OPTIONS)
def test_load_config_from_schema(mock_init):
config = GalaxyAppConfiguration()
config = BaseAppConfiguration()
assert len(config._raw_config) == 6
assert config._raw_config['property1'] == 'a'
@@ -48,7 +41,7 @@ def test_load_config_from_schema(mock_init):
def test_update_raw_config_from_kwargs(mock_init):
config = GalaxyAppConfiguration(property2=2, property3=2.0, another_key=66)
config = BaseAppConfiguration(property2=2, property3=2.0, another_key=66)
assert len(config._raw_config) == 6 # no change: another_key NOT added
assert config._raw_config['property1'] == 'a' # no change
@@ -66,7 +59,7 @@ def test_update_raw_config_from_kwargs(mock_init):
def test_update_raw_config_from_string_kwargs(mock_init):
# kwargs may be passed as strings: property data types should not be affected
config = GalaxyAppConfiguration(property1='b', property2='2', property3='2.0', property4='false')
config = BaseAppConfiguration(property1='b', property2='2', property3='2.0', property4='false')
assert len(config._raw_config) == 6 # no change
assert config._raw_config['property1'] == 'b' # updated
@@ -82,7 +75,7 @@ def test_update_raw_config_from_string_kwargs(mock_init):
def test_update_raw_config_from_kwargs_with_none(mock_init):
# should be able to set to null regardless of property's datatype
config = GalaxyAppConfiguration(
config = BaseAppConfiguration(
property1=None, property2=None, property3=None, property4=None, property5=None, property6=None,
)
@@ -97,19 +90,51 @@ def test_update_raw_config_from_kwargs_with_none(mock_init):
def test_update_raw_config_from_kwargs_falsy_not_none(mock_init):
# if kwargs supplies a falsy value, it should not evaluate to null
# (ensures code is 'if value is not None' vs. 'if value')
config = GalaxyAppConfiguration(property1=0)
config = BaseAppConfiguration(property1=0)
assert config._raw_config['property1'] == '0' # updated
assert type(config._raw_config['property1']) is str # and converted to str
def test_unset_renamed_option_set_by_old_option(mock_init):
config = GalaxyAppConfiguration(old_property1='b')
def test_unset_renamed_option_set_by_old_option(mock_init, monkeypatch):
monkeypatch.setattr(BaseAppConfiguration, 'renamed_options', {'old_property1': 'property1'})
config = BaseAppConfiguration(old_property1='b')
assert config._raw_config['property1'] == 'b'
def test_set_renamed_option_not_overridden_by_old_option(mock_init):
config = GalaxyAppConfiguration(old_property1='b', property1='c')
config = BaseAppConfiguration(old_property1='b', property1='c')
assert config._raw_config['property1'] == 'c'
def test_is_set(mock_init):
# if an option is set from kwargs, is_set() returns True, otherwise False
# Note: is_set() here means 'value is set by user', which includes setting
# to None or setting to the same value as the schema default.
# First, test that none are set
config = BaseAppConfiguration()
assert not config.is_set('property1')
assert not config.is_set('property2')
assert not config.is_set('property3')
assert not config.is_set('property4')
assert not config.is_set('property5')
assert not config.is_set('property6')
# Now set all values, including setting to None and setting to the schema default
config = BaseAppConfiguration(
property1='b', # default = 'a' (overwrites default w/'a')
property2=None, # default = 1 (overwrites default w/None)
property3=1.0, # default = 1.0 (same as default: 1.0)
property4=True, # default = True (same as default: True)
property5=None, # default = None (same as default: None)
property6=1) # default = None (overwrites default w/None)
assert config.is_set('property1')
assert config.is_set('property2')
assert config.is_set('property3')
assert config.is_set('property4')
assert config.is_set('property4')
assert config.is_set('property6')
test/unit/config/test_path_graph.py
+10 -12
View File
@@ -1,6 +1,6 @@
import pytest
from galaxy.config import GalaxyAppConfiguration
from galaxy.config import BaseAppConfiguration
from galaxy.config.schema import AppSchema
from galaxy.exceptions import ConfigurationError
@@ -17,7 +17,7 @@ from galaxy.exceptions import ConfigurationError
def get_schema(app_mapping):
return {'mapping': {'galaxy': {'mapping': app_mapping}}}
return {'mapping': {'_': {'mapping': app_mapping}}}
def test_basecase(monkeypatch):
@@ -48,10 +48,9 @@ def test_basecase(monkeypatch):
},
}
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(mock_schema))
monkeypatch.setattr(GalaxyAppConfiguration, '_process_config', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
monkeypatch.setattr(BaseAppConfiguration, '_load_schema', lambda a: AppSchema(None, '_'))
config = GalaxyAppConfiguration()
config = BaseAppConfiguration()
assert config.component1_path0 == 'value0'
assert config.component1_path1 == 'value0/value1'
assert config.component1_path2 == 'value0/value1/value2'
@@ -75,7 +74,7 @@ def test_resolves_to_invalid_property(monkeypatch):
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(mock_schema))
with pytest.raises(ConfigurationError):
AppSchema(None, 'galaxy').validate_path_resolution_graph()
AppSchema(None, '_').validate_path_resolution_graph()
def test_path_resolution_cycle(monkeypatch):
@@ -100,7 +99,7 @@ def test_path_resolution_cycle(monkeypatch):
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(mock_schema))
with pytest.raises(ConfigurationError):
AppSchema(None, 'galaxy').validate_path_resolution_graph()
AppSchema(None, '_').validate_path_resolution_graph()
def test_path_invalid_type(monkeypatch):
@@ -119,7 +118,7 @@ def test_path_invalid_type(monkeypatch):
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(mock_schema))
with pytest.raises(ConfigurationError):
AppSchema(None, 'galaxy').validate_path_resolution_graph()
AppSchema(None, '_').validate_path_resolution_graph()
def test_resolves_to_invalid_type(monkeypatch):
@@ -138,7 +137,7 @@ def test_resolves_to_invalid_type(monkeypatch):
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(mock_schema))
with pytest.raises(ConfigurationError):
AppSchema(None, 'galaxy').validate_path_resolution_graph()
AppSchema(None, '_').validate_path_resolution_graph()
def test_resolves_with_empty_component(monkeypatch):
@@ -159,10 +158,9 @@ def test_resolves_with_empty_component(monkeypatch):
},
}
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(mock_schema))
monkeypatch.setattr(GalaxyAppConfiguration, '_process_config', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
monkeypatch.setattr(BaseAppConfiguration, '_load_schema', lambda a: AppSchema(None, '_'))
config = GalaxyAppConfiguration()
config = BaseAppConfiguration()
assert config.path0 == 'value0'
assert config.path1 == 'value0'
assert config.path2 == 'value0/value2'
test/unit/config/test_path_resolves_to.py
+83 -25
View File
@@ -1,6 +1,6 @@
import pytest
from galaxy.config import GalaxyAppConfiguration
from galaxy.config import BaseAppConfiguration
from galaxy.config.schema import AppSchema
@@ -32,56 +32,53 @@ MOCK_SCHEMA = {
'path3': {
'type': 'str',
'default': 'my-other-files',
}
},
'path4': {
'type': 'str',
'default': 'conf1, conf2, conf3',
'path_resolves_to': 'my_config_dir',
},
}
def get_schema(app_mapping):
return {'mapping': {'galaxy': {'mapping': app_mapping}}}
def test_deprecated_prefixes_set_correctly(monkeypatch):
# Before we mock them, check that correct values are assigned
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(MOCK_SCHEMA))
monkeypatch.setattr(GalaxyAppConfiguration, '_process_config', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
config = GalaxyAppConfiguration()
assert config.deprecated_dirs == {'config_dir': 'config', 'data_dir': 'database'}
return {'mapping': {'_': {'mapping': app_mapping}}}
@pytest.fixture
def mock_init(monkeypatch):
monkeypatch.setattr(BaseAppConfiguration, '_load_schema', lambda a: AppSchema(None, '_'))
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(MOCK_SCHEMA))
monkeypatch.setattr(GalaxyAppConfiguration, '_process_config', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, 'deprecated_dirs', MOCK_DEPRECATED_DIRS)
monkeypatch.setattr(BaseAppConfiguration, 'deprecated_dirs', MOCK_DEPRECATED_DIRS)
monkeypatch.setattr(BaseAppConfiguration, 'listify_options', {'path4'})
def test_mock_schema_is_loaded(mock_init):
# Check that mock is loaded as expected
config = GalaxyAppConfiguration()
assert len(config._raw_config) == 5
config = BaseAppConfiguration()
assert len(config._raw_config) == 6
assert config._raw_config['my_config_dir'] == 'my-config'
assert config._raw_config['my_data_dir'] == 'my-data'
assert config._raw_config['path1'] == 'my-config-files'
assert config._raw_config['path2'] == 'my-data-files'
assert config._raw_config['path3'] == 'my-other-files'
assert config._raw_config['path4'] == 'conf1, conf2, conf3'
def test_no_kwargs(mock_init):
# Expected: use default from schema, then resolve
config = GalaxyAppConfiguration()
config = BaseAppConfiguration()
assert config.path1 == 'my-config/my-config-files' # resolved
assert config.path2 == 'my-data/my-data-files' # resolved
assert config.path3 == 'my-other-files' # no change
assert config.path4 == ['my-config/conf3'] # last value resolved and listified; others dropped as files do not exist
def test_kwargs_relative_path(mock_init):
# Expected: use value from kwargs, then resolve
new_path1 = 'foo1/bar'
new_path2 = 'foo2/bar'
config = GalaxyAppConfiguration(path1=new_path1, path2=new_path2)
config = BaseAppConfiguration(path1=new_path1, path2=new_path2)
assert config.path1 == 'my-config/' + new_path1 # resolved
assert config.path2 == 'my-data/' + new_path2 # resolved
@@ -92,7 +89,7 @@ def test_kwargs_ablsolute_path(mock_init):
# Expected: use value from kwargs, do NOT resolve
new_path1 = '/foo1/bar'
new_path2 = '/foo2/bar'
config = GalaxyAppConfiguration(path1=new_path1, path2=new_path2)
config = BaseAppConfiguration(path1=new_path1, path2=new_path2)
assert config.path1 == new_path1 # NOT resolved
assert config.path2 == new_path2 # NOT resolved
@@ -103,7 +100,7 @@ def test_kwargs_relative_path_old_prefix(mock_init):
# Expect: use value from kwargs, strip old prefix, then resolve
new_path1 = 'old-config/foo1/bar'
new_path2 = 'old-database/foo2/bar'
config = GalaxyAppConfiguration(path1=new_path1, path2=new_path2)
config = BaseAppConfiguration(path1=new_path1, path2=new_path2)
assert config.path1 == 'my-config/foo1/bar' # stripped of old prefix, resolved
assert config.path2 == 'my-data/foo2/bar' # stripped of old prefix, resolved
@@ -116,7 +113,7 @@ def test_kwargs_relative_path_old_prefix_for_other_option(mock_init):
# (deprecated for the config_dir option) if it's used for another option
new_path1 = 'old-database/foo1/bar'
new_path2 = 'old-config/foo2/bar'
config = GalaxyAppConfiguration(path1=new_path1, path2=new_path2)
config = BaseAppConfiguration(path1=new_path1, path2=new_path2)
assert config.path1 == 'my-config/' + new_path1 # resolved
assert config.path2 == 'my-data/' + new_path2 # resolved
@@ -126,7 +123,7 @@ def test_kwargs_relative_path_old_prefix_for_other_option(mock_init):
def test_kwargs_relative_path_old_prefix_empty_after_strip(mock_init):
# Expect: use value from kwargs, strip old prefix, then resolve
new_path1 = 'old-config'
config = GalaxyAppConfiguration(path1=new_path1)
config = BaseAppConfiguration(path1=new_path1)
assert config.path1 == 'my-config/' # stripped of old prefix, then resolved
assert config.path2 == 'my-data/my-data-files' # stripped of old prefix, then resolved
@@ -137,8 +134,69 @@ def test_kwargs_set_to_null(mock_init):
# Expected: allow overriding with null, then resolve
# This is not a common scenario, but it does happen: one example is
# `job_config` set to `None` when testing
config = GalaxyAppConfiguration(path1=None)
config = BaseAppConfiguration(path1=None)
assert config.path1 == 'my-config' # resolved
assert config.path2 == 'my-data/my-data-files' # resolved
assert config.path3 == 'my-other-files' # no change
def test_add_sample_file(mock_init, monkeypatch):
# Expected: sample file appended to list of defaults:
# - resolved w.r.t sample-dir (_in_sample_dir mocked)
# - has ".sample" suffix
# Last value (sample file) resolved and listified; others dropped as files do not exist
monkeypatch.setattr(BaseAppConfiguration, 'add_sample_file_to_defaults', {'path1', 'path4'})
monkeypatch.setattr(BaseAppConfiguration, '_in_sample_dir', lambda a, path: '/sample-dir/%s' % path)
config = BaseAppConfiguration()
assert config._raw_config['path1'] == 'my-config-files'
assert config.path1 == '/sample-dir/my-config-files.sample'
assert config._raw_config['path4'] == 'conf1, conf2, conf3'
assert config.path4 == ['/sample-dir/conf3.sample']
def test_select_one_path_from_list(mock_init, monkeypatch):
# Expected: files do not exist, so use last file in list (would be sample file); value is not a list
monkeypatch.setattr(BaseAppConfiguration, 'add_sample_file_to_defaults', {'path1'})
monkeypatch.setattr(BaseAppConfiguration, '_in_sample_dir', lambda a, path: '/sample-dir/%s' % path)
config = BaseAppConfiguration()
assert config._raw_config['path1'] == 'my-config-files'
assert config.path1 == '/sample-dir/my-config-files.sample'
def test_select_one_path_from_list_all_files_exist(mock_init, monkeypatch):
# Expected: all files exist, so use first file in list; value is not a list
monkeypatch.setattr(BaseAppConfiguration, 'add_sample_file_to_defaults', {'path1'})
monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda a, b: True)
config = BaseAppConfiguration()
assert config._raw_config['path1'] == 'my-config-files'
assert config.path1 == 'my-config/my-config-files'
def test_no_kwargs_listify(mock_init, monkeypatch):
# Expected: last value resolved and listified; others dropped as files do not exist
config = BaseAppConfiguration()
assert config._raw_config['path4'] == 'conf1, conf2, conf3'
assert config.path4 == ['my-config/conf3']
def test_no_kwargs_listify_all_files_exist(mock_init, monkeypatch):
# Expected: each value resolved and listified (mock: all files exist)
monkeypatch.setattr(BaseAppConfiguration, '_path_exists', lambda a, b: True)
config = BaseAppConfiguration()
assert config._raw_config['path4'] == 'conf1, conf2, conf3'
assert config.path4 == ['my-config/conf1', 'my-config/conf2', 'my-config/conf3']
def test_kwargs_listify(mock_init, monkeypatch):
# Expected: use values from kwargs; each value resolved and listified
new_path4 = 'new1, new2'
config = BaseAppConfiguration(path4=new_path4)
assert config._raw_config['path4'] == 'new1, new2'
assert config.path4 == ['my-config/new1', 'my-config/new2']
test/unit/config/test_reload_config.py
+6 -8
View File
@@ -1,7 +1,7 @@
import pytest
from galaxy import config
from galaxy.config import GalaxyAppConfiguration
from galaxy.config import BaseAppConfiguration
from galaxy.config import reload_config_options
from galaxy.config.schema import AppSchema
@@ -17,15 +17,13 @@ MOCK_SCHEMA = {
def get_schema(app_mapping):
return {'mapping': {'galaxy': {'mapping': app_mapping}}}
return {'mapping': {'_': {'mapping': app_mapping}}}
@pytest.fixture
def mock_init(monkeypatch):
monkeypatch.setattr(BaseAppConfiguration, '_load_schema', lambda a: AppSchema(None, '_'))
monkeypatch.setattr(AppSchema, '_read_schema', lambda a, b: get_schema(MOCK_SCHEMA))
monkeypatch.setattr(GalaxyAppConfiguration, '_process_config', lambda a, b: None)
monkeypatch.setattr(GalaxyAppConfiguration, '_override_tempdir', lambda a, b: None)
def test_update_property(mock_init, monkeypatch):
@@ -36,7 +34,7 @@ def test_update_property(mock_init, monkeypatch):
# edits to config file: R2, N1 modified
monkeypatch.setattr(config, 'read_properties_from_file', lambda _: {R1: 1, R2: 42, N1: 99})
appconfig = GalaxyAppConfiguration()
appconfig = BaseAppConfiguration()
assert getattr(appconfig, R1) == 1
assert getattr(appconfig, R2) == 2
@@ -61,7 +59,7 @@ def test_overwrite_reloadable_attribute(mock_init, monkeypatch):
# edits to config file: R2 modified
monkeypatch.setattr(config, 'read_properties_from_file', lambda _: {R1: 1, R2: 42})
appconfig = GalaxyAppConfiguration()
appconfig = BaseAppConfiguration()
assert getattr(appconfig, R1) == 1
assert getattr(appconfig, R2) == 2
@@ -83,7 +81,7 @@ def test_cant_delete_property(mock_init, monkeypatch):
# edits to config file: R2, N2 deleted
monkeypatch.setattr(config, 'read_properties_from_file', lambda _: {R1: 1, N1: 3})
appconfig = GalaxyAppConfiguration()
appconfig = BaseAppConfiguration()
assert getattr(appconfig, R1) == 1
assert getattr(appconfig, R2) == 2
test/unit/unittest_utils/galaxy_mock.py
+3
View File
@@ -180,6 +180,9 @@ class MockAppConfig(Bunch):
return self.dict()
def __getattr__(self, name):
# Handle the automatic [option]_set options: for tests, assume none are set
if name == 'is_set':
return lambda x: False
# Handle the automatic config file _set options
if name.endswith('_file_set'):
return False
test/unit/webapps/test_webapp_base.py
+2 -1
View File
@@ -24,7 +24,8 @@ class CORSParsingMockConfig(galaxy_mock.MockAppConfig):
@staticmethod
def _parse_allowed_origin_hostnames(kwargs):
return galaxy.config.Configuration._parse_allowed_origin_hostnames(kwargs)
hostnames = kwargs.get('allowed_origin_hostnames')
return galaxy.config.Configuration._parse_allowed_origin_hostnames(hostnames)
class GalaxyWebTransaction_Headers_TestCase(unittest.TestCase):