From 6b2220e1ac16eeec566344dd272bdb12f75113d5 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 21 Jan 2021 10:30:12 -0500 Subject: [PATCH 1/2] Add cache_dir to schema, rebuild --- doc/source/admin/galaxy_options.rst | 28 +++++++++++++++------ lib/galaxy/config/sample/galaxy.yml.sample | 21 ++++++++++------ lib/galaxy/webapps/galaxy/config_schema.yml | 23 +++++++++++------ test/unit/config/test_config_values.py | 18 +++++++------ 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 6469d27017f..89b1648ae13 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -37,6 +37,20 @@ :Type: str +~~~~~~~~~~~~~ +``cache_dir`` +~~~~~~~~~~~~~ + +:Description: + Top level cache directory. Any other cache directories + (tool_cache_data_dir, template_cache_path, etc.) should be + subdirectories. + The value of this option will be resolved with respect to + . +:Default: ``cache`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~ ``database_connection`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -995,7 +1009,7 @@ Mako templates are compiled as needed and cached for reuse, this directory is used for the cache The value of this option will be resolved with respect to - . + . :Default: ``compiled_templates`` :Type: str @@ -1074,7 +1088,7 @@ configured in (``shed_``)tool_conf.xml files using the tool_cache_data_dir attribute. The value of this option will be resolved with respect to - . + . :Default: ``tool_cache`` :Type: str @@ -1125,7 +1139,7 @@ the following parameters can be used to control the caching used to store this information. The value of this option will be resolved with respect to - . + . :Default: ``citations/data`` :Type: str @@ -1140,7 +1154,7 @@ the following parameters can be used to control the caching used to store this information. The value of this option will be resolved with respect to - . + . :Default: ``citations/locks`` :Type: str @@ -1165,7 +1179,7 @@ Data directory used by beaker for caching mulled resolution requests. The value of this option will be resolved with respect to - . + . :Default: ``mulled/data`` :Type: str @@ -1178,7 +1192,7 @@ Lock directory used by beaker for caching mulled resolution requests. The value of this option will be resolved with respect to - . + . :Default: ``mulled/locks`` :Type: str @@ -3463,7 +3477,7 @@ :Description: If OpenID is enabled, consumer cache directory to use. The value of this option will be resolved with respect to - . + . :Default: ``openid_consumer_cache`` :Type: str diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 7fccb510097..a6d15301c30 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -128,6 +128,13 @@ galaxy: # if running Galaxy from source or `/data` otherwise. #data_dir: null + # Top level cache directory. Any other cache directories + # (tool_cache_data_dir, template_cache_path, etc.) should be + # subdirectories. + # The value of this option will be resolved with respect to + # . + #cache_dir: cache + # By default, Galaxy uses a SQLite database at # '/universe.sqlite'. You may use a SQLAlchemy connection # string to specify an external database instead. @@ -593,7 +600,7 @@ galaxy: # Mako templates are compiled as needed and cached for reuse, this # directory is used for the cache # The value of this option will be resolved with respect to - # . + # . #template_cache_path: compiled_templates # Set to false to disable various checks Galaxy will do to ensure it @@ -630,7 +637,7 @@ galaxy: # in (``shed_``)tool_conf.xml files using the tool_cache_data_dir # attribute. # The value of this option will be resolved with respect to - # . + # . #tool_cache_data_dir: tool_cache # Directory in which the toolbox search index is stored. The value of @@ -653,7 +660,7 @@ galaxy: # following parameters can be used to control the caching used to # store this information. # The value of this option will be resolved with respect to - # . + # . #citation_cache_data_dir: citations/data # Citation related caching. Tool citations information maybe fetched @@ -661,7 +668,7 @@ galaxy: # following parameters can be used to control the caching used to # store this information. # The value of this option will be resolved with respect to - # . + # . #citation_cache_lock_dir: citations/locks # Mulled resolution caching. Mulled resolution uses external APIs of @@ -672,13 +679,13 @@ galaxy: # Data directory used by beaker for caching mulled resolution # requests. # The value of this option will be resolved with respect to - # . + # . #mulled_resolution_cache_data_dir: mulled/data # Lock directory used by beaker for caching mulled resolution # requests. # The value of this option will be resolved with respect to - # . + # . #mulled_resolution_cache_lock_dir: mulled/locks # Configuration file for the object store If this is set and exists, @@ -1724,7 +1731,7 @@ galaxy: # If OpenID is enabled, consumer cache directory to use. # The value of this option will be resolved with respect to - # . + # . #openid_consumer_cache_path: openid_consumer_cache # Enable tool tags (associating tools with tags). This has its own diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 5471ac209ae..9e3c05117fc 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -52,6 +52,15 @@ mapping: file_path, etc.). Defaults to `database/` if running Galaxy from source or `/data` otherwise. + cache_dir: + type: str + default: cache + path_resolves_to: data_dir + required: false + desc: | + Top level cache directory. Any other cache directories (tool_cache_data_dir, + template_cache_path, etc.) should be subdirectories. + database_connection: type: str required: false @@ -725,7 +734,7 @@ mapping: template_cache_path: type: str default: compiled_templates - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | Mako templates are compiled as needed and cached for reuse, this directory is @@ -779,7 +788,7 @@ mapping: tool_cache_data_dir: type: str default: tool_cache - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | Tool related caching. Fully expanded tools and metadata will be stored at this path. @@ -815,7 +824,7 @@ mapping: citation_cache_data_dir: type: str default: citations/data - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | Citation related caching. Tool citations information maybe fetched from @@ -825,7 +834,7 @@ mapping: citation_cache_lock_dir: type: str default: citations/locks - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | Citation related caching. Tool citations information maybe fetched from @@ -843,7 +852,7 @@ mapping: mulled_resolution_cache_data_dir: type: str default: mulled/data - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | Data directory used by beaker for caching mulled resolution requests. @@ -851,7 +860,7 @@ mapping: mulled_resolution_cache_lock_dir: type: str default: mulled/locks - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | Lock directory used by beaker for caching mulled resolution requests. @@ -2525,7 +2534,7 @@ mapping: openid_consumer_cache_path: type: str default: openid_consumer_cache - path_resolves_to: data_dir + path_resolves_to: cache_dir required: false desc: | If OpenID is enabled, consumer cache directory to use. diff --git a/test/unit/config/test_config_values.py b/test/unit/config/test_config_values.py index 07c11ba0e37..b0cfa1856d9 100644 --- a/test/unit/config/test_config_values.py +++ b/test/unit/config/test_config_values.py @@ -102,8 +102,9 @@ class ExpectedValues: 'beta_edam_toolbox_ontology_path': self._in_data_dir('EDAM.tsv'), 'build_sites_config_file': self._in_sample_dir('build_sites.yml.sample'), 'builds_file_path': self._in_root_dir('tool-data/shared/ucsc/builds.txt'), - 'citation_cache_data_dir': self._in_data_dir('citations/data'), - 'citation_cache_lock_dir': self._in_data_dir('citations/locks'), + 'cache_dir': self._in_data_dir('cache'), + 'citation_cache_data_dir': self._in_cache_dir('citations/data'), + 'citation_cache_lock_dir': self._in_cache_dir('citations/locks'), 'cluster_files_directory': self._in_data_dir('pbs'), 'config_dir': self._in_config_dir(), 'containers_config_file': self._in_config_dir('containers_conf.yml'), @@ -130,20 +131,20 @@ class ExpectedValues: 'markdown_export_css_pages': self._in_config_dir('markdown_export_pages.css'), 'migrated_tools_config': self._in_managed_config_dir('migrated_tools_conf.xml'), 'modules_mapping_files': self._in_config_dir('environment_modules_mapping.yml'), - 'mulled_resolution_cache_data_dir': self._in_data_dir('mulled/data'), - 'mulled_resolution_cache_lock_dir': self._in_data_dir('mulled/locks'), + 'mulled_resolution_cache_data_dir': self._in_cache_dir('mulled/data'), + 'mulled_resolution_cache_lock_dir': self._in_cache_dir('mulled/locks'), 'new_file_path': self._in_data_dir('tmp'), 'object_store_config_file': self._in_config_dir('object_store_conf.xml'), 'oidc_backends_config_file': self._in_config_dir('oidc_backends_config.xml'), 'oidc_config_file': self._in_config_dir('oidc_config.xml'), - 'openid_consumer_cache_path': self._in_data_dir('openid_consumer_cache'), + 'openid_consumer_cache_path': self._in_cache_dir('openid_consumer_cache'), 'sanitize_allowlist_file': self._in_managed_config_dir('sanitize_allowlist.txt'), 'shed_data_manager_config_file': self._in_managed_config_dir('shed_data_manager_conf.xml'), 'shed_tool_config_file': self._in_managed_config_dir('shed_tool_conf.xml'), 'shed_tool_data_path': self._in_root_dir('tool-data'), 'shed_tool_data_table_config': self._in_managed_config_dir('shed_tool_data_table_conf.xml'), - 'template_cache_path': self._in_data_dir('compiled_templates'), - 'tool_cache_data_dir': self._in_data_dir('tool_cache'), + 'template_cache_path': self._in_cache_dir('compiled_templates'), + 'tool_cache_data_dir': self._in_cache_dir('tool_cache'), 'tool_config_file': self._in_sample_dir('tool_conf.xml.sample'), 'tool_data_path': self._in_root_dir('tool-data'), 'tool_data_table_config_path': self._in_sample_dir('tool_data_table_conf.xml.sample'), @@ -171,6 +172,9 @@ class ExpectedValues: def _in_config_dir(self, path=None): return self._in_dir(self._config.config_dir, path) + def _in_cache_dir(self, path=None): + return self._in_dir(self._config.cache_dir, path) + def _in_data_dir(self, path=None): return self._in_dir(self._config.data_dir, path) From d7bec8796e07f255e75c526d72181ce52d11aa6e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 21 Jan 2021 14:10:49 -0500 Subject: [PATCH 2/2] Fix integration test --- test/integration/test_config_defaults.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/integration/test_config_defaults.py b/test/integration/test_config_defaults.py index 7bcdd768e0c..130034bae00 100644 --- a/test/integration/test_config_defaults.py +++ b/test/integration/test_config_defaults.py @@ -37,6 +37,8 @@ from galaxy_test.driver.driver_util import GalaxyTestDriver OptionData = namedtuple('OptionData', ('key', 'expected', 'loaded')) +driver_created = False + # Configuration properties that are paths should be absolute paths, by default resolved w.r.t root. PATH_CONFIG_PROPERTIES = [ # For now, these include base config properties @@ -44,6 +46,7 @@ PATH_CONFIG_PROPERTIES = [ 'config_file', 'config_dir', 'managed_config_dir', + 'cache_dir', 'data_dir', 'auth_config_file', 'email_domain_blocklist_file', @@ -226,9 +229,12 @@ def create_driver(): # Ideally `create_driver` would be a fixture and clean up after the yield, # but that's not compatible with the use use of pytest.mark.parametrize: # a fixture is not directly callable, so it cannot be used in place of get_config_data. - global DRIVER - DRIVER = GalaxyTestDriver() - DRIVER.setup() + global driver_created + if not driver_created: + global DRIVER + DRIVER = GalaxyTestDriver() + DRIVER.setup() + driver_created = True def get_config_data(): @@ -240,6 +246,7 @@ def get_config_data(): 'managed_config_dir': DRIVER.app.config.managed_config_dir, 'data_dir': DRIVER.app.config.data_dir, 'tool_data_path': DRIVER.app.config.tool_data_path, + 'cache_dir': DRIVER.app.config.cache_dir } def resolve(parent, child): @@ -269,6 +276,7 @@ def get_config_data(): def get_path_data(): + create_driver() # create + setup DRIVER yield from PATH_CONFIG_PROPERTIES