Merge pull request #11202 from ic4f/dev_cache_dir

Add cache_dir to schema

doc/source/admin/galaxy_options.rst

@@ -37,6 +37,20 @@
 :Type: str
 
 
+~~~~~~~~~~~~~
+``cache_dir``
+~~~~~~~~~~~~~
+
+:Description:
+    Top level cache directory. Any other cache directories
+    (tool_cache_data_dir, template_cache_path, etc.) should be
+    subdirectories.
+    The value of this option will be resolved with respect to
+    <data_dir>.
+:Default: ``cache``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~
 ``database_connection``
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -995,7 +1009,7 @@
     Mako templates are compiled as needed and cached for reuse, this
     directory is used for the cache
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``compiled_templates``
 :Type: str
 
@@ -1074,7 +1088,7 @@
     configured in (``shed_``)tool_conf.xml files using the
     tool_cache_data_dir attribute.
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``tool_cache``
 :Type: str
 
@@ -1125,7 +1139,7 @@
     the following parameters can be used to control the caching used
     to store this information.
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``citations/data``
 :Type: str
 
@@ -1140,7 +1154,7 @@
     the following parameters can be used to control the caching used
     to store this information.
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``citations/locks``
 :Type: str
 
@@ -1165,7 +1179,7 @@
     Data directory used by beaker for caching mulled resolution
     requests.
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``mulled/data``
 :Type: str
 
@@ -1178,7 +1192,7 @@
     Lock directory used by beaker for caching mulled resolution
     requests.
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``mulled/locks``
 :Type: str
 
@@ -3463,7 +3477,7 @@
 :Description:
     If OpenID is enabled, consumer cache directory to use.
     The value of this option will be resolved with respect to
-    <data_dir>.
+    <cache_dir>.
 :Default: ``openid_consumer_cache``
 :Type: str
 

lib/galaxy/config/sample/galaxy.yml.sample

@@ -128,6 +128,13 @@ galaxy:
   # if running Galaxy from source or `<config_dir>/data` otherwise.
   #data_dir: null
 
+  # Top level cache directory. Any other cache directories
+  # (tool_cache_data_dir, template_cache_path, etc.) should be
+  # subdirectories.
+  # The value of this option will be resolved with respect to
+  # <data_dir>.
+  #cache_dir: cache
+
   # By default, Galaxy uses a SQLite database at
   # '<data_dir>/universe.sqlite'.  You may use a SQLAlchemy connection
   # string to specify an external database instead.
@@ -593,7 +600,7 @@ galaxy:
   # Mako templates are compiled as needed and cached for reuse, this
   # directory is used for the cache
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #template_cache_path: compiled_templates
 
   # Set to false to disable various checks Galaxy will do to ensure it
@@ -630,7 +637,7 @@ galaxy:
   # in (``shed_``)tool_conf.xml files using the tool_cache_data_dir
   # attribute.
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #tool_cache_data_dir: tool_cache
 
   # Directory in which the toolbox search index is stored. The value of
@@ -653,7 +660,7 @@ galaxy:
   # following parameters can be used to control the caching used to
   # store this information.
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #citation_cache_data_dir: citations/data
 
   # Citation related caching.  Tool citations information maybe fetched
@@ -661,7 +668,7 @@ galaxy:
   # following parameters can be used to control the caching used to
   # store this information.
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #citation_cache_lock_dir: citations/locks
 
   # Mulled resolution caching. Mulled resolution uses external APIs of
@@ -672,13 +679,13 @@ galaxy:
   # Data directory used by beaker for caching mulled resolution
   # requests.
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #mulled_resolution_cache_data_dir: mulled/data
 
   # Lock directory used by beaker for caching mulled resolution
   # requests.
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #mulled_resolution_cache_lock_dir: mulled/locks
 
   # Configuration file for the object store If this is set and exists,
@@ -1724,7 +1731,7 @@ galaxy:
 
   # If OpenID is enabled, consumer cache directory to use.
   # The value of this option will be resolved with respect to
-  # <data_dir>.
+  # <cache_dir>.
   #openid_consumer_cache_path: openid_consumer_cache
 
   # Enable tool tags (associating tools with tags).  This has its own

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -52,6 +52,15 @@ mapping:
           file_path, etc.). Defaults to `database/` if running Galaxy from source or
           `<config_dir>/data` otherwise.
 
+      cache_dir:
+        type: str
+        default: cache
+        path_resolves_to: data_dir
+        required: false
+        desc: |
+          Top level cache directory. Any other cache directories (tool_cache_data_dir,
+          template_cache_path, etc.) should be subdirectories.
+
       database_connection:
         type: str
         required: false
@@ -725,7 +734,7 @@ mapping:
       template_cache_path:
         type: str
         default: compiled_templates
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           Mako templates are compiled as needed and cached for reuse, this directory is
@@ -779,7 +788,7 @@ mapping:
       tool_cache_data_dir:
         type: str
         default: tool_cache
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           Tool related caching. Fully expanded tools and metadata will be stored at this path.
@@ -815,7 +824,7 @@ mapping:
       citation_cache_data_dir:
         type: str
         default: citations/data
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           Citation related caching.  Tool citations information maybe fetched from
@@ -825,7 +834,7 @@ mapping:
       citation_cache_lock_dir:
         type: str
         default: citations/locks
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           Citation related caching.  Tool citations information maybe fetched from
@@ -843,7 +852,7 @@ mapping:
       mulled_resolution_cache_data_dir:
         type: str
         default: mulled/data
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           Data directory used by beaker for caching mulled resolution requests.
@@ -851,7 +860,7 @@ mapping:
       mulled_resolution_cache_lock_dir:
         type: str
         default: mulled/locks
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           Lock directory used by beaker for caching mulled resolution requests.
@@ -2525,7 +2534,7 @@ mapping:
       openid_consumer_cache_path:
         type: str
         default: openid_consumer_cache
-        path_resolves_to: data_dir
+        path_resolves_to: cache_dir
         required: false
         desc: |
           If OpenID is enabled, consumer cache directory to use.

test/integration/test_config_defaults.py

@@ -37,6 +37,8 @@ from galaxy_test.driver.driver_util import GalaxyTestDriver
 
 OptionData = namedtuple('OptionData', ('key', 'expected', 'loaded'))
 
+driver_created = False
+
 # Configuration properties that are paths should be absolute paths, by default resolved w.r.t root.
 PATH_CONFIG_PROPERTIES = [
     # For now, these include base config properties
@@ -44,6 +46,7 @@ PATH_CONFIG_PROPERTIES = [
     'config_file',
     'config_dir',
     'managed_config_dir',
+    'cache_dir',
     'data_dir',
     'auth_config_file',
     'email_domain_blocklist_file',
@@ -226,9 +229,12 @@ def create_driver():
     # Ideally `create_driver` would be a fixture and clean up after the yield,
     # but that's not compatible with the use use of pytest.mark.parametrize:
     # a fixture is not directly callable, so it cannot be used in place of get_config_data.
-    global DRIVER
-    DRIVER = GalaxyTestDriver()
-    DRIVER.setup()
+    global driver_created
+    if not driver_created:
+        global DRIVER
+        DRIVER = GalaxyTestDriver()
+        DRIVER.setup()
+        driver_created = True
 
 
 def get_config_data():
@@ -240,6 +246,7 @@ def get_config_data():
             'managed_config_dir': DRIVER.app.config.managed_config_dir,
             'data_dir': DRIVER.app.config.data_dir,
             'tool_data_path': DRIVER.app.config.tool_data_path,
+            'cache_dir': DRIVER.app.config.cache_dir
         }
 
     def resolve(parent, child):
@@ -269,6 +276,7 @@ def get_config_data():
 
 
 def get_path_data():
+    create_driver()  # create + setup DRIVER
     yield from PATH_CONFIG_PROPERTIES
 
 

test/unit/config/test_config_values.py

@@ -102,8 +102,9 @@ class ExpectedValues:
             'beta_edam_toolbox_ontology_path': self._in_data_dir('EDAM.tsv'),
             'build_sites_config_file': self._in_sample_dir('build_sites.yml.sample'),
             'builds_file_path': self._in_root_dir('tool-data/shared/ucsc/builds.txt'),
-            'citation_cache_data_dir': self._in_data_dir('citations/data'),
-            'citation_cache_lock_dir': self._in_data_dir('citations/locks'),
+            'cache_dir': self._in_data_dir('cache'),
+            'citation_cache_data_dir': self._in_cache_dir('citations/data'),
+            'citation_cache_lock_dir': self._in_cache_dir('citations/locks'),
             'cluster_files_directory': self._in_data_dir('pbs'),
             'config_dir': self._in_config_dir(),
             'containers_config_file': self._in_config_dir('containers_conf.yml'),
@@ -130,20 +131,20 @@ class ExpectedValues:
             'markdown_export_css_pages': self._in_config_dir('markdown_export_pages.css'),
             'migrated_tools_config': self._in_managed_config_dir('migrated_tools_conf.xml'),
             'modules_mapping_files': self._in_config_dir('environment_modules_mapping.yml'),
-            'mulled_resolution_cache_data_dir': self._in_data_dir('mulled/data'),
-            'mulled_resolution_cache_lock_dir': self._in_data_dir('mulled/locks'),
+            'mulled_resolution_cache_data_dir': self._in_cache_dir('mulled/data'),
+            'mulled_resolution_cache_lock_dir': self._in_cache_dir('mulled/locks'),
             'new_file_path': self._in_data_dir('tmp'),
             'object_store_config_file': self._in_config_dir('object_store_conf.xml'),
             'oidc_backends_config_file': self._in_config_dir('oidc_backends_config.xml'),
             'oidc_config_file': self._in_config_dir('oidc_config.xml'),
-            'openid_consumer_cache_path': self._in_data_dir('openid_consumer_cache'),
+            'openid_consumer_cache_path': self._in_cache_dir('openid_consumer_cache'),
             'sanitize_allowlist_file': self._in_managed_config_dir('sanitize_allowlist.txt'),
             'shed_data_manager_config_file': self._in_managed_config_dir('shed_data_manager_conf.xml'),
             'shed_tool_config_file': self._in_managed_config_dir('shed_tool_conf.xml'),
             'shed_tool_data_path': self._in_root_dir('tool-data'),
             'shed_tool_data_table_config': self._in_managed_config_dir('shed_tool_data_table_conf.xml'),
-            'template_cache_path': self._in_data_dir('compiled_templates'),
-            'tool_cache_data_dir': self._in_data_dir('tool_cache'),
+            'template_cache_path': self._in_cache_dir('compiled_templates'),
+            'tool_cache_data_dir': self._in_cache_dir('tool_cache'),
             'tool_config_file': self._in_sample_dir('tool_conf.xml.sample'),
             'tool_data_path': self._in_root_dir('tool-data'),
             'tool_data_table_config_path': self._in_sample_dir('tool_data_table_conf.xml.sample'),
@@ -171,6 +172,9 @@ class ExpectedValues:
     def _in_config_dir(self, path=None):
         return self._in_dir(self._config.config_dir, path)
 
+    def _in_cache_dir(self, path=None):
+        return self._in_dir(self._config.cache_dir, path)
+
     def _in_data_dir(self, path=None):
         return self._in_dir(self._config.data_dir, path)