From 27c49d284a8bcf283f8037b97205d2600fd4aa41 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 18 May 2023 11:34:22 -0400 Subject: [PATCH] Better (IMO) separation of concerns when disabling cache monitor for set_metadata. --- lib/galaxy/metadata/set_metadata.py | 4 +++- lib/galaxy/objectstore/__init__.py | 8 +++++--- lib/galaxy/objectstore/azure_blob.py | 3 ++- lib/galaxy/objectstore/caching.py | 7 +++++++ lib/galaxy/objectstore/cloud.py | 4 ++-- lib/galaxy/objectstore/s3.py | 4 ++-- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index fbfb2db78f0..53c7a38c8ed 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -145,7 +145,9 @@ def get_object_store(tool_job_working_directory, object_store=None): with open(object_store_conf_path) as f: config_dict = json.load(f) assert config_dict is not None - object_store = build_object_store_from_config(None, config_dict=config_dict) + # build an object store but disable any process management associated with it + # we're using it as a library - not as a service. + object_store = build_object_store_from_config(None, config_dict=config_dict, disable_process_management=True) Dataset.object_store = object_store return object_store diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index f02848d86c6..0e9ca240d9d 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -1364,7 +1364,9 @@ def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[Ba return objectstore_class, objectstore_constructor_kwds -def build_object_store_from_config(config, fsmon=False, config_xml=None, config_dict=None): +def build_object_store_from_config( + config, fsmon=False, config_xml=None, config_dict=None, disable_process_management=False +): """ Invoke the appropriate object store. @@ -1378,8 +1380,8 @@ def build_object_store_from_config(config, fsmon=False, config_xml=None, config_ from_object = "xml" if config is None and config_dict is not None and "config" in config_dict: - # Build a config object from to_dict of an ObjectStore. - config = Bunch(**config_dict["config"]) + # Build an application config object from to_dict of an ObjectStore. + config = Bunch(disable_process_management=disable_process_management, **config_dict["config"]) elif config is None: raise Exception( "build_object_store_from_config sent None as config parameter and one cannot be recovered from config_dict" diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py index 00e9b9f2e97..7f4a212fceb 100644 --- a/lib/galaxy/objectstore/azure_blob.py +++ b/lib/galaxy/objectstore/azure_blob.py @@ -29,6 +29,7 @@ from galaxy.util.path import safe_relpath from . import ConcreteObjectStore from .caching import ( CacheTarget, + enable_cache_monitor, InProcessCacheMonitor, parse_caching_config_dict_from_xml, ) @@ -98,7 +99,7 @@ class AzureBlobObjectStore(ConcreteObjectStore): auth_dict = config_dict["auth"] container_dict = config_dict["container"] cache_dict = config_dict.get("cache") or {} - self.enable_cache_monitor = config_dict.get("enable_cache_monitor", True) + self.enable_cache_monitor = enable_cache_monitor(config, config_dict) self.account_name = auth_dict.get("account_name") self.account_key = auth_dict.get("account_key") diff --git a/lib/galaxy/objectstore/caching.py b/lib/galaxy/objectstore/caching.py index 36b9dd44748..01a12488b0a 100644 --- a/lib/galaxy/objectstore/caching.py +++ b/lib/galaxy/objectstore/caching.py @@ -115,6 +115,13 @@ def parse_caching_config_dict_from_xml(config_xml): return cache_dict +def enable_cache_monitor(config, config_dict): + if getattr(config, "disable_process_management", False): + return True + return config_dict.get("enable_cache_monitor", True) + + + class InProcessCacheMonitor: def __init__(self, cache_target: CacheTarget, interval: int = 30, initial_sleep: Optional[int] = 2): # This Event object is initialized to False diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py index 78be31c83d9..f7060d269b9 100644 --- a/lib/galaxy/objectstore/cloud.py +++ b/lib/galaxy/objectstore/cloud.py @@ -24,6 +24,7 @@ from galaxy.util import ( from . import ConcreteObjectStore from .caching import ( CacheTarget, + enable_cache_monitor, InProcessCacheMonitor, ) from .s3 import parse_config_xml @@ -66,7 +67,6 @@ class CloudConfigMixin: "size": self.cache_size, "path": self.staging_path, }, - "enable_cache_monitor": False, } @@ -87,7 +87,7 @@ class Cloud(ConcreteObjectStore, CloudConfigMixin): bucket_dict = config_dict["bucket"] connection_dict = config_dict.get("connection", {}) cache_dict = config_dict.get("cache") or {} - self.enable_cache_monitor = config_dict.get("enable_cache_monitor", True) + self.enable_cache_monitor = enable_cache_monitor(config, config_dict) self.provider = config_dict["provider"] self.credentials = config_dict["auth"] diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py index 2db7fd7e076..94fdfab27a8 100644 --- a/lib/galaxy/objectstore/s3.py +++ b/lib/galaxy/objectstore/s3.py @@ -34,6 +34,7 @@ from galaxy.util.path import safe_relpath from . import ConcreteObjectStore from .caching import ( CacheTarget, + enable_cache_monitor, InProcessCacheMonitor, parse_caching_config_dict_from_xml, ) @@ -130,7 +131,6 @@ class CloudConfigMixin: "size": self.cache_size, "path": self.staging_path, }, - "enable_cache_monitor": False, } @@ -154,7 +154,7 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin): bucket_dict = config_dict["bucket"] connection_dict = config_dict.get("connection", {}) cache_dict = config_dict.get("cache") or {} - self.enable_cache_monitor = config_dict.get("enable_cache_monitor", True) + self.enable_cache_monitor = enable_cache_monitor(config, config_dict) self.access_key = auth_dict.get("access_key") self.secret_key = auth_dict.get("secret_key")