From 06f5cfa56263fa206fa4dd55d427e87978f47e91 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Thu, 25 Apr 2019 20:35:49 -0400 Subject: [PATCH 1/7] Add option to wait for db to become available Configurable option to wait for the database to become available instead of failing immediately. _configure_models() seems to be the earliest/reasonable place for it. --- config/galaxy.yml.sample | 9 +++++++++ lib/galaxy/config.py | 22 ++++++++++++++++++++++ test/unit/test_configuration.py | 25 +++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 test/unit/test_configuration.py diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index c7381a87c8f..1a93a968133 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -163,6 +163,15 @@ galaxy: # not recommended for production use. #database_auto_migrate: false + # Wait for database to become available instead of failing immediately. + #database_wait: false + + # Number of attempts before failing. + #database_wait_attempts: 60 + + # Sleep period between attepmts (seconds; can be a floating point number). + #database_wait_sleep: 1 + # Where dataset files are stored. It must accessible at the same path # on any cluster nodes that will run Galaxy jobs, unless using Pulsar. #file_path: database/files diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 8a927244cdd..8ae5c81423b 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -194,6 +194,13 @@ class Configuration(object): self.install_database_connection = kwargs.get("install_database_connection", None) self.install_database_engine_options = get_database_engine_options(kwargs, model_prefix="install_") + # Wait for database to become available instead of failing + self.database_wait = string_as_bool(kwargs.get("database_wait", "False")) + # Attempts before failing + self.database_wait_attempts = int(kwargs.get("database_wait_attempts", 60)) + # Sleep period between attepmts (seconds) + self.database_wait_sleep = float(kwargs.get("database_wait_sleep", 1)) + # Where dataset files are stored self.file_path = resolve_path(kwargs.get("file_path", "database/files"), self.root) # new_file_path and legacy_home_dir can be overridden per destination in job_conf. @@ -1154,6 +1161,9 @@ class ConfiguresGalaxyMixin(object): combined_install_database = not(install_db_url and install_db_url != db_url) install_db_url = install_db_url or db_url + if self.config.database_wait: + self._wait_for_database(db_url) + if getattr(self.config, "max_metadata_value_size", None): from galaxy.model import custom_types custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size @@ -1196,3 +1206,15 @@ class ConfiguresGalaxyMixin(object): def _configure_signal_handlers(self, handlers): for sig, handler in handlers.items(): signal.signal(sig, handler) + + def _wait_for_database(self, url): + from time import sleep + from sqlalchemy_utils import database_exists + attempts = self.config.database_wait_attempts + pause = self.config.database_wait_sleep + for i in range(1, attempts): + try: + database_exists(url) + except: + log.info("Waiting for database: attempt %d of %d" % (i, attempts)) + sleep(pause) diff --git a/test/unit/test_configuration.py b/test/unit/test_configuration.py new file mode 100644 index 00000000000..033ee19babf --- /dev/null +++ b/test/unit/test_configuration.py @@ -0,0 +1,25 @@ +from galaxy.config import Configuration + +# TODO This, of course, needs A LOT more tests... + +def test_dabase_wait_default_values(): + config = Configuration() + assert not config.database_wait + assert config.database_wait_attempts == 60 + assert config.database_wait_sleep == 1 + + +def test_database_wait_assigned_values(): + db_wait = True + db_attempts = 42 + db_sleep = 999 + + values = {} + values['database_wait'] = db_wait + values['database_wait_attempts'] = db_attempts + values['database_wait_sleep'] = db_sleep + + config = Configuration(**values) + assert config.database_wait == db_wait + assert config.database_wait_attempts == db_attempts + assert config.database_wait_sleep == db_sleep From 7ead1cedd43373a58891cb3cb26120ace8e33a06 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 26 Apr 2019 18:03:57 -0400 Subject: [PATCH 2/7] Delete test file This test file contained 2 basic unit tests that tested the proposed new config values. Instantiating a Configuration object without passing the required arguments seems to have caused other tests to fail. Trying same PR without additional unit tests. --- test/unit/test_configuration.py | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 test/unit/test_configuration.py diff --git a/test/unit/test_configuration.py b/test/unit/test_configuration.py deleted file mode 100644 index 033ee19babf..00000000000 --- a/test/unit/test_configuration.py +++ /dev/null @@ -1,25 +0,0 @@ -from galaxy.config import Configuration - -# TODO This, of course, needs A LOT more tests... - -def test_dabase_wait_default_values(): - config = Configuration() - assert not config.database_wait - assert config.database_wait_attempts == 60 - assert config.database_wait_sleep == 1 - - -def test_database_wait_assigned_values(): - db_wait = True - db_attempts = 42 - db_sleep = 999 - - values = {} - values['database_wait'] = db_wait - values['database_wait_attempts'] = db_attempts - values['database_wait_sleep'] = db_sleep - - config = Configuration(**values) - assert config.database_wait == db_wait - assert config.database_wait_attempts == db_attempts - assert config.database_wait_sleep == db_sleep From 5cab87947ae713bf64f713e658823e13214eab5a Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 26 Apr 2019 19:34:32 -0400 Subject: [PATCH 3/7] Add Exception to except statement Fixing lint error --- lib/galaxy/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 8ae5c81423b..200af223167 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1215,6 +1215,6 @@ class ConfiguresGalaxyMixin(object): for i in range(1, attempts): try: database_exists(url) - except: + except Exception: log.info("Waiting for database: attempt %d of %d" % (i, attempts)) sleep(pause) From cf6ab64a890daec1c964abd42ee23ac3a210f5ce Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 26 Apr 2019 19:51:06 -0400 Subject: [PATCH 4/7] Update config/galaxy.yml.sample Co-Authored-By: ic4f --- config/galaxy.yml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 1a93a968133..deb1b172db6 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -169,7 +169,7 @@ galaxy: # Number of attempts before failing. #database_wait_attempts: 60 - # Sleep period between attepmts (seconds; can be a floating point number). + # Sleep period between attempts (seconds; can be a floating point number). #database_wait_sleep: 1 # Where dataset files are stored. It must accessible at the same path From f8bfd16d5a6800732ae1d77dcc8e2ed5ae07969e Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Fri, 26 Apr 2019 20:37:05 -0400 Subject: [PATCH 5/7] Remove inline import; call time.sleep instead time module already imported. --- lib/galaxy/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 200af223167..f1f3537fdb2 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1208,7 +1208,6 @@ class ConfiguresGalaxyMixin(object): signal.signal(sig, handler) def _wait_for_database(self, url): - from time import sleep from sqlalchemy_utils import database_exists attempts = self.config.database_wait_attempts pause = self.config.database_wait_sleep @@ -1217,4 +1216,4 @@ class ConfiguresGalaxyMixin(object): database_exists(url) except Exception: log.info("Waiting for database: attempt %d of %d" % (i, attempts)) - sleep(pause) + time.sleep(pause) From 158fb6b454ad7fde1d632e3f0469fc3b5c14ae2c Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Sat, 27 Apr 2019 01:19:57 -0400 Subject: [PATCH 6/7] Update config_schema + run make config-rebuild --- config/galaxy.yml.sample | 8 ++++-- doc/source/admin/galaxy_options.rst | 32 +++++++++++++++++++++ lib/galaxy/webapps/galaxy/config_schema.yml | 21 ++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index deb1b172db6..c911ebe358f 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -163,13 +163,15 @@ galaxy: # not recommended for production use. #database_auto_migrate: false - # Wait for database to become available instead of failing immediately. + # Wait for database to become available instead of failing + # immediately. #database_wait: false - # Number of attempts before failing. + # Number of attempts before failing if database_wait is enabled. #database_wait_attempts: 60 - # Sleep period between attempts (seconds; can be a floating point number). + # Time to sleep between attempts if database_wait is enabled (in + # seconds). #database_wait_sleep: 1 # Where dataset files are stored. It must accessible at the same path diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 0620457d9ad..82791c59d71 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -142,6 +142,38 @@ :Type: bool +~~~~~~~~~~~~~~~~~ +``database_wait`` +~~~~~~~~~~~~~~~~~ + +:Description: + Wait for database to become available instead of failing + immediately. +:Default: ``false`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~~~~~~~ +``database_wait_attempts`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Number of attempts before failing if database_wait is enabled. +:Default: ``60`` +:Type: int + + +~~~~~~~~~~~~~~~~~~~~~~~ +``database_wait_sleep`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Time to sleep between attempts if database_wait is enabled (in + seconds). +:Default: ``1`` +:Type: float + + ~~~~~~~~~~~~~ ``file_path`` ~~~~~~~~~~~~~ diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index a935fde6779..7447f18be59 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -128,6 +128,27 @@ mapping: migrate the database forward after updates. This is not recommended for production use. + database_wait: + type: bool + default: false + required: false + desc: | + Wait for database to become available instead of failing immediately. + + database_wait_attempts: + type: int + default: 60 + required: false + desc: | + Number of attempts before failing if database_wait is enabled. + + database_wait_sleep: + type: float + default: 1 + required: false + desc: | + Time to sleep between attempts if database_wait is enabled (in seconds). + file_path: type: str default: database/files From b4f4a12fb2177d085ca72c20988a3a543ebc5b58 Mon Sep 17 00:00:00 2001 From: Sergey Golitsynskiy Date: Mon, 29 Apr 2019 11:02:40 -0400 Subject: [PATCH 7/7] Add break stmt in try block Break out of loop upon successfully connecting to db. --- lib/galaxy/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index f1f3537fdb2..8754a41a175 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -1214,6 +1214,7 @@ class ConfiguresGalaxyMixin(object): for i in range(1, attempts): try: database_exists(url) + break except Exception: log.info("Waiting for database: attempt %d of %d" % (i, attempts)) time.sleep(pause)