Merge pull request #7827 from ic4f/dev

Add option to wait for db to become available

config/galaxy.yml.sample

@@ -163,6 +163,17 @@ galaxy:
   # not recommended for production use.
   #database_auto_migrate: false
 
+  # Wait for database to become available instead of failing
+  # immediately.
+  #database_wait: false
+
+  # Number of attempts before failing if database_wait is enabled.
+  #database_wait_attempts: 60
+
+  # Time to sleep between attempts if database_wait is enabled (in
+  # seconds).
+  #database_wait_sleep: 1
+
   # Where dataset files are stored. It must accessible at the same path
   # on any cluster nodes that will run Galaxy jobs, unless using Pulsar.
   #file_path: database/files

doc/source/admin/galaxy_options.rst

@@ -142,6 +142,38 @@
 :Type: bool
 
 
+~~~~~~~~~~~~~~~~~
+``database_wait``
+~~~~~~~~~~~~~~~~~
+
+:Description:
+    Wait for database to become available instead of failing
+    immediately.
+:Default: ``false``
+:Type: bool
+
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+``database_wait_attempts``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Number of attempts before failing if database_wait is enabled.
+:Default: ``60``
+:Type: int
+
+
+~~~~~~~~~~~~~~~~~~~~~~~
+``database_wait_sleep``
+~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Time to sleep between attempts if database_wait is enabled (in
+    seconds).
+:Default: ``1``
+:Type: float
+
+
 ~~~~~~~~~~~~~
 ``file_path``
 ~~~~~~~~~~~~~

lib/galaxy/config.py

@@ -194,6 +194,13 @@ class Configuration(object):
         self.install_database_connection = kwargs.get("install_database_connection", None)
         self.install_database_engine_options = get_database_engine_options(kwargs, model_prefix="install_")
 
+        # Wait for database to become available instead of failing
+        self.database_wait = string_as_bool(kwargs.get("database_wait", "False"))
+        # Attempts before failing
+        self.database_wait_attempts = int(kwargs.get("database_wait_attempts", 60))
+        # Sleep period between attepmts (seconds)
+        self.database_wait_sleep = float(kwargs.get("database_wait_sleep", 1))
+
         # Where dataset files are stored
         self.file_path = resolve_path(kwargs.get("file_path", "database/files"), self.root)
         # new_file_path and legacy_home_dir can be overridden per destination in job_conf.
@@ -1154,6 +1161,9 @@ class ConfiguresGalaxyMixin(object):
         combined_install_database = not(install_db_url and install_db_url != db_url)
         install_db_url = install_db_url or db_url
 
+        if self.config.database_wait:
+            self._wait_for_database(db_url)
+
         if getattr(self.config, "max_metadata_value_size", None):
             from galaxy.model import custom_types
             custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size
@@ -1196,3 +1206,15 @@ class ConfiguresGalaxyMixin(object):
     def _configure_signal_handlers(self, handlers):
         for sig, handler in handlers.items():
             signal.signal(sig, handler)
+
+    def _wait_for_database(self, url):
+        from sqlalchemy_utils import database_exists
+        attempts = self.config.database_wait_attempts
+        pause = self.config.database_wait_sleep
+        for i in range(1, attempts):
+            try:
+                database_exists(url)
+                break
+            except Exception:
+                log.info("Waiting for database: attempt %d of %d" % (i, attempts))
+                time.sleep(pause)

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -128,6 +128,27 @@ mapping:
           migrate the database forward after updates. This is not recommended for production
           use.
 
+      database_wait:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Wait for database to become available instead of failing immediately.
+
+      database_wait_attempts:
+        type: int
+        default: 60
+        required: false
+        desc: |
+          Number of attempts before failing if database_wait is enabled.
+
+      database_wait_sleep:
+        type: float
+        default: 1
+        required: false
+        desc: |
+          Time to sleep between attempts if database_wait is enabled (in seconds).
+
       file_path:
         type: str
         default: database/files