diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index a8b2c79e9b7..84be1d207cc 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -212,12 +212,10 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin): self.model.engine.dispose() self.server_starttime = int(time.time()) # used for cachebusting - # When running the application without a web stack, exit == True signals the application loop to break and call - # the shutdown method - self.exit = False log.info("Galaxy app startup finished %s" % self.startup_timer) def shutdown(self): + log.debug('Shutting down') exception = None try: self.watchers.shutdown() @@ -263,12 +261,17 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin): exception = exception or e log.exception("Failed to shutdown SA database engine cleanly") - self.application_stack.shutdown() - # This is used to signal the webless application loop to terminate - self.exit = True + try: + self.application_stack.shutdown() + except Exception as e: + exception = exception or e + log.exception("Failed to shutdown application stack interface cleanly") if exception: raise exception + else: + log.debug('Finished shutting down') + def configure_fluent_log(self): if self.config.fluent_log: diff --git a/lib/galaxy/web/stack/__init__.py b/lib/galaxy/web/stack/__init__.py index 5f0d71bebda..7476afec21b 100644 --- a/lib/galaxy/web/stack/__init__.py +++ b/lib/galaxy/web/stack/__init__.py @@ -5,7 +5,6 @@ from __future__ import absolute_import import inspect import logging import os -import signal # The uwsgi module is automatically injected by the parent uwsgi # process and only exists that way. If anything works, this is a @@ -210,21 +209,6 @@ class UWSGIApplicationStack(MessageApplicationStack): self._mules_list = None super(UWSGIApplicationStack, self).__init__(app=app, config=config) - def __register_signal_handlers(self): - for name in ('TERM', 'INT', 'HUP'): - sig = getattr(signal, 'SIG%s' % name) - signal.signal(sig, self._handle_signal) - - def _handle_signal(self, signum, frame): - # uWSGI always sends SIGINT even if the master received SIGTERM - if signum in (signal.SIGTERM, signal.SIGINT): - log.info('Received SIGTERM/SIGINT, shutting down gracefully') - elif signum == signal.SIGHUP: - log.debug('Received SIGHUP, restarting') - self.shutdown() - # this terminates the application loop in the mule script, in the case of HUP, uWSGI will restart the mule - self.app.exit = True - @property def _configured_mules(self): if self._mules_list is None: diff --git a/scripts/galaxy-main b/scripts/galaxy-main index 4c82e41ecfb..8e3fea4f83f 100755 --- a/scripts/galaxy-main +++ b/scripts/galaxy-main @@ -25,7 +25,9 @@ The --py-call-osafterfork allows for proper shutdown on SIGTERM/SIGINT. import functools import logging import os +import signal import sys +import threading import time from argparse import ArgumentParser from logging.config import fileConfig @@ -55,6 +57,10 @@ DEFAULT_PID = "galaxy.pid" DEFAULT_VERBOSE = True DESCRIPTION = "Daemonized entry point for Galaxy." +SHUTDOWN_MSG = '__SHUTDOWN__' + +exit = threading.Event() + def load_galaxy_app( config_builder, @@ -88,6 +94,17 @@ def load_galaxy_app( return app +def handle_signal(signum, frame): + log.info('Received signal %d, exiting', signum) + exit.set() + + +def register_signals(): + for name in ('TERM', 'INT', 'HUP'): + sig = getattr(signal, 'SIG%s' % name) + signal.signal(sig, handle_signal) + + def app_loop(args, log): try: config_builder = GalaxyConfigBuilder(args) @@ -99,15 +116,12 @@ def app_loop(args, log): except BaseException: log.exception("Failed to initialize Galaxy application") raise - while not galaxy_app.exit: - try: - time.sleep(5) - except KeyboardInterrupt: - galaxy_app.exit = True - except SystemExit: - galaxy_app.exit = True - except Exception: + try: + # A timeout is required or the signals won't be handled + while not exit.wait(20): pass + except (KeyboardInterrupt, SystemExit): + pass try: galaxy_app.shutdown() except Exception: @@ -221,6 +235,7 @@ def main(): log.setLevel(logging.DEBUG) log.propagate = False + register_signals() if args.daemonize: if Daemonize is None: raise ImportError(REQUIRES_DAEMONIZE_MESSAGE)