From 131aeb66b4884200c6c9433aef9a80aabf98d132 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 15 Mar 2017 17:55:43 -0400 Subject: [PATCH 1/2] Abstract uWSGI handling into new galaxy.web.stack package. --- lib/galaxy/app.py | 7 +- lib/galaxy/config.py | 2 +- lib/galaxy/queue_worker.py | 5 +- lib/galaxy/queues.py | 11 +- lib/galaxy/tools/toolbox/watcher.py | 2 +- lib/galaxy/util/postfork.py | 43 ------- lib/galaxy/web/framework/middleware/batch.py | 6 +- lib/galaxy/web/framework/middleware/sentry.py | 2 +- lib/galaxy/web/stack/__init__.py | 106 ++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 90 +++++++-------- lib/galaxy/webapps/reports/app.py | 2 + lib/galaxy/webapps/reports/buildapp.py | 52 ++++----- lib/galaxy/webapps/tool_shed/app.py | 2 + lib/galaxy/webapps/tool_shed/buildapp.py | 72 ++++++------ lib/galaxy/webapps/util.py | 37 ++++++ .../update_repository_manager.py | 3 +- 16 files changed, 258 insertions(+), 184 deletions(-) delete mode 100644 lib/galaxy/util/postfork.py create mode 100644 lib/galaxy/web/stack/__init__.py diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index d5f73b9ad17..bad054e29d3 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -23,9 +23,9 @@ from galaxy.openid.providers import OpenIDProviders from galaxy.tools.data_manager.manager import DataManagers from galaxy.jobs import metrics as job_metrics from galaxy.web.proxy import ProxyManager +from galaxy.web.stack import application_stack_instance from galaxy.queue_worker import GalaxyQueueWorker from galaxy.util import heartbeat -from galaxy.util.postfork import register_postfork_function from tool_shed.galaxy_install import update_repository_manager @@ -44,6 +44,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ): log.debug( "python path is: %s", ", ".join( sys.path ) ) self.name = 'galaxy' self.new_installation = False + self.application_stack = application_stack_instance() # Read config file and check for errors self.config = config.Configuration( **kwargs ) self.config.check() @@ -151,7 +152,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ): fname=self.config.heartbeat_log ) self.heartbeat.daemon = True - register_postfork_function(self.heartbeat.start) + self.application_stack.register_postfork_function(self.heartbeat.start) self.sentry_client = None if self.config.sentry_dsn: @@ -159,7 +160,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ): import raven self.sentry_client = raven.Client(self.config.sentry_dsn) - register_postfork_function(postfork_sentry_client) + self.application_stack.register_postfork_function(postfork_sentry_client) # Transfer manager client if self.config.get_bool( 'enable_beta_job_managers', False ): diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index c965906517b..0c0fb50bdc8 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -25,8 +25,8 @@ from galaxy.exceptions import ConfigurationError from galaxy.util import listify from galaxy.util import string_as_bool from galaxy.util.dbkeys import GenomeBuilds -from galaxy.util.postfork import register_postfork_function from galaxy.web.formatting import expand_pretty_datetime_format +from galaxy.web.stack import register_postfork_function from .version import VERSION_MAJOR log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 763eefa42f5..b913c611ba9 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -51,8 +51,9 @@ def send_control_task(app, task, noop_self=False, kwargs={}): try: c = Connection(app.config.amqp_internal_connection) with producers[c].acquire(block=True) as producer: + control_queues = galaxy.queues.all_control_queues_for_declare(app.config, app.application_stack) producer.publish(payload, exchange=galaxy.queues.galaxy_exchange, - declare=[galaxy.queues.galaxy_exchange] + galaxy.queues.all_control_queues_for_declare(app.config), + declare=[galaxy.queues.galaxy_exchange] + control_queues, routing_key='control') except Exception: # This is likely connection refused. @@ -203,7 +204,7 @@ class GalaxyQueueWorker(ConsumerMixin, threading.Thread): # Default to figuring out which control queue to use based on the app config. queue = galaxy.queues.control_queue_from_config(app.config) self.task_mapping = task_mapping - self.declare_queues = galaxy.queues.all_control_queues_for_declare(app.config) + self.declare_queues = galaxy.queues.all_control_queues_for_declare(app.config, app.application_stack) # TODO we may want to purge the queue at the start to avoid executing # stale 'reload_tool', etc messages. This can happen if, say, a web # process goes down and messages get sent before it comes back up. diff --git a/lib/galaxy/queues.py b/lib/galaxy/queues.py index 150c4163e64..884d452103b 100644 --- a/lib/galaxy/queues.py +++ b/lib/galaxy/queues.py @@ -4,26 +4,21 @@ All message queues used by Galaxy """ -from galaxy.util.postfork import process_is_uwsgi - from kombu import Exchange, Queue, Connection ALL_CONTROL = "control.*" galaxy_exchange = Exchange('galaxy_core_exchange', type='topic') -def all_control_queues_for_declare(config): +def all_control_queues_for_declare(config, application_stack): """ For in-memory routing (used by sqlalchemy-based transports), we need to be able to build the entire routing table in producers. Refactor later to actually persist this somewhere instead of building it repeatedly. """ - possible_uwsgi_queues = [] - if process_is_uwsgi: - import uwsgi - possible_uwsgi_queues = [Queue("control.%s.%s" % (config.server_name.split('.')[0], wkr['id']), galaxy_exchange, routing_key='control') for wkr in uwsgi.workers()] - return possible_uwsgi_queues + [Queue('control.%s' % q, galaxy_exchange, routing_key='control') for q in config.server_names] + possible_stack_queues = [Queue("control.%s.%s" % (config.server_name.split('.')[0], wkr['id']), galaxy_exchange, routing_key='control') for wkr in application_stack.workers()] + return possible_stack_queues + [Queue('control.%s' % q, galaxy_exchange, routing_key='control') for q in config.server_names] def control_queue_from_config(config): diff --git a/lib/galaxy/tools/toolbox/watcher.py b/lib/galaxy/tools/toolbox/watcher.py index 264d8d85e5a..862b30017ee 100644 --- a/lib/galaxy/tools/toolbox/watcher.py +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -15,7 +15,7 @@ except ImportError: can_watch = False from galaxy.util.hash_util import md5_hash_file -from galaxy.util.postfork import register_postfork_function +from galaxy.web.stack import register_postfork_function log = logging.getLogger( __name__ ) diff --git a/lib/galaxy/util/postfork.py b/lib/galaxy/util/postfork.py deleted file mode 100644 index 8cb7c86adec..00000000000 --- a/lib/galaxy/util/postfork.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Handle postfork functions under uWSGI -""" - -# The uwsgi module is automatically injected by the parent uwsgi -# process and only exists that way. If anything works, this is a -# uwsgi-managed process. -try: - import uwsgi - if hasattr(uwsgi, "numproc"): - process_is_uwsgi = True - else: - process_is_uwsgi = False -except ImportError: - # This is not a uwsgi process, or something went horribly wrong. - process_is_uwsgi = False - -try: - from uwsgidecorators import postfork -except: - def pf_dec(func): - return func - postfork = pf_dec - if process_is_uwsgi: - print("WARNING: This is a uwsgi process but the uwsgidecorators library" - " is unavailable. This is likely due to using an external (not" - " in Galaxy's virtualenv) uwsgi and you may experience errors.") - - -postfork_functions = [] - - -@postfork -def do_postfork(): - for f, args, kwargs in [ t for t in postfork_functions ]: - f(*args, **kwargs) - - -def register_postfork_function(f, *args, **kwargs): - if process_is_uwsgi: - postfork_functions.append((f, args, kwargs)) - else: - f(*args, **kwargs) diff --git a/lib/galaxy/web/framework/middleware/batch.py b/lib/galaxy/web/framework/middleware/batch.py index 58dfd7b64ae..631b8db5796 100644 --- a/lib/galaxy/web/framework/middleware/batch.py +++ b/lib/galaxy/web/framework/middleware/batch.py @@ -59,11 +59,11 @@ class BatchMiddleware( object ): ] } - def __init__( self, galaxy, application, config=None ): - #: the original galaxy webapp - self.galaxy = galaxy + def __init__( self, application, galaxy, config=None ): #: the wrapped webapp self.application = application + #: the original galaxy webapp + self.galaxy = galaxy self.config = self.DEFAULT_CONFIG.copy() self.config.update( config ) self.base_url = routes.url_for( '/' ) diff --git a/lib/galaxy/web/framework/middleware/sentry.py b/lib/galaxy/web/framework/middleware/sentry.py index 980dec49205..aec0ee3b420 100644 --- a/lib/galaxy/web/framework/middleware/sentry.py +++ b/lib/galaxy/web/framework/middleware/sentry.py @@ -12,7 +12,7 @@ try: except: Client = None -from galaxy.util.postfork import register_postfork_function +from galaxy.web.stack import register_postfork_function RAVEN_IMPORT_MESSAGE = ('The Python raven package is required to use this ' diff --git a/lib/galaxy/web/stack/__init__.py b/lib/galaxy/web/stack/__init__.py new file mode 100644 index 00000000000..d843822e598 --- /dev/null +++ b/lib/galaxy/web/stack/__init__.py @@ -0,0 +1,106 @@ +"""Web application stack operations +""" +from __future__ import print_function + +import inspect +import logging +import os + +# The uwsgi module is automatically injected by the parent uwsgi +# process and only exists that way. If anything works, this is a +# uwsgi-managed process. +try: + import uwsgi +except ImportError: + uwsgi = None + +try: + from uwsgidecorators import postfork as uwsgi_postfork +except: + uwsgi_postfork = lambda x: x # noqa: E731 + if uwsgi is not None and hasattr(uwsgi, 'numproc'): + print("WARNING: This is a uwsgi process but the uwsgidecorators library" + " is unavailable. This is likely due to using an external (not" + " in Galaxy's virtualenv) uwsgi and you may experience errors. " + "HINT:\n {venv}/bin/pip install uwsgidecorators".format( + venv=os.environ.get('VIRTUAL_ENV', '/path/to/venv'))) + + +log = logging.getLogger(__name__) + + +class ApplicationStack(object): + name = None + prohibited_middleware = frozenset() + + @classmethod + def register_postfork_function(cls, f, *args, **kwargs): + f(*args, **kwargs) + + def workers(self): + return [] + + def allowed_middleware(self, middleware): + if hasattr(middleware, '__name__'): + middleware = middleware.__name__ + return middleware not in self.prohibited_middleware + + def set_postfork_server_name(self, app): + pass + + +class UWSGIApplicationStack(ApplicationStack): + name = 'uWSGI' + prohibited_middleware = frozenset([ + 'wrap_in_static', + 'EvalException', + ]) + + postfork_functions = [] + + @classmethod + def register_postfork_function(cls, f, *args, **kwargs): + cls.postfork_functions.append((f, args, kwargs)) + + def workers(self): + return uwsgi.workers() + + def set_postfork_server_name(self, app): + app.config.server_name += ".%d" % uwsgi.worker_id() + + +class PasteApplicationStack(ApplicationStack): + name = 'Python Paste' + + +class WeblessApplicationStack(ApplicationStack): + name = 'Webless' + + +def application_stack_class(): + """Returns the correct ApplicationStack class for the stack under which + this Galaxy process is running. + """ + if uwsgi is not None and hasattr(uwsgi, 'numproc'): + return UWSGIApplicationStack + else: + # cleverer ideas welcome + for frame in inspect.stack(): + if frame[1].endswith(os.path.join('pastescript', 'loadwsgi.py')): + return PasteApplicationStack + return WeblessApplicationStack + + +def application_stack_instance(): + stack_class = application_stack_class() + return stack_class() + + +def register_postfork_function(f, *args, **kwargs): + application_stack_class().register_postfork_function(f, *args, **kwargs) + + +@uwsgi_postfork +def _do_postfork(): + for f, args, kwargs in [ t for t in UWSGIApplicationStack.postfork_functions ]: + f(*args, **kwargs) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index f6b9ac807e6..1448f05d0c6 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -19,10 +19,9 @@ import galaxy.model.mapping import galaxy.datatypes.registry import galaxy.web.framework import galaxy.web.framework.webapp -from galaxy.webapps.util import build_template_error_formatters +from galaxy.webapps.util import build_template_error_formatters, wrap_if_allowed from galaxy import util from galaxy.util import asbool -from galaxy.util.postfork import process_is_uwsgi, register_postfork_function from galaxy.util.properties import load_app_properties from paste import httpexceptions @@ -117,12 +116,11 @@ def paste_app_factory( global_conf, **kwargs ): # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): - webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) + webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs) if asbool( kwargs.get( 'static_enabled', True) ): - if process_is_uwsgi: - log.error("Static middleware is enabled in your configuration but this is a uwsgi process. Refusing to wrap in static middleware.") - else: - webapp = wrap_in_static( webapp, global_conf, plugin_frameworks=[ app.visualizations_registry ], **kwargs ) + webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static, + args=(global_conf,), + kwargs=dict(plugin_frameworks=[app.visualizations_registry], **kwargs)) # Close any pooled database connections before forking try: galaxy.model.mapping.metadata.bind.dispose() @@ -135,7 +133,7 @@ def paste_app_factory( global_conf, **kwargs ): except: log.exception("Unable to dispose of pooled toolshed install model database connections.") - register_postfork_function(postfork_setup) + app.application_stack.register_postfork_function(postfork_setup) for th in threading.enumerate(): if th.is_alive(): @@ -162,9 +160,7 @@ def uwsgi_app_factory(): def postfork_setup(): from galaxy.app import app - if process_is_uwsgi: - import uwsgi - app.config.server_name += ".%s" % uwsgi.worker_id() + app.application_stack.set_postfork_server_name(app) app.control_worker.bind_and_start() @@ -848,12 +844,13 @@ def _add_item_provenance_controller( webapp, name_prefix, path_prefix, **kwd ): webapp.mapper.resource(name, "provenance", path_prefix=path_prefix, controller=controller) -def wrap_in_middleware( app, global_conf, **local_conf ): +def wrap_in_middleware( app, global_conf, application_stack, **local_conf ): """ Based on the configuration wrap `app` in a set of common and useful middleware. """ webapp = app + stack = application_stack # Merge the global and local configurations conf = global_conf.copy() @@ -862,16 +859,15 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): - app = httpexceptions.make_middleware( app, conf ) - log.debug( "Enabling 'httpexceptions' middleware" ) + app = wrap_if_allowed( app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,) ) # Statsd request timing and profiling statsd_host = conf.get('statsd_host', None) if statsd_host: from galaxy.web.framework.middleware.statsd import StatsdMiddleware - app = StatsdMiddleware( app, - statsd_host, - conf.get('statsd_port', 8125), - conf.get('statsd_prefix', 'galaxy') ) + app = wrap_if_allowed( app, stack, StatsdMiddleware, + args=( statsd_host, + conf.get('statsd_port', 8125), + conf.get('statsd_prefix', 'galaxy') ) ) log.debug( "Enabling 'statsd' middleware" ) # If we're using remote_user authentication, add middleware that # protects Galaxy from improperly configured authentication in the @@ -880,25 +876,26 @@ def wrap_in_middleware( app, global_conf, **local_conf ): use_remote_user = asbool(conf.get( 'use_remote_user', False )) or single_user if use_remote_user: from galaxy.web.framework.middleware.remoteuser import RemoteUser - app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ), - display_servers=util.listify( conf.get( 'display_servers', '' ) ), - single_user=single_user, - admin_users=conf.get( 'admin_users', '' ).split( ',' ), - remote_user_header=conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ), - remote_user_secret_header=conf.get('remote_user_secret', None), - normalize_remote_user_email=conf.get('normalize_remote_user_email', False)) + app = wrap_if_allowed( app, stack, RemoteUser, + kwargs=dict( + maildomain=conf.get('remote_user_maildomain', None), + display_servers=util.listify( conf.get('display_servers', '')), + single_user=single_user, + admin_users=conf.get('admin_users', '').split(','), + remote_user_header=conf.get('remote_user_header', 'HTTP_REMOTE_USER'), + remote_user_secret_header=conf.get('remote_user_secret', None), + normalize_remote_user_email=conf.get('normalize_remote_user_email', False)) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive - app = recursive.RecursiveMiddleware( app, conf ) - log.debug( "Enabling 'recursive' middleware" ) + app = wrap_if_allowed( app, stack, recursive.RecursiveMiddleware, args=(conf,) ) # If sentry logging is enabled, log here before propogating up to # the error middleware sentry_dsn = conf.get( 'sentry_dsn', None ) if sentry_dsn: from galaxy.web.framework.middleware.sentry import Sentry - app = Sentry( app, sentry_dsn ) + app = wrap_if_allowed( app, stack, Sentry, args=(sentry_dsn,) ) # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance @@ -906,48 +903,39 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # Middleware to check for WSGI compliance if asbool( conf.get( 'use_lint', False ) ): from paste import lint - app = lint.make_middleware( app, conf ) - log.debug( "Enabling 'lint' middleware" ) + app = wrap_if_allowed( app, stack, lint.make_middleware, name='paste.lint', args=(conf,) ) # Middleware to run the python profiler on each request if asbool( conf.get( 'use_profile', False ) ): from paste.debug import profile - app = profile.ProfileMiddleware( app, conf ) - log.debug( "Enabling 'profile' middleware" ) - if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi: + app = wrap_if_allowed( app, stack, profile.ProfileMiddleware, args=(conf,) ) + if debug and asbool( conf.get( 'use_interactive', False ) ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. from weberror import evalexception - app = evalexception.EvalException( app, conf, - templating_formatters=build_template_error_formatters() ) - log.debug( "Enabling 'eval exceptions' middleware" ) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, evalexception.EvalException, + args=(conf,), + kwargs=dict(templating_formatters=build_template_error_formatters()), + alt_wrap=galaxy.web.framework.middleware.error.ErrorMiddleware, + alt_args=(conf,) ) else: - if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi: - log.error("Interactive debugging middleware is enabled in your configuration " - "but this is a uwsgi process. Refusing to wrap in interactive error middleware.") # Not in interactive debug mode, just use the regular error middleware import galaxy.web.framework.middleware.error - app = galaxy.web.framework.middleware.error.ErrorMiddleware( app, conf ) - log.debug( "Enabling 'error' middleware" ) + app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) ) # Transaction logging (apache access.log style) if asbool( conf.get( 'use_translogger', True ) ): from galaxy.web.framework.middleware.translogger import TransLogger - app = TransLogger( app ) - log.debug( "Enabling 'trans logger' middleware" ) + app = wrap_if_allowed( app, stack, TransLogger ) # X-Forwarded-Host handling from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware - app = XForwardedHostMiddleware( app ) - log.debug( "Enabling 'x-forwarded-host' middleware" ) + app = wrap_if_allowed( app, stack, XForwardedHostMiddleware ) # Request ID middleware from galaxy.web.framework.middleware.request_id import RequestIDMiddleware - app = RequestIDMiddleware( app ) - log.debug( "Enabling 'Request ID' middleware" ) - + app = wrap_if_allowed( app, stack, RequestIDMiddleware ) # api batch call processing middleware from galaxy.web.framework.middleware.batch import BatchMiddleware - app = BatchMiddleware( webapp, app, {}) - log.debug( "Enabling 'Batch' middleware" ) - + app = wrap_if_allowed( app, stack, BatchMiddleware, args=(webapp, {}) ) return app diff --git a/lib/galaxy/webapps/reports/app.py b/lib/galaxy/webapps/reports/app.py index 74761446588..c21c3dcfa7d 100644 --- a/lib/galaxy/webapps/reports/app.py +++ b/lib/galaxy/webapps/reports/app.py @@ -4,6 +4,7 @@ import time import galaxy.model from galaxy.web import security +from galaxy.web.stack import application_stack_instance import logging log = logging.getLogger( __name__ ) @@ -17,6 +18,7 @@ class UniverseApplication( object ): self.config = config.Configuration( **kwargs ) self.config.check() config.configure_logging( self.config ) + self.application_stack = application_stack_instance() # Determine the database url if self.config.database_connection: db_url = self.config.database_connection diff --git a/lib/galaxy/webapps/reports/buildapp.py b/lib/galaxy/webapps/reports/buildapp.py index bba669debad..1a3cb3e70b9 100644 --- a/lib/galaxy/webapps/reports/buildapp.py +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -11,8 +11,7 @@ from inspect import isclass from paste import httpexceptions from galaxy.util import asbool -from galaxy.util.postfork import process_is_uwsgi -from galaxy.webapps.util import build_template_error_formatters +from galaxy.webapps.util import build_template_error_formatters, wrap_if_allowed import galaxy.model import galaxy.model.mapping @@ -69,9 +68,11 @@ def app_factory( global_conf, **kwargs ): webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): - webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) + webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs) if asbool( kwargs.get( 'static_enabled', True ) ): - webapp = wrap_in_static( webapp, global_conf, **kwargs ) + webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static, + args=(global_conf,), + kwargs=kwargs) # Close any pooled database connections before forking try: galaxy.model.mapping.metadata.bind.dispose() @@ -81,8 +82,9 @@ def app_factory( global_conf, **kwargs ): return webapp -def wrap_in_middleware( app, global_conf, **local_conf ): +def wrap_in_middleware( app, global_conf, application_stack, **local_conf ): """Based on the configuration wrap `app` in a set of common and useful middleware.""" + stack = application_stack # Merge the global and local configurations conf = global_conf.copy() conf.update(local_conf) @@ -90,14 +92,12 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): - app = httpexceptions.make_middleware( app, conf ) - log.debug( "Enabling 'httpexceptions' middleware" ) + app = wrap_if_allowed( app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive - app = recursive.RecursiveMiddleware( app, conf ) - log.debug( "Enabling 'recursive' middleware" ) + app = wrap_if_allowed( app, stack, recursive.RecursiveMiddleware, args=(conf,) ) # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance @@ -105,44 +105,38 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # Middleware to check for WSGI compliance if asbool( conf.get( 'use_lint', True ) ): from paste import lint - app = lint.make_middleware( app, conf ) - log.debug( "Enabling 'lint' middleware" ) + app = wrap_if_allowed( app, stack, lint.make_middleware, name='paste.lint', args=(conf,) ) # Middleware to run the python profiler on each request if asbool( conf.get( 'use_profile', False ) ): import profile - app = profile.ProfileMiddleware( app, conf ) - log.debug( "Enabling 'profile' middleware" ) + app = wrap_if_allowed( app, stack, profile.ProfileMiddleware, args=(conf,) ) # Middleware that intercepts print statements and shows them on the # returned page if asbool( conf.get( 'use_printdebug', True ) ): from paste.debug import prints - app = prints.PrintDebugMiddleware( app, conf ) - log.debug( "Enabling 'print debug' middleware" ) - if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi: + app = wrap_if_allowed( app, stack, prints.PrintDebugMiddleware, args=(conf,) ) + if debug and asbool( conf.get( 'use_interactive', False ) ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. from weberror import evalexception - app = evalexception.EvalException( app, conf, - templating_formatters=build_template_error_formatters() ) - log.debug( "Enabling 'eval exceptions' middleware" ) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, evalexception.EvalException, + args=(conf,), + kwargs=dict(templating_formatters=build_template_error_formatters()), + alt_wrap=galaxy.web.framework.middleware.error.ErrorMiddleware, + alt_args=(conf,) ) else: - if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi: - log.error("Interactive debugging middleware is enabled in your configuration " - "but this is a uwsgi process. Refusing to wrap in interactive error middleware.") # Not in interactive debug mode, just use the regular error middleware - from paste.exceptions import errormiddleware - app = errormiddleware.ErrorMiddleware( app, conf ) - log.debug( "Enabling 'error' middleware" ) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) ) # Transaction logging (apache access.log style) if asbool( conf.get( 'use_translogger', True ) ): from paste.translogger import TransLogger - app = TransLogger( app ) - log.debug( "Enabling 'trans logger' middleware" ) + app = wrap_if_allowed( app, stack, TransLogger ) # X-Forwarded-Host handling from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware - app = XForwardedHostMiddleware( app ) - log.debug( "Enabling 'x-forwarded-host' middleware" ) + app = wrap_if_allowed( app, stack, XForwardedHostMiddleware ) return app diff --git a/lib/galaxy/webapps/tool_shed/app.py b/lib/galaxy/webapps/tool_shed/app.py index b05fb939b62..f1b4f45f515 100644 --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -11,6 +11,7 @@ from galaxy.managers.tags import CommunityTagManager from galaxy.openid.providers import OpenIDProviders from galaxy.util.dbkeys import GenomeBuilds from galaxy.web import security +from galaxy.web.stack import application_stack_instance import tool_shed.repository_registry import tool_shed.repository_types.registry from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager @@ -29,6 +30,7 @@ class UniverseApplication( object ): self.config = config.Configuration( **kwd ) self.config.check() configure_logging( self.config ) + self.application_stack = application_stack_instance() # Initialize the Galaxy datatypes registry. self.datatypes_registry = galaxy.datatypes.registry.Registry() self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config ) diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 59c09a1d5a0..1ccb8de440a 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -14,9 +14,8 @@ from galaxy.util import asbool import galaxy.webapps.tool_shed.model import galaxy.webapps.tool_shed.model.mapping import galaxy.web.framework.webapp -from galaxy.webapps.util import build_template_error_formatters +from galaxy.webapps.util import build_template_error_formatters, wrap_if_allowed from galaxy import util -from galaxy.util.postfork import process_is_uwsgi from galaxy.util.properties import load_app_properties from routes.middleware import RoutesMiddleware @@ -186,12 +185,11 @@ def app_factory( global_conf, **kwargs ): webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): - webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) + webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs) if asbool( kwargs.get( 'static_enabled', True) ): - if process_is_uwsgi: - log.error("Static middleware is enabled in your configuration but this is a uwsgi process. Refusing to wrap in static middleware.") - else: - webapp = wrap_in_static( webapp, global_conf, **kwargs ) + webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static, + args=(global_conf,), + kwargs=kwargs) # Close any pooled database connections before forking try: galaxy.webapps.tool_shed.model.mapping.metadata.bind.dispose() @@ -201,8 +199,9 @@ def app_factory( global_conf, **kwargs ): return webapp -def wrap_in_middleware( app, global_conf, **local_conf ): +def wrap_in_middleware( app, global_conf, application_stack, **local_conf ): """Based on the configuration wrap `app` in a set of common and useful middleware.""" + stack = application_stack # Merge the global and local configurations conf = global_conf.copy() conf.update( local_conf ) @@ -210,63 +209,59 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # First put into place httpexceptions, which must be most closely # wrapped around the application (it can interact poorly with # other middleware): - app = httpexceptions.make_middleware( app, conf ) - log.debug( "Enabling 'httpexceptions' middleware" ) + app = wrap_if_allowed( app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,) ) # Create a separate mapper for redirects to prevent conflicts. redirect_mapper = routes.Mapper() redirect_mapper = _map_redirects( redirect_mapper ) # Load the Routes middleware which we use for redirecting - app = RoutesMiddleware( app, redirect_mapper ) - log.debug( "Enabling 'routes' middleware" ) + app = wrap_if_allowed( app, stack, RoutesMiddleware, args=(redirect_mapper,) ) # If we're using remote_user authentication, add middleware that # protects Galaxy from improperly configured authentication in the # upstream server if asbool(conf.get( 'use_remote_user', False )): from galaxy.webapps.tool_shed.framework.middleware.remoteuser import RemoteUser - app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ), - display_servers=util.listify( conf.get( 'display_servers', '' ) ), - admin_users=conf.get( 'admin_users', '' ).split( ',' ), - remote_user_secret_header=conf.get('remote_user_secret', None) ) - log.debug( "Enabling 'remote user' middleware" ) + app = wrap_if_allowed( app, stack, RemoteUser, + kwargs=dict( + maildomain=conf.get('remote_user_maildomain', None), + display_servers=util.listify( conf.get('display_servers', '')), + admin_users=conf.get('admin_users', '').split(','), + remote_user_header=conf.get('remote_user_header', 'HTTP_REMOTE_USER'), + remote_user_secret_header=conf.get('remote_user_secret', None), + normalize_remote_user_email=conf.get('normalize_remote_user_email', False)) ) # The recursive middleware allows for including requests in other # requests or forwarding of requests, all on the server side. if asbool(conf.get('use_recursive', True)): from paste import recursive - app = recursive.RecursiveMiddleware( app, conf ) - log.debug( "Enabling 'recursive' middleware" ) - if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi: + app = wrap_if_allowed( app, stack, recursive.RecursiveMiddleware, args=(conf,) ) + if debug and asbool( conf.get( 'use_interactive', False ) ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. from weberror import evalexception - app = evalexception.EvalException( app, conf, - templating_formatters=build_template_error_formatters() ) - log.debug( "Enabling 'eval exceptions' middleware" ) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, evalexception.EvalException, + args=(conf,), + kwargs=dict(templating_formatters=build_template_error_formatters()), + alt_wrap=galaxy.web.framework.middleware.error.ErrorMiddleware, + alt_args=(conf,) ) else: - if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi: - log.error("Interactive debugging middleware is enabled in your configuration " - "but this is a uwsgi process. Refusing to wrap in interactive error middleware.") # Not in interactive debug mode, just use the regular error middleware import galaxy.web.framework.middleware.error - app = galaxy.web.framework.middleware.error.ErrorMiddleware( app, conf ) - log.debug( "Enabling 'error' middleware" ) + app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) ) # Transaction logging (apache access.log style) if asbool( conf.get( 'use_translogger', True ) ): from paste.translogger import TransLogger - app = TransLogger( app ) - log.debug( "Enabling 'trans logger' middleware" ) + app = wrap_if_allowed( app, stack, TransLogger ) # If sentry logging is enabled, log here before propogating up to # the error middleware # TODO sentry config is duplicated between tool_shed/galaxy, refactor this. sentry_dsn = conf.get( 'sentry_dsn', None ) if sentry_dsn: from galaxy.web.framework.middleware.sentry import Sentry - log.debug( "Enabling 'sentry' middleware" ) - app = Sentry( app, sentry_dsn ) + app = wrap_if_allowed( app, stack, Sentry, args=(sentry_dsn,) ) # X-Forwarded-Host handling from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware - app = XForwardedHostMiddleware( app ) - log.debug( "Enabling 'x-forwarded-host' middleware" ) + app = wrap_if_allowed( app, stack, XForwardedHostMiddleware ) # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance. The print debug middleware needs to be loaded last, @@ -276,19 +271,16 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # Middleware to check for WSGI compliance if asbool( conf.get( 'use_lint', True ) ): from paste import lint - app = lint.make_middleware( app, conf ) - log.debug( "Enabling 'lint' middleware" ) + app = wrap_if_allowed( app, stack, lint.make_middleware, name='paste.lint', args=(conf,) ) # Middleware to run the python profiler on each request if asbool( conf.get( 'use_profile', False ) ): import profile - app = profile.ProfileMiddleware( app, conf ) - log.debug( "Enabling 'profile' middleware" ) + app = wrap_if_allowed( app, stack, profile.ProfileMiddleware, args=(conf,) ) # Middleware that intercepts print statements and shows them on the # returned page if asbool( conf.get( 'use_printdebug', True ) ): from paste.debug import prints - app = prints.PrintDebugMiddleware( app, conf ) - log.debug( "Enabling 'print debug' middleware" ) + app = wrap_if_allowed( app, stack, prints.PrintDebugMiddleware, args=(conf,) ) return app diff --git a/lib/galaxy/webapps/util.py b/lib/galaxy/webapps/util.py index 618f27dda0f..9cb0029d99e 100644 --- a/lib/galaxy/webapps/util.py +++ b/lib/galaxy/webapps/util.py @@ -1,8 +1,13 @@ from __future__ import absolute_import +import logging + import mako.exceptions +log = logging.getLogger(__name__) + + def build_template_error_formatters(): """ Build a list of template error formatters for WebError. When an error @@ -19,3 +24,35 @@ def build_template_error_formatters(): return mako.exceptions.html_error_template().render( full=False, css=False ) formatters.append( mako_html_data ) return formatters + + +def wrap_if_allowed(app, stack, wrap, name=None, args=None, kwargs=None, + alt_wrap=None, alt_name=None, alt_args=None, alt_kwargs=None): + """ + Wrap the application with the given method if the application stack allows for it. + + :type app: :class:`galaxy.web.framework.webapp.WebApplication` subclass + :param app: application to wrap + :type stack: :class:`galaxy.web.stack.ApplicationStack` subclass + :param stack: instance of application stack implementing `allowed_middleware()` method + :type wrap: types.FunctionType or types.LambdaType + :param wrap: function to wrap application with + :type name: str + :param name: alternative wrap function name for logging purposes (`wrap.__name__` if None) + :type args: list + :param args: arguments to pass to `wrap` (not including `app` itself) + :type kwargs: dict + :param kwargs: keyword arguments to pass to `wrap` + """ + name = name or wrap.__name__ + if stack.allowed_middleware(wrap): + args = args or [] + kwargs = kwargs or {} + log.debug("Enabling '%s' middleware", name) + return wrap(app, *args, **kwargs) + else: + log.warning("'%s' is enabled in your configuration but the %s application stack does not support it, this " + "middleware has been disabled", name, stack.name) + if alt_wrap: + app = wrap_if_allowed(app, stack, alt_wrap, name=alt_name, args=alt_args, kwargs=alt_kwargs) + return app diff --git a/lib/tool_shed/galaxy_install/update_repository_manager.py b/lib/tool_shed/galaxy_install/update_repository_manager.py index 3846f301658..572a72a7abd 100644 --- a/lib/tool_shed/galaxy_install/update_repository_manager.py +++ b/lib/tool_shed/galaxy_install/update_repository_manager.py @@ -8,7 +8,6 @@ from sqlalchemy import false import tool_shed.util.shed_util_common as suc from galaxy import util -from galaxy.util.postfork import register_postfork_function from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import repository_util @@ -27,7 +26,7 @@ class UpdateRepositoryManager( object ): self.sleeper = Sleeper() self.restarter = threading.Thread( target=self.__restarter ) self.restarter.daemon = True - register_postfork_function(self.restarter.start) + self.app.application_stack.register_postfork_function(self.restarter.start) self.seconds_to_sleep = int( app.config.hours_between_check * 3600 ) def get_update_to_changeset_revision_and_ctx_rev( self, repository ): From d18b4d13cbc25c85005e4b19a02abedbed7aba46 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 20 Mar 2017 10:56:44 -0400 Subject: [PATCH 2/2] Add an exception for unsupported middleware wrapping for better failure handling. --- lib/galaxy/webapps/galaxy/buildapp.py | 23 ++++++++----- lib/galaxy/webapps/reports/buildapp.py | 24 ++++++++----- lib/galaxy/webapps/tool_shed/buildapp.py | 24 ++++++++----- lib/galaxy/webapps/util.py | 43 +++++++++++++++++------- 4 files changed, 76 insertions(+), 38 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 1448f05d0c6..b75b124543c 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -19,7 +19,12 @@ import galaxy.model.mapping import galaxy.datatypes.registry import galaxy.web.framework import galaxy.web.framework.webapp -from galaxy.webapps.util import build_template_error_formatters, wrap_if_allowed +from galaxy.webapps.util import ( + MiddlewareWrapUnsupported, + build_template_error_formatters, + wrap_if_allowed, + wrap_if_allowed_or_fail +) from galaxy import util from galaxy.util import asbool from galaxy.util.properties import load_app_properties @@ -912,13 +917,15 @@ def wrap_in_middleware( app, global_conf, application_stack, **local_conf ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. - from weberror import evalexception - import galaxy.web.framework.middleware.error - app = wrap_if_allowed( app, stack, evalexception.EvalException, - args=(conf,), - kwargs=dict(templating_formatters=build_template_error_formatters()), - alt_wrap=galaxy.web.framework.middleware.error.ErrorMiddleware, - alt_args=(conf,) ) + try: + from weberror import evalexception + app = wrap_if_allowed_or_fail( app, stack, evalexception.EvalException, + args=(conf,), + kwargs=dict(templating_formatters=build_template_error_formatters()) ) + except MiddlewareWrapUnsupported as exc: + log.warning(str(exc)) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) ) else: # Not in interactive debug mode, just use the regular error middleware import galaxy.web.framework.middleware.error diff --git a/lib/galaxy/webapps/reports/buildapp.py b/lib/galaxy/webapps/reports/buildapp.py index 1a3cb3e70b9..e2e6317b742 100644 --- a/lib/galaxy/webapps/reports/buildapp.py +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -11,8 +11,12 @@ from inspect import isclass from paste import httpexceptions from galaxy.util import asbool -from galaxy.webapps.util import build_template_error_formatters, wrap_if_allowed - +from galaxy.webapps.util import ( + MiddlewareWrapUnsupported, + build_template_error_formatters, + wrap_if_allowed, + wrap_if_allowed_or_fail +) import galaxy.model import galaxy.model.mapping import galaxy.web.framework.webapp @@ -119,13 +123,15 @@ def wrap_in_middleware( app, global_conf, application_stack, **local_conf ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. - from weberror import evalexception - import galaxy.web.framework.middleware.error - app = wrap_if_allowed( app, stack, evalexception.EvalException, - args=(conf,), - kwargs=dict(templating_formatters=build_template_error_formatters()), - alt_wrap=galaxy.web.framework.middleware.error.ErrorMiddleware, - alt_args=(conf,) ) + try: + from weberror import evalexception + app = wrap_if_allowed_or_fail( app, stack, evalexception.EvalException, + args=(conf,), + kwargs=dict(templating_formatters=build_template_error_formatters()) ) + except MiddlewareWrapUnsupported as exc: + log.warning(str(exc)) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) ) else: # Not in interactive debug mode, just use the regular error middleware import galaxy.web.framework.middleware.error diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 1ccb8de440a..32f710308d4 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -14,7 +14,12 @@ from galaxy.util import asbool import galaxy.webapps.tool_shed.model import galaxy.webapps.tool_shed.model.mapping import galaxy.web.framework.webapp -from galaxy.webapps.util import build_template_error_formatters, wrap_if_allowed +from galaxy.webapps.util import ( + MiddlewareWrapUnsupported, + build_template_error_formatters, + wrap_if_allowed, + wrap_if_allowed_or_fail +) from galaxy import util from galaxy.util.properties import load_app_properties from routes.middleware import RoutesMiddleware @@ -237,13 +242,16 @@ def wrap_in_middleware( app, global_conf, application_stack, **local_conf ): # Interactive exception debugging, scary dangerous if publicly # accessible, if not enabled we'll use the regular error printing # middleware. - from weberror import evalexception - import galaxy.web.framework.middleware.error - app = wrap_if_allowed( app, stack, evalexception.EvalException, - args=(conf,), - kwargs=dict(templating_formatters=build_template_error_formatters()), - alt_wrap=galaxy.web.framework.middleware.error.ErrorMiddleware, - alt_args=(conf,) ) + try: + from weberror import evalexception + app = wrap_if_allowed_or_fail( app, stack, evalexception.EvalException, + args=(conf,), + kwargs=dict(templating_formatters=build_template_error_formatters()) ) + except MiddlewareWrapUnsupported as exc: + log.warning(str(exc)) + import galaxy.web.framework.middleware.error + app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) ) + else: # Not in interactive debug mode, just use the regular error middleware import galaxy.web.framework.middleware.error diff --git a/lib/galaxy/webapps/util.py b/lib/galaxy/webapps/util.py index 9cb0029d99e..442b938fa5d 100644 --- a/lib/galaxy/webapps/util.py +++ b/lib/galaxy/webapps/util.py @@ -8,6 +8,10 @@ import mako.exceptions log = logging.getLogger(__name__) +class MiddlewareWrapUnsupported(Exception): + pass + + def build_template_error_formatters(): """ Build a list of template error formatters for WebError. When an error @@ -26,8 +30,26 @@ def build_template_error_formatters(): return formatters -def wrap_if_allowed(app, stack, wrap, name=None, args=None, kwargs=None, - alt_wrap=None, alt_name=None, alt_args=None, alt_kwargs=None): +def wrap_if_allowed_or_fail(app, stack, wrap, name=None, args=None, kwargs=None): + """ + Wrap the application with the given method if the application stack allows for it. + + Arguments are the same as for :func:`wrap_if_allowed`. + + Raises :exception:`MiddlewareWrapUnsupported` if the stack does not allow the middleware. + """ + name = name or wrap.__name__ + if not stack.allowed_middleware(wrap): + raise MiddlewareWrapUnsupported( + "'%s' is enabled in your configuration but the %s application stack does not support it, this " + "middleware has been disabled" % (name, stack.name)) + args = args or [] + kwargs = kwargs or {} + log.debug("Enabling '%s' middleware", name) + return wrap(app, *args, **kwargs) + + +def wrap_if_allowed(app, stack, wrap, name=None, args=None, kwargs=None): """ Wrap the application with the given method if the application stack allows for it. @@ -43,16 +65,11 @@ def wrap_if_allowed(app, stack, wrap, name=None, args=None, kwargs=None, :param args: arguments to pass to `wrap` (not including `app` itself) :type kwargs: dict :param kwargs: keyword arguments to pass to `wrap` + + Returns `app` unmodified if the stack does not allow the middleware. """ - name = name or wrap.__name__ - if stack.allowed_middleware(wrap): - args = args or [] - kwargs = kwargs or {} - log.debug("Enabling '%s' middleware", name) - return wrap(app, *args, **kwargs) - else: - log.warning("'%s' is enabled in your configuration but the %s application stack does not support it, this " - "middleware has been disabled", name, stack.name) - if alt_wrap: - app = wrap_if_allowed(app, stack, alt_wrap, name=alt_name, args=alt_args, kwargs=alt_kwargs) + try: + return wrap_if_allowed_or_fail(app, stack, wrap, name=name, args=args, kwargs=kwargs) + except MiddlewareWrapUnsupported as exc: + log.warning(str(exc)) return app