opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3759 from natefoo/abstract-app-stack

Browse Source 
Abstract uWSGI handling into new galaxy.web.stack package
This commit is contained in:
Dannon Baker
2017-03-20 12:16:45 -04:00
committed by GitHub
parent cd36aa0878 d18b4d13cb
commit 5440889bf6
16 changed files with 300 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/app.py
+4 -3
View File
@@ -23,9 +23,9 @@ from galaxy.openid.providers import OpenIDProviders
from galaxy.tools.data_manager.manager import DataManagers
from galaxy.jobs import metrics as job_metrics
from galaxy.web.proxy import ProxyManager
from galaxy.web.stack import application_stack_instance
from galaxy.queue_worker import GalaxyQueueWorker
from galaxy.util import heartbeat
from galaxy.util.postfork import register_postfork_function
from tool_shed.galaxy_install import update_repository_manager
@@ -44,6 +44,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
log.debug( "python path is: %s", ", ".join( sys.path ) )
self.name = 'galaxy'
self.new_installation = False
self.application_stack = application_stack_instance()
# Read config file and check for errors
self.config = config.Configuration( **kwargs )
self.config.check()
@@ -151,7 +152,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
fname=self.config.heartbeat_log
)
self.heartbeat.daemon = True
register_postfork_function(self.heartbeat.start)
self.application_stack.register_postfork_function(self.heartbeat.start)
self.sentry_client = None
if self.config.sentry_dsn:
@@ -159,7 +160,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ):
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn)
register_postfork_function(postfork_sentry_client)
self.application_stack.register_postfork_function(postfork_sentry_client)
# Transfer manager client
if self.config.get_bool( 'enable_beta_job_managers', False ):
lib/galaxy/config.py
+1 -1
View File
@@ -25,8 +25,8 @@ from galaxy.exceptions import ConfigurationError
from galaxy.util import listify
from galaxy.util import string_as_bool
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.util.postfork import register_postfork_function
from galaxy.web.formatting import expand_pretty_datetime_format
from galaxy.web.stack import register_postfork_function
from .version import VERSION_MAJOR
log = logging.getLogger( __name__ )
lib/galaxy/queue_worker.py
+3 -2
View File
@@ -51,8 +51,9 @@ def send_control_task(app, task, noop_self=False, kwargs={}):
try:
c = Connection(app.config.amqp_internal_connection)
with producers[c].acquire(block=True) as producer:
control_queues = galaxy.queues.all_control_queues_for_declare(app.config, app.application_stack)
producer.publish(payload, exchange=galaxy.queues.galaxy_exchange,
declare=[galaxy.queues.galaxy_exchange] + galaxy.queues.all_control_queues_for_declare(app.config),
declare=[galaxy.queues.galaxy_exchange] + control_queues,
routing_key='control')
except Exception:
# This is likely connection refused.
@@ -204,7 +205,7 @@ class GalaxyQueueWorker(ConsumerMixin, threading.Thread):
# Default to figuring out which control queue to use based on the app config.
queue = galaxy.queues.control_queue_from_config(app.config)
self.task_mapping = task_mapping
self.declare_queues = galaxy.queues.all_control_queues_for_declare(app.config)
self.declare_queues = galaxy.queues.all_control_queues_for_declare(app.config, app.application_stack)
# TODO we may want to purge the queue at the start to avoid executing
# stale 'reload_tool', etc messages. This can happen if, say, a web
# process goes down and messages get sent before it comes back up.
lib/galaxy/queues.py
+3 -8
View File
@@ -4,26 +4,21 @@ All message queues used by Galaxy
"""
from galaxy.util.postfork import process_is_uwsgi
from kombu import Exchange, Queue, Connection
ALL_CONTROL = "control.*"
galaxy_exchange = Exchange('galaxy_core_exchange', type='topic')
def all_control_queues_for_declare(config):
def all_control_queues_for_declare(config, application_stack):
"""
For in-memory routing (used by sqlalchemy-based transports), we need to be able to
build the entire routing table in producers.
Refactor later to actually persist this somewhere instead of building it repeatedly.
"""
possible_uwsgi_queues = []
if process_is_uwsgi:
import uwsgi
possible_uwsgi_queues = [Queue("control.%s.%s" % (config.server_name.split('.')[0], wkr['id']), galaxy_exchange, routing_key='control') for wkr in uwsgi.workers()]
return possible_uwsgi_queues + [Queue('control.%s' % q, galaxy_exchange, routing_key='control') for q in config.server_names]
possible_stack_queues = [Queue("control.%s.%s" % (config.server_name.split('.')[0], wkr['id']), galaxy_exchange, routing_key='control') for wkr in application_stack.workers()]
return possible_stack_queues + [Queue('control.%s' % q, galaxy_exchange, routing_key='control') for q in config.server_names]
def control_queue_from_config(config):
lib/galaxy/tools/toolbox/watcher.py
+1 -1
View File
@@ -15,7 +15,7 @@ except ImportError:
can_watch = False
from galaxy.util.hash_util import md5_hash_file
from galaxy.util.postfork import register_postfork_function
from galaxy.web.stack import register_postfork_function
log = logging.getLogger( __name__ )
lib/galaxy/util/postfork.py
-43
View File
@@ -1,43 +0,0 @@
"""
Handle postfork functions under uWSGI
"""
# The uwsgi module is automatically injected by the parent uwsgi
# process and only exists that way. If anything works, this is a
# uwsgi-managed process.
try:
import uwsgi
if hasattr(uwsgi, "numproc"):
process_is_uwsgi = True
else:
process_is_uwsgi = False
except ImportError:
# This is not a uwsgi process, or something went horribly wrong.
process_is_uwsgi = False
try:
from uwsgidecorators import postfork
except:
def pf_dec(func):
return func
postfork = pf_dec
if process_is_uwsgi:
print("WARNING: This is a uwsgi process but the uwsgidecorators library"
" is unavailable. This is likely due to using an external (not"
" in Galaxy's virtualenv) uwsgi and you may experience errors.")
postfork_functions = []
@postfork
def do_postfork():
for f, args, kwargs in [ t for t in postfork_functions ]:
f(*args, **kwargs)
def register_postfork_function(f, *args, **kwargs):
if process_is_uwsgi:
postfork_functions.append((f, args, kwargs))
else:
f(*args, **kwargs)
lib/galaxy/web/framework/middleware/batch.py
+3 -3
View File
@@ -59,11 +59,11 @@ class BatchMiddleware( object ):
]
}
def __init__( self, galaxy, application, config=None ):
#: the original galaxy webapp
self.galaxy = galaxy
def __init__( self, application, galaxy, config=None ):
#: the wrapped webapp
self.application = application
#: the original galaxy webapp
self.galaxy = galaxy
self.config = self.DEFAULT_CONFIG.copy()
self.config.update( config )
self.base_url = routes.url_for( '/' )
lib/galaxy/web/framework/middleware/sentry.py
+1 -1
View File
@@ -12,7 +12,7 @@ try:
except:
Client = None
from galaxy.util.postfork import register_postfork_function
from galaxy.web.stack import register_postfork_function
RAVEN_IMPORT_MESSAGE = ('The Python raven package is required to use this '
lib/galaxy/web/stack/__init__.py
+106
View File
@@ -0,0 +1,106 @@
"""Web application stack operations
"""
from __future__ import print_function
import inspect
import logging
import os
# The uwsgi module is automatically injected by the parent uwsgi
# process and only exists that way. If anything works, this is a
# uwsgi-managed process.
try:
import uwsgi
except ImportError:
uwsgi = None
try:
from uwsgidecorators import postfork as uwsgi_postfork
except:
uwsgi_postfork = lambda x: x # noqa: E731
if uwsgi is not None and hasattr(uwsgi, 'numproc'):
print("WARNING: This is a uwsgi process but the uwsgidecorators library"
" is unavailable. This is likely due to using an external (not"
" in Galaxy's virtualenv) uwsgi and you may experience errors. "
"HINT:\n {venv}/bin/pip install uwsgidecorators".format(
venv=os.environ.get('VIRTUAL_ENV', '/path/to/venv')))
log = logging.getLogger(__name__)
class ApplicationStack(object):
name = None
prohibited_middleware = frozenset()
@classmethod
def register_postfork_function(cls, f, *args, **kwargs):
f(*args, **kwargs)
def workers(self):
return []
def allowed_middleware(self, middleware):
if hasattr(middleware, '__name__'):
middleware = middleware.__name__
return middleware not in self.prohibited_middleware
def set_postfork_server_name(self, app):
pass
class UWSGIApplicationStack(ApplicationStack):
name = 'uWSGI'
prohibited_middleware = frozenset([
'wrap_in_static',
'EvalException',
])
postfork_functions = []
@classmethod
def register_postfork_function(cls, f, *args, **kwargs):
cls.postfork_functions.append((f, args, kwargs))
def workers(self):
return uwsgi.workers()
def set_postfork_server_name(self, app):
app.config.server_name += ".%d" % uwsgi.worker_id()
class PasteApplicationStack(ApplicationStack):
name = 'Python Paste'
class WeblessApplicationStack(ApplicationStack):
name = 'Webless'
def application_stack_class():
"""Returns the correct ApplicationStack class for the stack under which
this Galaxy process is running.
"""
if uwsgi is not None and hasattr(uwsgi, 'numproc'):
return UWSGIApplicationStack
else:
# cleverer ideas welcome
for frame in inspect.stack():
if frame[1].endswith(os.path.join('pastescript', 'loadwsgi.py')):
return PasteApplicationStack
return WeblessApplicationStack
def application_stack_instance():
stack_class = application_stack_class()
return stack_class()
def register_postfork_function(f, *args, **kwargs):
application_stack_class().register_postfork_function(f, *args, **kwargs)
@uwsgi_postfork
def _do_postfork():
for f, args, kwargs in [ t for t in UWSGIApplicationStack.postfork_functions ]:
f(*args, **kwargs)
lib/galaxy/webapps/galaxy/buildapp.py
+47 -52
View File
@@ -19,10 +19,14 @@ import galaxy.model.mapping
import galaxy.datatypes.registry
import galaxy.web.framework
import galaxy.web.framework.webapp
from galaxy.webapps.util import build_template_error_formatters
from galaxy.webapps.util import (
MiddlewareWrapUnsupported,
build_template_error_formatters,
wrap_if_allowed,
wrap_if_allowed_or_fail
)
from galaxy import util
from galaxy.util import asbool
from galaxy.util.postfork import process_is_uwsgi, register_postfork_function
from galaxy.util.properties import load_app_properties
from paste import httpexceptions
@@ -117,12 +121,11 @@ def paste_app_factory( global_conf, **kwargs ):
# Wrap the webapp in some useful middleware
if kwargs.get( 'middleware', True ):
webapp = wrap_in_middleware( webapp, global_conf, **kwargs )
webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs)
if asbool( kwargs.get( 'static_enabled', True) ):
if process_is_uwsgi:
log.error("Static middleware is enabled in your configuration but this is a uwsgi process. Refusing to wrap in static middleware.")
else:
webapp = wrap_in_static( webapp, global_conf, plugin_frameworks=[ app.visualizations_registry ], **kwargs )
webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static,
args=(global_conf,),
kwargs=dict(plugin_frameworks=[app.visualizations_registry], **kwargs))
# Close any pooled database connections before forking
try:
galaxy.model.mapping.metadata.bind.dispose()
@@ -135,7 +138,7 @@ def paste_app_factory( global_conf, **kwargs ):
except:
log.exception("Unable to dispose of pooled toolshed install model database connections.")
register_postfork_function(postfork_setup)
app.application_stack.register_postfork_function(postfork_setup)
for th in threading.enumerate():
if th.is_alive():
@@ -162,9 +165,7 @@ def uwsgi_app_factory():
def postfork_setup():
from galaxy.app import app
if process_is_uwsgi:
import uwsgi
app.config.server_name += ".%s" % uwsgi.worker_id()
app.application_stack.set_postfork_server_name(app)
app.control_worker.bind_and_start()
@@ -848,12 +849,13 @@ def _add_item_provenance_controller( webapp, name_prefix, path_prefix, **kwd ):
webapp.mapper.resource(name, "provenance", path_prefix=path_prefix, controller=controller)
def wrap_in_middleware( app, global_conf, **local_conf ):
def wrap_in_middleware( app, global_conf, application_stack, **local_conf ):
"""
Based on the configuration wrap `app` in a set of common and useful
middleware.
"""
webapp = app
stack = application_stack
# Merge the global and local configurations
conf = global_conf.copy()
@@ -862,16 +864,15 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware( app, conf )
log.debug( "Enabling 'httpexceptions' middleware" )
app = wrap_if_allowed( app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,) )
# Statsd request timing and profiling
statsd_host = conf.get('statsd_host', None)
if statsd_host:
from galaxy.web.framework.middleware.statsd import StatsdMiddleware
app = StatsdMiddleware( app,
statsd_host,
conf.get('statsd_port', 8125),
conf.get('statsd_prefix', 'galaxy') )
app = wrap_if_allowed( app, stack, StatsdMiddleware,
args=( statsd_host,
conf.get('statsd_port', 8125),
conf.get('statsd_prefix', 'galaxy') ) )
log.debug( "Enabling 'statsd' middleware" )
# If we're using remote_user authentication, add middleware that
# protects Galaxy from improperly configured authentication in the
@@ -880,25 +881,26 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
use_remote_user = asbool(conf.get( 'use_remote_user', False )) or single_user
if use_remote_user:
from galaxy.web.framework.middleware.remoteuser import RemoteUser
app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ),
display_servers=util.listify( conf.get( 'display_servers', '' ) ),
single_user=single_user,
admin_users=conf.get( 'admin_users', '' ).split( ',' ),
remote_user_header=conf.get( 'remote_user_header', 'HTTP_REMOTE_USER' ),
remote_user_secret_header=conf.get('remote_user_secret', None),
normalize_remote_user_email=conf.get('normalize_remote_user_email', False))
app = wrap_if_allowed( app, stack, RemoteUser,
kwargs=dict(
maildomain=conf.get('remote_user_maildomain', None),
display_servers=util.listify( conf.get('display_servers', '')),
single_user=single_user,
admin_users=conf.get('admin_users', '').split(','),
remote_user_header=conf.get('remote_user_header', 'HTTP_REMOTE_USER'),
remote_user_secret_header=conf.get('remote_user_secret', None),
normalize_remote_user_email=conf.get('normalize_remote_user_email', False)) )
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware( app, conf )
log.debug( "Enabling 'recursive' middleware" )
app = wrap_if_allowed( app, stack, recursive.RecursiveMiddleware, args=(conf,) )
# If sentry logging is enabled, log here before propogating up to
# the error middleware
sentry_dsn = conf.get( 'sentry_dsn', None )
if sentry_dsn:
from galaxy.web.framework.middleware.sentry import Sentry
app = Sentry( app, sentry_dsn )
app = wrap_if_allowed( app, stack, Sentry, args=(sentry_dsn,) )
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
@@ -906,48 +908,41 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# Middleware to check for WSGI compliance
if asbool( conf.get( 'use_lint', False ) ):
from paste import lint
app = lint.make_middleware( app, conf )
log.debug( "Enabling 'lint' middleware" )
app = wrap_if_allowed( app, stack, lint.make_middleware, name='paste.lint', args=(conf,) )
# Middleware to run the python profiler on each request
if asbool( conf.get( 'use_profile', False ) ):
from paste.debug import profile
app = profile.ProfileMiddleware( app, conf )
log.debug( "Enabling 'profile' middleware" )
if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi:
app = wrap_if_allowed( app, stack, profile.ProfileMiddleware, args=(conf,) )
if debug and asbool( conf.get( 'use_interactive', False ) ):
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
from weberror import evalexception
app = evalexception.EvalException( app, conf,
templating_formatters=build_template_error_formatters() )
log.debug( "Enabling 'eval exceptions' middleware" )
try:
from weberror import evalexception
app = wrap_if_allowed_or_fail( app, stack, evalexception.EvalException,
args=(conf,),
kwargs=dict(templating_formatters=build_template_error_formatters()) )
except MiddlewareWrapUnsupported as exc:
log.warning(str(exc))
import galaxy.web.framework.middleware.error
app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) )
else:
if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi:
log.error("Interactive debugging middleware is enabled in your configuration "
"but this is a uwsgi process. Refusing to wrap in interactive error middleware.")
# Not in interactive debug mode, just use the regular error middleware
import galaxy.web.framework.middleware.error
app = galaxy.web.framework.middleware.error.ErrorMiddleware( app, conf )
log.debug( "Enabling 'error' middleware" )
app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) )
# Transaction logging (apache access.log style)
if asbool( conf.get( 'use_translogger', True ) ):
from galaxy.web.framework.middleware.translogger import TransLogger
app = TransLogger( app )
log.debug( "Enabling 'trans logger' middleware" )
app = wrap_if_allowed( app, stack, TransLogger )
# X-Forwarded-Host handling
from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware
app = XForwardedHostMiddleware( app )
log.debug( "Enabling 'x-forwarded-host' middleware" )
app = wrap_if_allowed( app, stack, XForwardedHostMiddleware )
# Request ID middleware
from galaxy.web.framework.middleware.request_id import RequestIDMiddleware
app = RequestIDMiddleware( app )
log.debug( "Enabling 'Request ID' middleware" )
app = wrap_if_allowed( app, stack, RequestIDMiddleware )
# api batch call processing middleware
from galaxy.web.framework.middleware.batch import BatchMiddleware
app = BatchMiddleware( webapp, app, {})
log.debug( "Enabling 'Batch' middleware" )
app = wrap_if_allowed( app, stack, BatchMiddleware, args=(webapp, {}) )
return app
lib/galaxy/webapps/reports/app.py
+2
View File
@@ -4,6 +4,7 @@ import time
import galaxy.model
from galaxy.web import security
from galaxy.web.stack import application_stack_instance
import logging
log = logging.getLogger( __name__ )
@@ -17,6 +18,7 @@ class UniverseApplication( object ):
self.config = config.Configuration( **kwargs )
self.config.check()
config.configure_logging( self.config )
self.application_stack = application_stack_instance()
# Determine the database url
if self.config.database_connection:
db_url = self.config.database_connection
lib/galaxy/webapps/reports/buildapp.py
+31 -31
View File
@@ -11,9 +11,12 @@ from inspect import isclass
from paste import httpexceptions
from galaxy.util import asbool
from galaxy.util.postfork import process_is_uwsgi
from galaxy.webapps.util import build_template_error_formatters
from galaxy.webapps.util import (
MiddlewareWrapUnsupported,
build_template_error_formatters,
wrap_if_allowed,
wrap_if_allowed_or_fail
)
import galaxy.model
import galaxy.model.mapping
import galaxy.web.framework.webapp
@@ -69,9 +72,11 @@ def app_factory( global_conf, **kwargs ):
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get( 'middleware', True ):
webapp = wrap_in_middleware( webapp, global_conf, **kwargs )
webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs)
if asbool( kwargs.get( 'static_enabled', True ) ):
webapp = wrap_in_static( webapp, global_conf, **kwargs )
webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static,
args=(global_conf,),
kwargs=kwargs)
# Close any pooled database connections before forking
try:
galaxy.model.mapping.metadata.bind.dispose()
@@ -81,8 +86,9 @@ def app_factory( global_conf, **kwargs ):
return webapp
def wrap_in_middleware( app, global_conf, **local_conf ):
def wrap_in_middleware( app, global_conf, application_stack, **local_conf ):
"""Based on the configuration wrap `app` in a set of common and useful middleware."""
stack = application_stack
# Merge the global and local configurations
conf = global_conf.copy()
conf.update(local_conf)
@@ -90,14 +96,12 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware( app, conf )
log.debug( "Enabling 'httpexceptions' middleware" )
app = wrap_if_allowed( app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,) )
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware( app, conf )
log.debug( "Enabling 'recursive' middleware" )
app = wrap_if_allowed( app, stack, recursive.RecursiveMiddleware, args=(conf,) )
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance
@@ -105,44 +109,40 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# Middleware to check for WSGI compliance
if asbool( conf.get( 'use_lint', True ) ):
from paste import lint
app = lint.make_middleware( app, conf )
log.debug( "Enabling 'lint' middleware" )
app = wrap_if_allowed( app, stack, lint.make_middleware, name='paste.lint', args=(conf,) )
# Middleware to run the python profiler on each request
if asbool( conf.get( 'use_profile', False ) ):
import profile
app = profile.ProfileMiddleware( app, conf )
log.debug( "Enabling 'profile' middleware" )
app = wrap_if_allowed( app, stack, profile.ProfileMiddleware, args=(conf,) )
# Middleware that intercepts print statements and shows them on the
# returned page
if asbool( conf.get( 'use_printdebug', True ) ):
from paste.debug import prints
app = prints.PrintDebugMiddleware( app, conf )
log.debug( "Enabling 'print debug' middleware" )
if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi:
app = wrap_if_allowed( app, stack, prints.PrintDebugMiddleware, args=(conf,) )
if debug and asbool( conf.get( 'use_interactive', False ) ):
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
from weberror import evalexception
app = evalexception.EvalException( app, conf,
templating_formatters=build_template_error_formatters() )
log.debug( "Enabling 'eval exceptions' middleware" )
try:
from weberror import evalexception
app = wrap_if_allowed_or_fail( app, stack, evalexception.EvalException,
args=(conf,),
kwargs=dict(templating_formatters=build_template_error_formatters()) )
except MiddlewareWrapUnsupported as exc:
log.warning(str(exc))
import galaxy.web.framework.middleware.error
app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) )
else:
if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi:
log.error("Interactive debugging middleware is enabled in your configuration "
"but this is a uwsgi process. Refusing to wrap in interactive error middleware.")
# Not in interactive debug mode, just use the regular error middleware
from paste.exceptions import errormiddleware
app = errormiddleware.ErrorMiddleware( app, conf )
log.debug( "Enabling 'error' middleware" )
import galaxy.web.framework.middleware.error
app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) )
# Transaction logging (apache access.log style)
if asbool( conf.get( 'use_translogger', True ) ):
from paste.translogger import TransLogger
app = TransLogger( app )
log.debug( "Enabling 'trans logger' middleware" )
app = wrap_if_allowed( app, stack, TransLogger )
# X-Forwarded-Host handling
from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware
app = XForwardedHostMiddleware( app )
log.debug( "Enabling 'x-forwarded-host' middleware" )
app = wrap_if_allowed( app, stack, XForwardedHostMiddleware )
return app
lib/galaxy/webapps/tool_shed/app.py
+2
View File
@@ -11,6 +11,7 @@ from galaxy.managers.tags import CommunityTagManager
from galaxy.openid.providers import OpenIDProviders
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.web import security
from galaxy.web.stack import application_stack_instance
import tool_shed.repository_registry
import tool_shed.repository_types.registry
from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager
@@ -29,6 +30,7 @@ class UniverseApplication( object ):
self.config = config.Configuration( **kwd )
self.config.check()
configure_logging( self.config )
self.application_stack = application_stack_instance()
# Initialize the Galaxy datatypes registry.
self.datatypes_registry = galaxy.datatypes.registry.Registry()
self.datatypes_registry.load_datatypes( self.config.root, self.config.datatypes_config )
lib/galaxy/webapps/tool_shed/buildapp.py
+41 -41
View File
@@ -14,9 +14,13 @@ from galaxy.util import asbool
import galaxy.webapps.tool_shed.model
import galaxy.webapps.tool_shed.model.mapping
import galaxy.web.framework.webapp
from galaxy.webapps.util import build_template_error_formatters
from galaxy.webapps.util import (
MiddlewareWrapUnsupported,
build_template_error_formatters,
wrap_if_allowed,
wrap_if_allowed_or_fail
)
from galaxy import util
from galaxy.util.postfork import process_is_uwsgi
from galaxy.util.properties import load_app_properties
from routes.middleware import RoutesMiddleware
@@ -186,12 +190,11 @@ def app_factory( global_conf, **kwargs ):
webapp.finalize_config()
# Wrap the webapp in some useful middleware
if kwargs.get( 'middleware', True ):
webapp = wrap_in_middleware( webapp, global_conf, **kwargs )
webapp = wrap_in_middleware(webapp, global_conf, app.application_stack, **kwargs)
if asbool( kwargs.get( 'static_enabled', True) ):
if process_is_uwsgi:
log.error("Static middleware is enabled in your configuration but this is a uwsgi process. Refusing to wrap in static middleware.")
else:
webapp = wrap_in_static( webapp, global_conf, **kwargs )
webapp = wrap_if_allowed(webapp, app.application_stack, wrap_in_static,
args=(global_conf,),
kwargs=kwargs)
# Close any pooled database connections before forking
try:
galaxy.webapps.tool_shed.model.mapping.metadata.bind.dispose()
@@ -201,8 +204,9 @@ def app_factory( global_conf, **kwargs ):
return webapp
def wrap_in_middleware( app, global_conf, **local_conf ):
def wrap_in_middleware( app, global_conf, application_stack, **local_conf ):
"""Based on the configuration wrap `app` in a set of common and useful middleware."""
stack = application_stack
# Merge the global and local configurations
conf = global_conf.copy()
conf.update( local_conf )
@@ -210,63 +214,62 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# First put into place httpexceptions, which must be most closely
# wrapped around the application (it can interact poorly with
# other middleware):
app = httpexceptions.make_middleware( app, conf )
log.debug( "Enabling 'httpexceptions' middleware" )
app = wrap_if_allowed( app, stack, httpexceptions.make_middleware, name='paste.httpexceptions', args=(conf,) )
# Create a separate mapper for redirects to prevent conflicts.
redirect_mapper = routes.Mapper()
redirect_mapper = _map_redirects( redirect_mapper )
# Load the Routes middleware which we use for redirecting
app = RoutesMiddleware( app, redirect_mapper )
log.debug( "Enabling 'routes' middleware" )
app = wrap_if_allowed( app, stack, RoutesMiddleware, args=(redirect_mapper,) )
# If we're using remote_user authentication, add middleware that
# protects Galaxy from improperly configured authentication in the
# upstream server
if asbool(conf.get( 'use_remote_user', False )):
from galaxy.webapps.tool_shed.framework.middleware.remoteuser import RemoteUser
app = RemoteUser( app, maildomain=conf.get( 'remote_user_maildomain', None ),
display_servers=util.listify( conf.get( 'display_servers', '' ) ),
admin_users=conf.get( 'admin_users', '' ).split( ',' ),
remote_user_secret_header=conf.get('remote_user_secret', None) )
log.debug( "Enabling 'remote user' middleware" )
app = wrap_if_allowed( app, stack, RemoteUser,
kwargs=dict(
maildomain=conf.get('remote_user_maildomain', None),
display_servers=util.listify( conf.get('display_servers', '')),
admin_users=conf.get('admin_users', '').split(','),
remote_user_header=conf.get('remote_user_header', 'HTTP_REMOTE_USER'),
remote_user_secret_header=conf.get('remote_user_secret', None),
normalize_remote_user_email=conf.get('normalize_remote_user_email', False)) )
# The recursive middleware allows for including requests in other
# requests or forwarding of requests, all on the server side.
if asbool(conf.get('use_recursive', True)):
from paste import recursive
app = recursive.RecursiveMiddleware( app, conf )
log.debug( "Enabling 'recursive' middleware" )
if debug and asbool( conf.get( 'use_interactive', False ) ) and not process_is_uwsgi:
app = wrap_if_allowed( app, stack, recursive.RecursiveMiddleware, args=(conf,) )
if debug and asbool( conf.get( 'use_interactive', False ) ):
# Interactive exception debugging, scary dangerous if publicly
# accessible, if not enabled we'll use the regular error printing
# middleware.
from weberror import evalexception
app = evalexception.EvalException( app, conf,
templating_formatters=build_template_error_formatters() )
log.debug( "Enabling 'eval exceptions' middleware" )
try:
from weberror import evalexception
app = wrap_if_allowed_or_fail( app, stack, evalexception.EvalException,
args=(conf,),
kwargs=dict(templating_formatters=build_template_error_formatters()) )
except MiddlewareWrapUnsupported as exc:
log.warning(str(exc))
import galaxy.web.framework.middleware.error
app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) )
else:
if debug and asbool( conf.get( 'use_interactive', False ) ) and process_is_uwsgi:
log.error("Interactive debugging middleware is enabled in your configuration "
"but this is a uwsgi process. Refusing to wrap in interactive error middleware.")
# Not in interactive debug mode, just use the regular error middleware
import galaxy.web.framework.middleware.error
app = galaxy.web.framework.middleware.error.ErrorMiddleware( app, conf )
log.debug( "Enabling 'error' middleware" )
app = wrap_if_allowed( app, stack, galaxy.web.framework.middleware.error.ErrorMiddleware, args=(conf,) )
# Transaction logging (apache access.log style)
if asbool( conf.get( 'use_translogger', True ) ):
from paste.translogger import TransLogger
app = TransLogger( app )
log.debug( "Enabling 'trans logger' middleware" )
app = wrap_if_allowed( app, stack, TransLogger )
# If sentry logging is enabled, log here before propogating up to
# the error middleware
# TODO sentry config is duplicated between tool_shed/galaxy, refactor this.
sentry_dsn = conf.get( 'sentry_dsn', None )
if sentry_dsn:
from galaxy.web.framework.middleware.sentry import Sentry
log.debug( "Enabling 'sentry' middleware" )
app = Sentry( app, sentry_dsn )
app = wrap_if_allowed( app, stack, Sentry, args=(sentry_dsn,) )
# X-Forwarded-Host handling
from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware
app = XForwardedHostMiddleware( app )
log.debug( "Enabling 'x-forwarded-host' middleware" )
app = wrap_if_allowed( app, stack, XForwardedHostMiddleware )
# Various debug middleware that can only be turned on if the debug
# flag is set, either because they are insecure or greatly hurt
# performance. The print debug middleware needs to be loaded last,
@@ -276,19 +279,16 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# Middleware to check for WSGI compliance
if asbool( conf.get( 'use_lint', True ) ):
from paste import lint
app = lint.make_middleware( app, conf )
log.debug( "Enabling 'lint' middleware" )
app = wrap_if_allowed( app, stack, lint.make_middleware, name='paste.lint', args=(conf,) )
# Middleware to run the python profiler on each request
if asbool( conf.get( 'use_profile', False ) ):
import profile
app = profile.ProfileMiddleware( app, conf )
log.debug( "Enabling 'profile' middleware" )
app = wrap_if_allowed( app, stack, profile.ProfileMiddleware, args=(conf,) )
# Middleware that intercepts print statements and shows them on the
# returned page
if asbool( conf.get( 'use_printdebug', True ) ):
from paste.debug import prints
app = prints.PrintDebugMiddleware( app, conf )
log.debug( "Enabling 'print debug' middleware" )
app = wrap_if_allowed( app, stack, prints.PrintDebugMiddleware, args=(conf,) )
return app
lib/galaxy/webapps/util.py
+54
View File
@@ -1,8 +1,17 @@
from __future__ import absolute_import
import logging
import mako.exceptions
log = logging.getLogger(__name__)
class MiddlewareWrapUnsupported(Exception):
pass
def build_template_error_formatters():
"""
Build a list of template error formatters for WebError. When an error
@@ -19,3 +28,48 @@ def build_template_error_formatters():
return mako.exceptions.html_error_template().render( full=False, css=False )
formatters.append( mako_html_data )
return formatters
def wrap_if_allowed_or_fail(app, stack, wrap, name=None, args=None, kwargs=None):
"""
Wrap the application with the given method if the application stack allows for it.
Arguments are the same as for :func:`wrap_if_allowed`.
Raises :exception:`MiddlewareWrapUnsupported` if the stack does not allow the middleware.
"""
name = name or wrap.__name__
if not stack.allowed_middleware(wrap):
raise MiddlewareWrapUnsupported(
"'%s' is enabled in your configuration but the %s application stack does not support it, this "
"middleware has been disabled" % (name, stack.name))
args = args or []
kwargs = kwargs or {}
log.debug("Enabling '%s' middleware", name)
return wrap(app, *args, **kwargs)
def wrap_if_allowed(app, stack, wrap, name=None, args=None, kwargs=None):
"""
Wrap the application with the given method if the application stack allows for it.
:type app: :class:`galaxy.web.framework.webapp.WebApplication` subclass
:param app: application to wrap
:type stack: :class:`galaxy.web.stack.ApplicationStack` subclass
:param stack: instance of application stack implementing `allowed_middleware()` method
:type wrap: types.FunctionType or types.LambdaType
:param wrap: function to wrap application with
:type name: str
:param name: alternative wrap function name for logging purposes (`wrap.__name__` if None)
:type args: list
:param args: arguments to pass to `wrap` (not including `app` itself)
:type kwargs: dict
:param kwargs: keyword arguments to pass to `wrap`
Returns `app` unmodified if the stack does not allow the middleware.
"""
try:
return wrap_if_allowed_or_fail(app, stack, wrap, name=name, args=args, kwargs=kwargs)
except MiddlewareWrapUnsupported as exc:
log.warning(str(exc))
return app
lib/tool_shed/galaxy_install/update_repository_manager.py
+1 -2
View File
@@ -8,7 +8,6 @@ from sqlalchemy import false
import tool_shed.util.shed_util_common as suc
from galaxy import util
from galaxy.util.postfork import register_postfork_function
from tool_shed.util import common_util
from tool_shed.util import encoding_util
from tool_shed.util import repository_util
@@ -27,7 +26,7 @@ class UpdateRepositoryManager( object ):
self.sleeper = Sleeper()
self.restarter = threading.Thread( target=self.__restarter )
self.restarter.daemon = True
register_postfork_function(self.restarter.start)
self.app.application_stack.register_postfork_function(self.restarter.start)
self.seconds_to_sleep = int( app.config.hours_between_check * 3600 )
def get_update_to_changeset_revision_and_ctx_rev( self, repository ):