From 14e9dddadec856a1a8aa916b4a4459f5fb487a06 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 25 Aug 2017 14:25:50 -0400 Subject: [PATCH] WIP: support log configuration from yaml/json app config with logging.config.dictConfig. Still working on yaml/json config loading in the webless app. --- lib/galaxy/app.py | 2 +- lib/galaxy/config.py | 12 ++++--- lib/galaxy/web/framework/webapp.py | 22 +++++++++--- lib/galaxy/web/stack/__init__.py | 37 ++++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 37 +++++--------------- scripts/galaxy-main | 49 ++++++++++++++++++--------- 6 files changed, 106 insertions(+), 53 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 51574c4421f..072b7224436 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -51,6 +51,7 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin): # configured. The handler added here gets dumped and replaced with # an appropriately configured logger in configure_logging below. logging.basicConfig(level=logging.DEBUG) + log.debug("python path is: %s", ", ".join( sys.path )) self.name = 'galaxy' self.startup_timer = ExecutionTimer() self.new_installation = False @@ -58,7 +59,6 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin): self.config = config.Configuration(**kwargs) self.config.check() config.configure_logging(self.config) - log.debug("python path is: %s", ", ".join( sys.path )) self.configure_fluent_log() # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions self.application_stack = application_stack_instance(app=self) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 78ee37c627b..c08f0179df0 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -578,6 +578,8 @@ class Configuration(object): self.api_folders = string_as_bool(kwargs.get('api_folders', False)) # This is for testing new library browsing capabilities. self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False)) + # Logging configuration with logging.config.configDict: + self.logging = kwargs.get('logging', {}) # Error logging with sentry self.sentry_dsn = kwargs.get('sentry_dsn', None) # Statistics and profiling with statsd @@ -878,7 +880,7 @@ def configure_logging(config): else: paste_configures_logging = False auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True")) - if auto_configure_logging: + if auto_configure_logging and not config.logging: format = config.get("log_format", "%(name)s %(levelname)s %(asctime)s %(message)s") level = logging._levelNames[config.get("log_level", "DEBUG")] destination = config.get("log_destination", "stdout") @@ -907,9 +909,11 @@ def configure_logging(config): handler.setFormatter(formatter) handler.addFilter(application_stack_log_filter()()) root.addHandler(handler) - else: - for h in root.handlers: - h.addFilter(application_stack_log_filter()()) + elif auto_configure_logging and config.logging: + # TODO: template filename here + logging.config.dictConfig(config.logging) + for h in root.handlers: + h.addFilter(application_stack_log_filter()()) # If sentry is configured, also log to it if getattr(config, "sentry_dsn", None): from raven.handlers.logging import SentryHandler diff --git a/lib/galaxy/web/framework/webapp.py b/lib/galaxy/web/framework/webapp.py index 786e9c4396f..0ed4148b6ad 100644 --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -31,12 +31,14 @@ from galaxy.util import ( safe_str_cmp ) from galaxy.util.sanitize_html import sanitize_html +from galaxy.util.properties import nice_config_parser from galaxy.web.framework import ( base, formbuilder, helpers, url_for ) +from galaxy.web.stack import get_app_kwds log = logging.getLogger(__name__) @@ -961,17 +963,29 @@ def build_native_uwsgi_app(paste_factory, config_section): """uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps. In particular these are useful with --yaml or --json for config.""" + ''' import uwsgi uwsgi_opt = uwsgi.opt config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json") + # legacy, support loading ini uWSGI config without --ini-paste but with the app config under Paste's [app:main] section + if config_file is None and uwsgi_opt.get("ini"): + config_file = uwsgi_opt.get("ini") + parser = nice_config_parser(config_file) + if not parser.has_section(config_section) and parser.has_section("app:main"): + config_section = "app:main" + # support no uWSGI config file or separate app config file, requires setting galaxy_config_file in the uWSGI config + config_file = uwsgi_opt.get("galaxy_config_file") or config_file + if not config_file: # Probably loaded via --ini-paste - expect paste app. return None + ''' + app_kwds = get_app_kwds(config_section, for_paste_app=True) + if not app_kwds: + # Probably loaded via --ini-paste - expect paste app + return None - uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds={ - "config_file": config_file, - "config_section": config_section, - }) + uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds=**app_kwds) return uwsgi_app diff --git a/lib/galaxy/web/stack/__init__.py b/lib/galaxy/web/stack/__init__.py index 9abb0e52e4a..8b930fb6603 100644 --- a/lib/galaxy/web/stack/__init__.py +++ b/lib/galaxy/web/stack/__init__.py @@ -49,6 +49,7 @@ class ApplicationStack(object): log_filter_class = ApplicationStackLogFilter # TODO: this belongs in the pool configuration server_name_template = '{server_name}' + default_app_name = 'main' # used both to route jobs to a pool with this name and indicate whether or # not a stack is using messaging for handler assignment @@ -56,6 +57,11 @@ class ApplicationStack(object): JOB_HANDLERS = 'job-handlers', ) + @staticmethod + def get_app_kwds(config_section, app_name=None, for_paste_app=False): + # TODO: how to implement for Paste/webless + return {} + @classmethod def register_postfork_function(cls, f, *args, **kwargs): f(*args, **kwargs) @@ -156,6 +162,33 @@ class UWSGIApplicationStack(MessageApplicationStack): postfork_functions = [] + @staticmethod + def get_app_kwds(config_section, app_name=None, for_paste_app=False): + kwds = { + 'config_file': None, + 'config_section': config_section, + } + # used by webless mules started under uWSGI + uwsgi_opt = uwsgi.opt + app_section = 'app:%s' % app_name if app_name else 'app:%s' % self.default_app_name + config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json") + # legacy, support loading ini uWSGI config without --ini-paste but with the app config under Paste's [app:main] section + if config_file is None and uwsgi_opt.get("ini"): + config_file = uwsgi_opt["ini"] + parser = nice_config_parser(config_file) + if not parser.has_section(config_section) and parser.has_section(app_section): + kwds['config_section'] = app_section + if config_file is None and for_paste_app: + return None + if config_file is None and uwsgi_opt.get("ini-paste"): + config_file = uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste") + kwds['config_section'] = app_section + if config_file is None: + return None + kwds['config_file'] = config_file + return kwds + + @classmethod def register_postfork_function(cls, f, *args, **kwargs): if uwsgi.mule_id() == 0: @@ -285,6 +318,10 @@ def register_postfork_function(f, *args, **kwargs): application_stack_class().register_postfork_function(f, *args, **kwargs) +def get_app_kwds(config_section): + return application_stack_class().get_app_kwds(config_section) + + def _uwsgi_configured_mules(): mules = uwsgi.opt.get('mule', []) return [mules] if isinstance(mules, string_types) or mules is True else mules diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 9faa25c93ac..7743756dab8 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -6,12 +6,6 @@ import sys import threading import atexit -try: - import configparser -except: - import ConfigParser as configparser - - import galaxy.app import galaxy.model import galaxy.model.mapping @@ -38,16 +32,13 @@ class GalaxyWebApplication(galaxy.web.framework.webapp.WebApplication): pass -def app_factory(global_conf, **kwargs): - return paste_app_factory(global_conf, **kwargs) - - -def paste_app_factory(global_conf, **kwargs): +def app_factory(global_conf, load_app_kwds={}, **kwargs): """ Return a wsgi application serving the root object """ kwargs = load_app_properties( - kwds=kwargs + kwds=kwargs, + **load_app_kwds ) # Create the Galaxy application unless passed in if 'app' in kwargs: @@ -161,22 +152,12 @@ def paste_app_factory(global_conf, **kwargs): return webapp -def uwsgi_app_factory(): - # TODO: synchronize with galaxy.web.framework.webapp.build_native_uwsgi_app - should - # at least be using nice_config_parser for instance. - import uwsgi - root = os.path.abspath(uwsgi.opt.get('galaxy_root', os.getcwd())) - config_file = uwsgi.opt.get('galaxy_config_file', os.path.join(root, 'config', 'galaxy.ini')) - global_conf = { - '__file__': config_file if os.path.exists(__file__) else None, - 'here': root} - parser = configparser.ConfigParser() - parser.read(config_file) - try: - kwargs = dict(parser.items('app:main')) - except configparser.NoSectionError: - kwargs = {} - return app_factory(global_conf, **kwargs) +def uwsgi_app(): + return galaxy.web.framework.webapp.build_native_uwsgi_app(app_factory, "galaxy") + + +# For backwards compatibility +uwsgi_app_factory = uwsgi_app def postfork_setup(): diff --git a/scripts/galaxy-main b/scripts/galaxy-main index f4d576cab2c..d0b968c6a92 100755 --- a/scripts/galaxy-main +++ b/scripts/galaxy-main @@ -45,6 +45,8 @@ try: except ImportError: uwsgi = None +from six import string_types + REQUIRES_DAEMONIZE_MESSAGE = "Attempted to use Galaxy in daemon mode, but daemonize is unavailable." log = logging.getLogger(__name__) @@ -99,7 +101,7 @@ def load_galaxy_app( def app_loop(args, log): try: - config_builder = GalaxyConfigBuilder(args, ini_path=ini_path_from_uwsgi()) + config_builder = GalaxyConfigBuilder(args, config_file=config_file_from_uwsgi()) galaxy_app = load_galaxy_app( config_builder, config_env=True, @@ -143,12 +145,24 @@ def find_ini(supplied_ini, galaxy_root): return guess -def ini_path_from_uwsgi(): +def config_file_from_uwsgi(app_name=None): if uwsgi: + for opt in ('yaml', 'json'): + if opt in uwsgi.opt: + return (uwsgi.opt[opt], app_name or 'galaxy') + if 'ini' in uwsgi.opt: + # FIXME: refactor (to properties) with galaxy.web.framework.webapp:build_native_uwsgi_app if 'ini-paste' in uwsgi.opt: - return uwsgi.opt['ini-paste'] + app_name = 'app:%s' % app_name if app_name else 'app:%s' % DEFAULT_INI_APP + return (uwsgi.opt['ini-paste'], app_name) + + + + # FIXME: still need this in get_app_kwds elif 'set' in uwsgi.opt: - for k, v in [x.split('=', 1) for x in uwsgi.opt['set']]: + set_options = uwsgi.opt['set'] + set_options = [set_options] if isinstance(set_options, string_types) else set_options + for k, v in [x.split('=', 1) for x in set_options]: if k == 'galaxy_config_file': return v return None @@ -159,31 +173,32 @@ class GalaxyConfigBuilder(object): """ def __init__(self, args=None, **kwds): - ini_path = kwds.get("ini_path", None) or (args and args.ini_path) - # If given app_conf_path - use that - else we need to ensure we have an - # ini path. - if not ini_path: + config_file = kwds.get("config_file", None) or (args and args.config_file) or (args and args.ini_path) + # If given app_conf_path - use that - else we need to ensure we have a + # config file path. + if not config_file: galaxy_root = kwds.get("galaxy_root", GALAXY_ROOT_DIR) - ini_path = find_ini(ini_path, galaxy_root) - ini_path = absolute_config_path(ini_path, galaxy_root=galaxy_root) - self.ini_path = ini_path + config_file = find_config(config_file, galaxy_root) + config_file = absolute_config_path(config_file, galaxy_root=galaxy_root) + self.config_file = config_file self.app_name = kwds.get("app") or (args and args.app) or DEFAULT_INI_APP self.log_file = (args and args.log_file) @classmethod def populate_options(cls, arg_parser): - arg_parser.add_argument("-c", "--ini-path", default=None, help="Galaxy ini config file (defaults to config/galaxy.ini)") - arg_parser.add_argument("--app", default=DEFAULT_INI_APP, help="app section in ini file (defaults to main)") + arg_parser.add_argument("-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)") + arg_parser.add_argument("--ini-path", default=None, help="DEPRECATED: use -c/--config-file") + arg_parser.add_argument("--app", default=None, help="app section in ini file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI") arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonzie process", action="store_true") arg_parser.add_argument("--daemon-log-file", default=None, help="log file for daemon script ") - arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in ini_path if set)") + arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)") arg_parser.add_argument("--pid-file", default=DEFAULT_PID, help="pid file (default is %s)" % DEFAULT_PID) arg_parser.add_argument("--server-name", default=None, help="set a galaxy server name") def app_kwds(self): config = dict( - ini_file=self.ini_path, - ini_section="app:%s" % self.app_name, + config_file=self.config_file, + config_section="app:%s" % self.app_name, ) return config @@ -211,6 +226,8 @@ def main(): arg_parser = ArgumentParser(description=DESCRIPTION) GalaxyConfigBuilder.populate_options(arg_parser) args = arg_parser.parse_args() + if args.ini_path and not args.config_file: + args.config_file = args.ini_path if args.log_file: os.environ["GALAXY_CONFIG_LOG_DESTINATION"] = os.path.abspath(args.log_file) if args.server_name: