WIP: support log configuration from yaml/json app config with

logging.config.dictConfig. Still working on yaml/json config loading in
the webless app.

lib/galaxy/app.py

@@ -51,6 +51,7 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
             # configured.  The handler added here gets dumped and replaced with
             # an appropriately configured logger in configure_logging below.
             logging.basicConfig(level=logging.DEBUG)
+        log.debug("python path is: %s", ", ".join( sys.path ))
         self.name = 'galaxy'
         self.startup_timer = ExecutionTimer()
         self.new_installation = False
@@ -58,7 +59,6 @@ class UniverseApplication(object, config.ConfiguresGalaxyMixin):
         self.config = config.Configuration(**kwargs)
         self.config.check()
         config.configure_logging(self.config)
-        log.debug("python path is: %s", ", ".join( sys.path ))
         self.configure_fluent_log()
         # A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
         self.application_stack = application_stack_instance(app=self)

lib/galaxy/config.py

@@ -578,6 +578,8 @@ class Configuration(object):
         self.api_folders = string_as_bool(kwargs.get('api_folders', False))
         # This is for testing new library browsing capabilities.
         self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False))
+        # Logging configuration with logging.config.configDict:
+        self.logging = kwargs.get('logging', {})
         # Error logging with sentry
         self.sentry_dsn = kwargs.get('sentry_dsn', None)
         # Statistics and profiling with statsd
@@ -878,7 +880,7 @@ def configure_logging(config):
     else:
         paste_configures_logging = False
     auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True"))
-    if auto_configure_logging:
+    if auto_configure_logging and not config.logging:
         format = config.get("log_format", "%(name)s %(levelname)s %(asctime)s %(message)s")
         level = logging._levelNames[config.get("log_level", "DEBUG")]
         destination = config.get("log_destination", "stdout")
@@ -907,9 +909,11 @@ def configure_logging(config):
         handler.setFormatter(formatter)
         handler.addFilter(application_stack_log_filter()())
         root.addHandler(handler)
-    else:
-        for h in root.handlers:
-            h.addFilter(application_stack_log_filter()())
+    elif auto_configure_logging and config.logging:
+        # TODO: template filename here
+        logging.config.dictConfig(config.logging)
+    for h in root.handlers:
+        h.addFilter(application_stack_log_filter()())
     # If sentry is configured, also log to it
     if getattr(config, "sentry_dsn", None):
         from raven.handlers.logging import SentryHandler

lib/galaxy/web/framework/webapp.py

@@ -31,12 +31,14 @@ from galaxy.util import (
     safe_str_cmp
 )
 from galaxy.util.sanitize_html import sanitize_html
+from galaxy.util.properties import nice_config_parser
 from galaxy.web.framework import (
     base,
     formbuilder,
     helpers,
     url_for
 )
+from galaxy.web.stack import get_app_kwds
 
 log = logging.getLogger(__name__)
 
@@ -961,17 +963,29 @@ def build_native_uwsgi_app(paste_factory, config_section):
     """uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps.
 
     In particular these are useful with --yaml or --json for config."""
+    '''
     import uwsgi
     uwsgi_opt = uwsgi.opt
     config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
+    # legacy, support loading ini uWSGI config without --ini-paste but with the app config under Paste's [app:main] section
+    if config_file is None and uwsgi_opt.get("ini"):
+        config_file = uwsgi_opt.get("ini")
+        parser = nice_config_parser(config_file)
+        if not parser.has_section(config_section) and parser.has_section("app:main"):
+            config_section = "app:main"
+    # support no uWSGI config file or separate app config file, requires setting galaxy_config_file in the uWSGI config
+    config_file = uwsgi_opt.get("galaxy_config_file") or config_file
+
     if not config_file:
         # Probably loaded via --ini-paste - expect paste app.
         return None
+    '''
+    app_kwds = get_app_kwds(config_section, for_paste_app=True)
+    if not app_kwds:
+        # Probably loaded via --ini-paste - expect paste app
+        return None
 
-    uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds={
-        "config_file": config_file,
-        "config_section": config_section,
-    })
+    uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds=**app_kwds)
     return uwsgi_app
 
 

lib/galaxy/web/stack/__init__.py

@@ -49,6 +49,7 @@ class ApplicationStack(object):
     log_filter_class = ApplicationStackLogFilter
     # TODO: this belongs in the pool configuration
     server_name_template = '{server_name}'
+    default_app_name = 'main'
 
     # used both to route jobs to a pool with this name and indicate whether or
     # not a stack is using messaging for handler assignment
@@ -56,6 +57,11 @@ class ApplicationStack(object):
         JOB_HANDLERS = 'job-handlers',
     )
 
+    @staticmethod
+    def get_app_kwds(config_section, app_name=None, for_paste_app=False):
+        # TODO: how to implement for Paste/webless
+        return {}
+
     @classmethod
     def register_postfork_function(cls, f, *args, **kwargs):
         f(*args, **kwargs)
@@ -156,6 +162,33 @@ class UWSGIApplicationStack(MessageApplicationStack):
 
     postfork_functions = []
 
+    @staticmethod
+    def get_app_kwds(config_section, app_name=None, for_paste_app=False):
+        kwds = {
+            'config_file': None,
+            'config_section': config_section,
+        }
+        # used by webless mules started under uWSGI
+        uwsgi_opt = uwsgi.opt
+        app_section = 'app:%s' % app_name if app_name else 'app:%s' % self.default_app_name
+        config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
+        # legacy, support loading ini uWSGI config without --ini-paste but with the app config under Paste's [app:main] section
+        if config_file is None and uwsgi_opt.get("ini"):
+            config_file = uwsgi_opt["ini"]
+            parser = nice_config_parser(config_file)
+            if not parser.has_section(config_section) and parser.has_section(app_section):
+                kwds['config_section'] = app_section
+        if config_file is None and for_paste_app:
+            return None
+        if config_file is None and uwsgi_opt.get("ini-paste"):
+            config_file = uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste")
+            kwds['config_section'] = app_section
+        if config_file is None:
+            return None
+        kwds['config_file'] = config_file
+        return kwds
+
+
     @classmethod
     def register_postfork_function(cls, f, *args, **kwargs):
         if uwsgi.mule_id() == 0:
@@ -285,6 +318,10 @@ def register_postfork_function(f, *args, **kwargs):
     application_stack_class().register_postfork_function(f, *args, **kwargs)
 
 
+def get_app_kwds(config_section):
+    return application_stack_class().get_app_kwds(config_section)
+
+
 def _uwsgi_configured_mules():
     mules = uwsgi.opt.get('mule', [])
     return [mules] if isinstance(mules, string_types) or mules is True else mules

lib/galaxy/webapps/galaxy/buildapp.py

@@ -6,12 +6,6 @@ import sys
 import threading
 import atexit
 
-try:
-    import configparser
-except:
-    import ConfigParser as configparser
-
-
 import galaxy.app
 import galaxy.model
 import galaxy.model.mapping
@@ -38,16 +32,13 @@ class GalaxyWebApplication(galaxy.web.framework.webapp.WebApplication):
     pass
 
 
-def app_factory(global_conf, **kwargs):
-    return paste_app_factory(global_conf, **kwargs)
-
-
-def paste_app_factory(global_conf, **kwargs):
+def app_factory(global_conf, load_app_kwds={}, **kwargs):
     """
     Return a wsgi application serving the root object
     """
     kwargs = load_app_properties(
-        kwds=kwargs
+        kwds=kwargs,
+        **load_app_kwds
     )
     # Create the Galaxy application unless passed in
     if 'app' in kwargs:
@@ -161,22 +152,12 @@ def paste_app_factory(global_conf, **kwargs):
     return webapp
 
 
-def uwsgi_app_factory():
-    # TODO: synchronize with galaxy.web.framework.webapp.build_native_uwsgi_app - should
-    # at least be using nice_config_parser for instance.
-    import uwsgi
-    root = os.path.abspath(uwsgi.opt.get('galaxy_root', os.getcwd()))
-    config_file = uwsgi.opt.get('galaxy_config_file', os.path.join(root, 'config', 'galaxy.ini'))
-    global_conf = {
-        '__file__': config_file if os.path.exists(__file__) else None,
-        'here': root}
-    parser = configparser.ConfigParser()
-    parser.read(config_file)
-    try:
-        kwargs = dict(parser.items('app:main'))
-    except configparser.NoSectionError:
-        kwargs = {}
-    return app_factory(global_conf, **kwargs)
+def uwsgi_app():
+    return galaxy.web.framework.webapp.build_native_uwsgi_app(app_factory, "galaxy")
+
+
+# For backwards compatibility
+uwsgi_app_factory = uwsgi_app
 
 
 def postfork_setup():

scripts/galaxy-main

@@ -45,6 +45,8 @@ try:
 except ImportError:
     uwsgi = None
 
+from six import string_types
+
 REQUIRES_DAEMONIZE_MESSAGE = "Attempted to use Galaxy in daemon mode, but daemonize is unavailable."
 
 log = logging.getLogger(__name__)
@@ -99,7 +101,7 @@ def load_galaxy_app(
 
 def app_loop(args, log):
     try:
-        config_builder = GalaxyConfigBuilder(args, ini_path=ini_path_from_uwsgi())
+        config_builder = GalaxyConfigBuilder(args, config_file=config_file_from_uwsgi())
         galaxy_app = load_galaxy_app(
             config_builder,
             config_env=True,
@@ -143,12 +145,24 @@ def find_ini(supplied_ini, galaxy_root):
 
     return guess
 
-def ini_path_from_uwsgi():
+def config_file_from_uwsgi(app_name=None):
     if uwsgi:
+        for opt in ('yaml', 'json'):
+            if opt in uwsgi.opt:
+                return (uwsgi.opt[opt], app_name or 'galaxy')
+        if 'ini' in uwsgi.opt:
+            # FIXME: refactor (to properties) with galaxy.web.framework.webapp:build_native_uwsgi_app
         if 'ini-paste' in uwsgi.opt:
-            return uwsgi.opt['ini-paste']
+            app_name = 'app:%s' % app_name if app_name else 'app:%s' % DEFAULT_INI_APP
+            return (uwsgi.opt['ini-paste'], app_name)
+
+
+
+        # FIXME: still need this in get_app_kwds
         elif 'set' in uwsgi.opt:
-            for k, v in [x.split('=', 1) for x in uwsgi.opt['set']]:
+            set_options = uwsgi.opt['set']
+            set_options = [set_options] if isinstance(set_options, string_types) else set_options
+            for k, v in [x.split('=', 1) for x in set_options]:
                 if k == 'galaxy_config_file':
                     return v
     return None
@@ -159,31 +173,32 @@ class GalaxyConfigBuilder(object):
     """
 
     def __init__(self, args=None, **kwds):
-        ini_path = kwds.get("ini_path", None) or (args and args.ini_path)
-        # If given app_conf_path - use that - else we need to ensure we have an
-        # ini path.
-        if not ini_path:
+        config_file = kwds.get("config_file", None) or (args and args.config_file) or (args and args.ini_path)
+        # If given app_conf_path - use that - else we need to ensure we have a
+        # config file path.
+        if not config_file:
             galaxy_root = kwds.get("galaxy_root", GALAXY_ROOT_DIR)
-            ini_path = find_ini(ini_path, galaxy_root)
-            ini_path = absolute_config_path(ini_path, galaxy_root=galaxy_root)
-        self.ini_path = ini_path
+            config_file = find_config(config_file, galaxy_root)
+            config_file = absolute_config_path(config_file, galaxy_root=galaxy_root)
+        self.config_file = config_file
         self.app_name = kwds.get("app") or (args and args.app) or DEFAULT_INI_APP
         self.log_file = (args and args.log_file)
 
     @classmethod
     def populate_options(cls, arg_parser):
-        arg_parser.add_argument("-c", "--ini-path", default=None, help="Galaxy ini config file (defaults to config/galaxy.ini)")
-        arg_parser.add_argument("--app", default=DEFAULT_INI_APP, help="app section in ini file (defaults to main)")
+        arg_parser.add_argument("-c", "--config-file", default=None, help="Galaxy config file (defaults to config/galaxy.ini)")
+        arg_parser.add_argument("--ini-path", default=None, help="DEPRECATED: use -c/--config-file")
+        arg_parser.add_argument("--app", default=None, help="app section in ini file (defaults to 'galaxy' for YAML/JSON, 'main' (w/ 'app:' prepended) for INI")
         arg_parser.add_argument("-d", "--daemonize", default=False, help="Daemonzie process", action="store_true")
         arg_parser.add_argument("--daemon-log-file", default=None, help="log file for daemon script ")
-        arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in ini_path if set)")
+        arg_parser.add_argument("--log-file", default=None, help="Galaxy log file (overrides log configuration in config_file if set)")
         arg_parser.add_argument("--pid-file", default=DEFAULT_PID, help="pid file (default is %s)" % DEFAULT_PID)
         arg_parser.add_argument("--server-name", default=None, help="set a galaxy server name")
 
     def app_kwds(self):
         config = dict(
-            ini_file=self.ini_path,
-            ini_section="app:%s" % self.app_name,
+            config_file=self.config_file,
+            config_section="app:%s" % self.app_name,
         )
         return config
 
@@ -211,6 +226,8 @@ def main():
     arg_parser = ArgumentParser(description=DESCRIPTION)
     GalaxyConfigBuilder.populate_options(arg_parser)
     args = arg_parser.parse_args()
+    if args.ini_path and not args.config_file:
+        args.config_file = args.ini_path
     if args.log_file:
         os.environ["GALAXY_CONFIG_LOG_DESTINATION"] = os.path.abspath(args.log_file)
     if args.server_name: