opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-01 05:31:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

- Do all logging configuration other than paste fileConfig logging from

Browse Source 
dictConfig. Default log message format varies by stack.
- Refactor a lot of file extension handling in to galaxy.util.path.
- Simplify and improve config file finding.
- Simplify and improve native app creation.
- Integrate uWSGI argument generation with @jmchilton's improvements to
  the startup scripts.
- Fix starting without a config file under uWSGI
This commit is contained in:
Nate Coraor
2017-09-01 00:56:33 -04:00
parent 3f0082515a
commit 09feaf6f4d
22 changed files with 433 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/config.py
+48 -37
View File
@@ -28,7 +28,7 @@ from galaxy.util import listify
from galaxy.util import string_as_bool
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.web.formatting import expand_pretty_datetime_format
from galaxy.web.stack import application_stack_log_filter, get_stack_facts, register_postfork_function
from galaxy.web.stack import get_stack_facts, register_postfork_function
from .version import VERSION_MAJOR
log = logging.getLogger(__name__)
@@ -77,6 +77,40 @@ PATH_LIST_DEFAULTS = dict(
'config/tool_conf.xml.sample,config/shed_tool_conf.xml']
)
LOGGING_CONFIG_DEFAULT = {
'version': 1,
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'loggers': {
'galaxy': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': 0,
'qualname': 'galaxy',
},
},
'filters': {
'stack': {
'()': 'galaxy.web.stack.application_stack_log_filter',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'stack',
'level': 'DEBUG',
'stream': 'ext://sys.stderr',
'filters': ['stack'],
},
},
'formatters': {
'stack': {
'()': 'galaxy.web.stack.application_stack_log_formatter',
},
},
}
def resolve_path(path, root):
"""If 'path' is relative make absolute by prepending 'root'"""
@@ -578,7 +612,7 @@ class Configuration(object):
# This is for testing new library browsing capabilities.
self.new_lib_browse = string_as_bool(kwargs.get('new_lib_browse', False))
# Logging configuration with logging.config.configDict:
self.logging = kwargs.get('logging', {})
self.logging = kwargs.get('logging', None)
# Error logging with sentry
self.sentry_dsn = kwargs.get('sentry_dsn', None)
# Statistics and profiling with statsd
@@ -879,44 +913,21 @@ def configure_logging(config):
else:
paste_configures_logging = False
auto_configure_logging = not paste_configures_logging and string_as_bool(config.get("auto_configure_logging", "True"))
if auto_configure_logging and (not hasattr(config, 'logging') or not config.logging):
format = config.get("log_format", "%(name)s %(levelname)s %(asctime)s %(message)s")
level = logging._levelNames[config.get("log_level", "DEBUG")]
destination = config.get("log_destination", "stdout")
log.info("Logging at '%s' level to '%s'" % (level, destination))
# Set level
root.setLevel(level)
disable_chatty_loggers = string_as_bool(config.get("auto_configure_logging_disable_chatty", "True"))
if disable_chatty_loggers:
# Turn down paste httpserver logging
if level <= logging.DEBUG:
for chatty_logger in ["paste.httpserver.ThreadPool", "routes.middleware"]:
logging.getLogger(chatty_logger).setLevel(logging.WARN)
# Remove old handlers
for h in root.handlers[:]:
root.removeHandler(h)
# Create handler
if destination == "stdout":
handler = logging.StreamHandler(sys.stdout)
else:
handler = logging.FileHandler(destination)
# Create formatter
formatter = logging.Formatter(format)
# Hook everything up
handler.setFormatter(formatter)
root.addHandler(handler)
elif auto_configure_logging and config.logging:
if auto_configure_logging:
logging_conf = config.get('logging', None)
if logging_conf is None:
# if using the default logging config, honor the log_level setting
logging_conf = LOGGING_CONFIG_DEFAULT
if config.get('log_level', 'DEBUG') != 'DEBUG':
logging_conf['handlers']['console']['level'] = config.get('log_level', 'DEBUG')
# configure logging with logging dict in config, template *FileHandler handler filenames with the `filename_template` option
for name, conf in config.logging['handlers'].items():
for name, conf in logging_conf.get('handlers', {}).items():
if conf['class'].startswith('logging.') and conf['class'].endswith('FileHandler') and 'filename_template' in conf:
# FIXME: this seems to be broken because python is claiming double-star doesn't work on Facts, but
# Facts is a MutableMapping, which double-star works on, soo...
conf['filename'] = conf.pop('filename_template').format(**get_stack_facts(config=config))
config.logging['handlers'][name] = conf
logging.config.dictConfig(config.logging)
for h in root.handlers:
h.addFilter(application_stack_log_filter()())
# If sentry is configured, also log to it
logging_conf['handlers'][name] = conf
logging.config.dictConfig(logging_conf)
if getattr(config, "sentry_dsn", None):
from raven.handlers.logging import SentryHandler
sentry_handler = SentryHandler(config.sentry_dsn)
lib/galaxy/model/orm/scripts.py
+11 -14
View File
@@ -4,12 +4,13 @@ Code to support database helper scripts (create_db.py, manage_db.py, etc...).
import logging
from galaxy.util import listify
from galaxy.util.path import get_ext
from galaxy.util.properties import find_config_file, load_app_properties
log = logging.getLogger(__name__)
DEFAULT_CONFIG_FILE = 'config/galaxy.ini'
DEFAULT_CONFIG_NAMES = ['galaxy', 'universe_wsgi']
DEFAULT_CONFIG_PREFIX = ''
DEFAULT_DATABASE = 'galaxy'
@@ -17,15 +18,13 @@ DATABASE = {
"galaxy":
{
'repo': 'lib/galaxy/model/migrate',
'old_config_files': ['universe_wsgi.ini'],
'default_sqlite_file': './database/universe.sqlite',
'config_override': 'GALAXY_CONFIG_',
},
"tool_shed":
{
'repo': 'lib/galaxy/webapps/tool_shed/model/migrate',
'config_file': 'config/tool_shed.yml',
'old_config_files': ['config/tool_shed.ini', 'tool_shed_wsgi.ini'],
'config_names': ['tool_shed', 'tool_shed_wsgi'],
'default_sqlite_file': './database/community.sqlite',
'config_override': 'TOOL_SHED_CONFIG_',
'config_section': 'tool_shed',
@@ -33,7 +32,6 @@ DATABASE = {
"install":
{
'repo': 'lib/galaxy/model/tool_shed_install/migrate',
'old_config_files': ['universe_wsgi.ini'],
'config_prefix': 'install_',
'default_sqlite_file': './database/install.sqlite',
'config_override': 'GALAXY_INSTALL_CONFIG_',
@@ -41,14 +39,14 @@ DATABASE = {
}
def read_config_file_arg(argv, default, old_defaults, cwd=None):
config_file = None
def read_config_file_arg(argv, config_names, cwd=None):
if '-c' in argv:
pos = argv.index('-c')
argv.pop(pos)
config_file = argv.pop(pos)
old_defaults = listify(old_defaults)
return find_config_file(default, old_defaults, config_file, cwd=cwd)
return argv.pop(pos)
if cwd:
cwd = [cwd]
return find_config_file(config_names, dirs=cwd)
def get_config(argv, cwd=None):
@@ -84,14 +82,13 @@ def get_config(argv, cwd=None):
database = 'galaxy'
database_defaults = DATABASE[database]
default = database_defaults.get('config_file', DEFAULT_CONFIG_FILE)
old_defaults = database_defaults.get('old_config_files')
config_file = read_config_file_arg(argv, default, old_defaults, cwd=cwd)
config_names = database_defaults.get('config_names', DEFAULT_CONFIG_NAMES)
config_file = read_config_file_arg(argv, config_names, cwd=cwd)
repo = database_defaults['repo']
config_prefix = database_defaults.get('config_prefix', DEFAULT_CONFIG_PREFIX)
config_override = database_defaults.get('config_override', 'GALAXY_CONFIG_')
default_sqlite_file = database_defaults['default_sqlite_file']
if config_file.endswith(".yml") or config_file.endswith(".yml.sample"):
if not config_file or get_ext(config_file, ignore='sample') == 'yaml':
config_section = database_defaults.get('config_section', None)
else:
# An .ini file - just let load_app_properties find app:main.
lib/galaxy/objectstore/__init__.py
+1 -1
View File
@@ -23,10 +23,10 @@ from galaxy.util import (
directory_hash_id,
force_symlink,
safe_makedirs,
safe_relpath,
umask_fix_perms,
)
from galaxy.util.odict import odict
from galaxy.util.path import safe_relpath
from galaxy.util.sleeper import Sleeper
NO_SESSION_ERROR_MESSAGE = "Attempted to 'create' object store entity in configuration with no database session present."
lib/galaxy/objectstore/azure_blob.py
+2 -1
View File
@@ -11,7 +11,8 @@ import time
from datetime import datetime
from galaxy.exceptions import ObjectInvalid, ObjectNotFound
from galaxy.util import directory_hash_id, safe_relpath, umask_fix_perms
from galaxy.util import directory_hash_id, umask_fix_perms
from galaxy.util.path import safe_relpath
from galaxy.util.sleeper import Sleeper
from ..objectstore import convert_bytes, ObjectStore
lib/galaxy/objectstore/pithos.py
+1 -1
View File
@@ -8,9 +8,9 @@ import shutil
from galaxy.exceptions import ObjectInvalid, ObjectNotFound
from galaxy.util import (
directory_hash_id,
safe_relpath,
umask_fix_perms,
)
from galaxy.util.path import safe_relpath
from ..objectstore import ObjectStore
try:
lib/galaxy/objectstore/rods.py
+1 -1
View File
@@ -13,7 +13,7 @@ from posixpath import dirname as path_dirname
from posixpath import join as path_join
from galaxy.exceptions import ObjectInvalid, ObjectNotFound
from galaxy.util import safe_relpath
from galaxy.util.path import safe_relpath
from ..objectstore import DiskObjectStore, local_extra_dirs
lib/galaxy/objectstore/s3.py
+1 -1
View File
@@ -15,10 +15,10 @@ from datetime import datetime
from galaxy.exceptions import ObjectInvalid, ObjectNotFound
from galaxy.util import (
directory_hash_id,
safe_relpath,
string_as_bool,
umask_fix_perms,
)
from galaxy.util.path import safe_relpath
from galaxy.util.sleeper import Sleeper
from .s3_multipart_upload import multipart_upload
lib/galaxy/script/__init__.py
+7 -2
View File
@@ -33,8 +33,9 @@ def main(argv=None):
def _app_properties(args):
config_file = find_config_file("config/galaxy.ini", "universe_wsgi.ini", args.config_file)
app_properties = load_app_properties(ini_file=config_file)
# FIXME: you can use galaxy.util.path.extensions for this
config_file = args.config_file or find_config_file(args.app)
app_properties = load_app_properties(config_file=config_file, config_section=args.config_section)
return app_properties
@@ -47,6 +48,10 @@ def _arg_parser():
help='action to perform')
parser.add_argument("-c", "--config-file",
default=os.environ.get('GALAXY_CONFIG_FILE', None))
parser.add_argument("--config-section",
default=os.environ.get('GALAXY_CONFIG_SECTION', None))
parser.add_argument("--app",
default=os.environ.get('GALAXY_APP', 'galaxy'))
for argument in ARGUMENTS:
parser.add_argument(*argument[0], **argument[1])
return parser
lib/galaxy/util/__init__.py
+9 -21
View File
@@ -846,17 +846,21 @@ def string_as_bool_or_none(string):
return False
def listify(item, do_strip=False):
def listify(item, do_strip=None, strip=False, split=True):
"""
Make a single item a single item list, or return a list if passed a
list. Passing a None returns an empty list.
list. Splits strings on commas unless split=False, strips whitespace
from around comma-split elements if strip=True. Passing a None returns
an empty list. Passing a tuple returns the tuple.
"""
if do_strip is not None and strip is False:
strip = do_strip
if not item:
return []
elif isinstance(item, list):
elif isinstance(item, list) or isinstance(item, tuple):
return item
elif isinstance(item, string_types) and item.count(','):
if do_strip:
elif split and isinstance(item, string_types) and item.count(','):
if strip:
return [token.strip() for token in item.split(',')]
else:
return item.split(',')
@@ -1502,22 +1506,6 @@ def download_to_file(url, dest_file_path, timeout=30, chunk_size=2 ** 20):
f.write(chunk)
def safe_relpath(path):
"""
Given what we expect to be a relative path, determine whether the path
would exist inside the current directory.
:type path: string
:param path: a path to check
:rtype: bool
:returns: ``True`` if path is relative and does not reference a path
in a parent directory, ``False`` otherwise.
"""
if path.startswith(os.sep) or normpath(path).startswith(os.pardir):
return False
return True
class ExecutionTimer(object):
def __init__(self):
lib/galaxy/util/path.py
+128
View File
@@ -0,0 +1,128 @@
"""Path manipulation functions
"""
from __future__ import absolute_import
from os import extsep, pardir, sep
from os.path import normpath
from six import iteritems, string_types
from ..util import listify
def safe_relpath(path):
"""
Given what we expect to be a relative path, determine whether the path would exist inside the current directory.
:type path: string
:param path: a path to check
:rtype: bool
:returns: ``True`` if path is relative and does not reference a path in a parent directory, ``False`` otherwise.
"""
return not (path.startswith(sep) or normpath(path).startswith(pardir))
def joinext(root, ext):
"""
Roughly the reverse of os.path.splitext.
:type root: string
:param root: part of the filename before the extension
:type root: string
:param ext: the extension
:rtype: string
:returns: ``root`` joined with ``ext`` separated by a single ``os.extsep``
"""
return extsep.join([root.rstrip(extsep), ext.lstrip(extsep)])
def has_ext(path, ext, aliases=False, ignore=None):
"""
Determine whether ``path`` has extension ``ext``
:type path: string
:param path: Path to check
:type ext: string
:param ext: Extension to check
:type aliases: bool
:param aliases: Check any known aliases for the given extension
:type ignore: string
:param ignore: Ignore this extension at the end of the path (e.g. ``sample``)
:rtype: bool
:returns: ``True`` if path is a YAML file, ``False`` otherwise.
"""
ext = __ext_strip_sep(ext)
root, _ext = __splitext_ignore(path, ignore=ignore)
if aliases:
return _ext in extensions[ext]
else:
return _ext == ext
def get_ext(path, ignore=None, canonicalize=True):
"""
Return the extension of ``path``
:type path: string
:param path: Path to check
:type ignore: string
:param ignore: Ignore this extension at the end of the path (e.g. ``sample``)
:type canonicalize: bool
:param canonicalize: If the extension is known to this module, return the canonicalized extension instead of the
file's actual extension
:rtype: string
"""
root, ext = __splitext_ignore(path, ignore=ignore)
if canonicalize:
try:
ext = extensions.canonicalize(ext)
except KeyError:
pass # should do something else here?
return ext
class Extensions(dict):
"""Mappings for extension aliases.
A dict-like object that returns values for keys that are not mapped if the key can be found in any of the dict's
values (which should be sequence types).
The first item in the sequence should match the key and is the "canonicalization".
"""
def __missing__(self, key):
for k, v in iteritems(self):
if key in v:
self[key] = v
return v
raise KeyError(key)
def canonicalize(self, ext):
# shouldn't raise an IndexError because it should raise a KeyError first
return self[ext][0]
extensions = Extensions({
'ini': ['ini'],
'json': ['json'],
'yaml': ['yaml', 'yml'],
})
def __ext_strip_sep(ext):
return ext.lstrip(extsep)
def __splitext_no_sep(path):
return (path.rsplit(extsep, 1) + [''])[0:2]
def __splitext_ignore(path, ignore=None):
# note: unlike os.path.splitext this strips extsep from ext
ignore = map(__ext_strip_sep, listify(ignore, split=False))
root, ext = __splitext_no_sep(path)
if ext in ignore:
root, ext = __splitext_no_sep(path)
return (root, ext)
__all__ = ('extensions', 'get_ext', 'has_ext', 'joinext', 'safe_relpath')
lib/galaxy/util/properties.py
+51 -36
View File
@@ -5,42 +5,30 @@ this should be reusable by tool shed and pulsar as well.
import os
import os.path
import sys
from functools import partial
from itertools import product, starmap
import yaml
from six import iteritems
from six import iteritems, string_types
from six.moves.configparser import ConfigParser
from galaxy.util import listify
from galaxy.util.path import has_ext, extensions, joinext
def find_config_file(default, old_defaults, explicit, cwd=None):
old_defaults = listify(old_defaults)
if cwd is not None:
default = os.path.join(cwd, default)
for i in range(len(old_defaults)):
old_defaults[i] = os.path.join(cwd, old_defaults[i])
if explicit is not None:
explicit = os.path.join(cwd, explicit)
if explicit:
if os.path.exists(explicit):
config_file = explicit
else:
raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.")
else:
config_file = None
if os.path.exists(default):
config_file = default
if config_file is None:
for old_default in old_defaults:
if os.path.exists(old_default):
config_file = old_default
if config_file is None:
config_file = default + ".sample"
return config_file
def find_config_file(names, exts=None, dirs=None, include_samples=False):
found = __find_config_files(
names,
exts=exts or extensions['yaml'] + extensions['ini'],
dirs=dirs or [os.getcwd(), os.path.join(os.getcwd(), 'config')],
include_samples=include_samples,
)
if not found:
return None
# doesn't really make sense to log here but we should probably generate a warning of some kind if more than one
# config is found.
return found[0]
def load_app_properties(
@@ -57,18 +45,22 @@ def load_app_properties(
config_section = ini_section
if config_file:
if not config_file.endswith(".yml") and not config_file.endswith(".yml.sample"):
if not has_ext(config_file, 'yaml', aliases=True, ignore='sample'):
if config_section is None:
config_section = "app:main"
parser = nice_config_parser(config_file)
properties.update(dict(parser.items(config_section)))
if parser.has_section(config_section):
properties.update(dict(parser.items(config_section)))
else:
properties.update(parser.defaults())
else:
if config_section is None:
config_section = "galaxy"
with open(config_file, "r") as f:
raw_properties = yaml.load(f)
properties = raw_properties[config_section] or {}
properties = __default_properties(config_file)
properties.update(raw_properties.get(config_section) or {})
override_prefix = "%sOVERRIDE_" % config_prefix
for key in os.environ:
@@ -84,11 +76,7 @@ def load_app_properties(
def nice_config_parser(path):
defaults = {
'here': os.path.dirname(os.path.abspath(path)),
'__file__': os.path.abspath(path)
}
parser = NicerConfigParser(path, defaults=defaults)
parser = NicerConfigParser(path, defaults=__default_properties(path))
parser.optionxform = str # Don't lower-case keys
with open(path) as f:
parser.read_file(f)
@@ -150,4 +138,31 @@ class NicerConfigParser(ConfigParser):
raise
def __get_all_configs(dirs, names):
return filter(os.path.exists, starmap(os.path.join, product(dirs, names)))
def __find_config_files(names, exts=None, dirs=None, include_samples=False):
sample_names = []
if isinstance(names, string_types):
names = [names]
if not dirs:
dirs = [os.getcwd()]
if exts:
# add exts to names
names = starmap(joinext, product(names, exts))
if include_samples:
sample_names = map(partial(joinext, ext='sample'), names)
# check for all names in each dir before moving to the next dir. could do it the other way around but that makes
# less sense to me.
return __get_all_configs(dirs, names) or __get_all_configs(dirs, sample_names)
def __default_properties(path):
return {
'here': os.path.dirname(os.path.abspath(path)),
'__file__': os.path.abspath(path)
}
__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser')
lib/galaxy/web/framework/webapp.py
+3 -24
View File
@@ -962,30 +962,9 @@ def build_native_uwsgi_app(paste_factory, config_section):
"""uwsgi can load paste factories with --ini-paste, but this builds non-paste uwsgi apps.
In particular these are useful with --yaml or --json for config."""
'''
import uwsgi
uwsgi_opt = uwsgi.opt
config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
# legacy, support loading ini uWSGI config without --ini-paste but with the app config under Paste's [app:main] section
if config_file is None and uwsgi_opt.get("ini"):
config_file = uwsgi_opt.get("ini")
parser = nice_config_parser(config_file)
if not parser.has_section(config_section) and parser.has_section("app:main"):
config_section = "app:main"
# support no uWSGI config file or separate app config file, requires setting galaxy_config_file in the uWSGI config
config_file = uwsgi_opt.get("galaxy_config_file") or config_file
if not config_file:
# Probably loaded via --ini-paste - expect paste app.
return None
'''
app_kwds = get_app_kwds(config_section, for_paste_app=True)
if not app_kwds:
# Probably loaded via --ini-paste - expect paste app
return None
import uwsgi
uwsgi_app = paste_factory(uwsgi.opt, load_app_kwds=app_kwds)
# TODO: just move this to a classmethod on stack?
app_kwds = get_app_kwds(config_section)
uwsgi_app = paste_factory({}, load_app_kwds=app_kwds)
return uwsgi_app
lib/galaxy/web/stack/__init__.py
+24 -22
View File
@@ -47,6 +47,7 @@ class ApplicationStack(object):
prohibited_middleware = frozenset()
transport_class = ApplicationStackTransport
log_filter_class = ApplicationStackLogFilter
log_format = '%(name)s %(levelname)s %(asctime)s %(message)s'
# TODO: this belongs in the pool configuration
server_name_template = '{server_name}'
default_app_name = 'main'
@@ -57,6 +58,10 @@ class ApplicationStack(object):
JOB_HANDLERS='job-handlers',
)
@classmethod
def log_filter(cls):
return cls.log_filter_class()
@classmethod
def get_app_kwds(cls, config_section, app_name=None, for_paste_app=False):
return {}
@@ -150,6 +155,8 @@ class MessageApplicationStack(ApplicationStack):
class UWSGIApplicationStack(MessageApplicationStack):
"""Interface to the uWSGI application stack. Supports running additional webless Galaxy workers as mules. Mules
must be farmed to be communicable via uWSGI mule messaging, unfarmed mules are not supported.
Note that mules will use this as their stack class even though they start with the "webless" loading point.
"""
name = 'uWSGI'
prohibited_middleware = frozenset([
@@ -158,42 +165,33 @@ class UWSGIApplicationStack(MessageApplicationStack):
])
transport_class = UWSGIFarmMessageTransport
log_filter_class = UWSGILogFilter
log_format = '%(name)s %(levelname)s %(asctime)s [p:%(process)s,w:%(worker_id)s,m:%(mule_id)s] [%(threadName)s] %(message)s'
server_name_template = '{server_name}.{server_id}'
postfork_functions = []
@classmethod
def get_app_kwds(cls, config_section, app_name=None, for_paste_app=False):
def get_app_kwds(cls, config_section, app_name=None):
kwds = {
'config_file': None,
'config_section': config_section,
}
# used by webless mules started under uWSGI
uwsgi_opt = uwsgi.opt
app_section = 'app:%s' % app_name if app_name else 'app:%s' % cls.default_app_name
# check for --yaml or --json uWSGI config options first
config_file = uwsgi_opt.get("yaml") or uwsgi_opt.get("json")
# legacy, support loading ini uWSGI config without --ini-paste but with the app config under a Paste [app:] section
if config_file is None and uwsgi_opt.get("ini"):
config_file = uwsgi_opt["ini"]
# --ini and --ini-paste don't behave the same way, but this method will only be called by mules if the main
# application was loaded with --ini-paste, so we can make some assumptions, most notably, uWSGI does not have
# any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding the alternate section
# name to `app:main` is fine.
if config_file is None and uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste"):
config_file = uwsgi_opt.get("ini") or uwsgi_opt.get("ini-paste")
parser = nice_config_parser(config_file)
if not parser.has_section(config_section) and parser.has_section(app_section):
kwds['config_section'] = app_section
# if we're getting kwargs for loading by paste, pastedeploy will set them up itself
if config_file is None and uwsgi_opt.get("ini-paste") and for_paste_app:
return None
if not parser.has_section(config_section) and parser.has_section('app:main'):
kwds['config_section'] = 'app:main'
# check for --set galaxy_config_file=<path>, this overrides whatever config file uWSGI was loaded with (which
# may not actually include a Galaxy config)
if uwsgi_opt.get("galaxy_config_file"):
config_file = uwsgi_opt.get("galaxy_config_file")
# otherwise, check --ini-paste
if config_file is None and uwsgi_opt.get("ini-paste"):
config_file = uwsgi_opt["ini-paste"]
parser = nice_config_parser(config_file)
if not parser.has_section(config_section) and parser.has_section('app:main'):
kwds['config_section'] = 'app:main'
if config_file is None:
return None
kwds['config_file'] = config_file
return kwds
@@ -319,15 +317,19 @@ def application_stack_instance(app=None, config=None):
def application_stack_log_filter():
return application_stack_class().log_filter_class
return application_stack_class().log_filter_class()
def application_stack_log_formatter():
return logging.Formatter(fmt=application_stack_class().log_format)
def register_postfork_function(f, *args, **kwargs):
application_stack_class().register_postfork_function(f, *args, **kwargs)
def get_app_kwds(config_section, app_name=None, for_paste_app=None):
return application_stack_class().get_app_kwds(config_section, app_name=app_name, for_paste_app=for_paste_app)
def get_app_kwds(config_section, app_name=None):
return application_stack_class().get_app_kwds(config_section, app_name=app_name)
def get_stack_facts(config=None):
lib/tool_shed/capsule/capsule_manager.py
+2 -1
View File
@@ -12,7 +12,8 @@ from sqlalchemy import and_, false
import tool_shed.repository_types.util as rt_util
from galaxy import web
from galaxy.util import asbool, build_url, CHUNK_SIZE, safe_relpath
from galaxy.util import asbool, build_url, CHUNK_SIZE
from galaxy.util.path import safe_relpath
from galaxy.util.odict import odict
from tool_shed.dependencies import attribute_handlers
from tool_shed.dependencies.repository.relation_builder import RelationBuilder
lib/tool_shed/util/commit_util.py
+2 -1
View File
@@ -10,7 +10,8 @@ from collections import namedtuple
from sqlalchemy.sql.expression import null
import tool_shed.repository_types.util as rt_util
from galaxy.util import checkers, safe_relpath
from galaxy.util import checkers
from galaxy.util.path import safe_relpath
from tool_shed.tools import data_table_manager
from tool_shed.util import basic_util, hg_util, shed_util_common as suc
run.sh
+28 -12
View File
@@ -1,5 +1,12 @@
#!/bin/sh
# Usage: ./run.sh <start|stop|restart>
#
#
# Description: This script can be used to start or stop the galaxy
# web application.
cd "$(dirname "$0")"
. ./scripts/common_startup_functions.sh
@@ -16,7 +23,10 @@ then
. $GALAXY_LOCAL_ENV_FILE
fi
./scripts/common_startup.sh $common_startup_args || exit 1
GALAXY_PID=${GALAXY_PID:-galaxy.pid}
GALAXY_LOG=${GALAXY_LOG:-galaxy.log}
PID_FILE=$GALAXY_PID
LOG_FILE=$GALAXY_LOG
parse_common_args $@
@@ -41,22 +51,27 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
elif [ -f config/galaxy.ini ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
else
elif [ -f config/galaxy.yml ]; then
GALAXY_CONFIG_FILE=config/galaxy.yml
elif [ -f config/galaxy.ini.sample -a -z "$GALAXY_UWSGI" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini.sample
fi
export GALAXY_CONFIG_FILE
fi
if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
# Install Conda environment if needed.
python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed
if [ -n "$GALAXY_CONFIG_FILE" ]; then
config_file_arg="-c $GALAXY_CONFIG_FILE"
fi
if [ -n "$GALAXY_UWSGI" ]; then
uwsgi_args="$(python ./scripts/get_uwsgi_args.py)"
echo "executing: uwsgi $uwsgi_args"
uwsgi $uwsgi_args
elif [ -n "$GALAXY_RUN_ALL" ]; then
if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
# Install Conda environment if needed.
python ./scripts/manage_tool_dependencies.py $config_file_arg init_if_needed
fi
[ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi'
find_server ${GALAXY_CONFIG_FILE:-none} galaxy
if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then
servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo)
if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then
echo "ERROR: \$GALAXY_RUN_ALL cannot be used without the '--daemon', '--stop-daemon' or 'restart' arguments to run.sh"
@@ -86,6 +101,7 @@ elif [ -n "$GALAXY_RUN_ALL" ]; then
fi
done
else
# Handle only 1 server, whose name can be specified with --server-name parameter (defaults to "main")
python ./scripts/paster.py serve "$GALAXY_CONFIG_FILE" $paster_args
echo "executing: $run_server $server_args"
# args are properly quoted so use eval
eval $run_server $server_args
fi
run_reports.sh
+3 -4
View File
@@ -39,8 +39,6 @@ if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
GALAXY_REPORTS_CONFIG=config/reports.ini
elif [ -f config/reports.yml ]; then
GALAXY_REPORTS_CONFIG=config/reports.yml
else
GALAXY_REPORTS_CONFIG=config/reports.yml.sample
fi
export GALAXY_REPORTS_CONFIG
fi
@@ -49,5 +47,6 @@ if [ -n "$GALAXY_REPORTS_CONFIG_DIR" ]; then
python ./scripts/build_universe_config.py "$GALAXY_REPORTS_CONFIG_DIR" "$GALAXY_REPORTS_CONFIG"
fi
find_server $GALAXY_REPORTS_CONFIG
$run_server $server_args
find_server ${GALAXY_REPORTS_CONFIG:-none} reports
echo "executing: $run_server $server_args"
eval $run_server $server_args
run_tool_shed.sh
+3 -4
View File
@@ -32,11 +32,10 @@ if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.ini
elif [ -f config/tool_shed.yml ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml
else
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml.sample
fi
export TOOL_SHED_CONFIG_FILE
fi
find_server $TOOL_SHED_CONFIG_FILE
$run_server $server_args
find_server ${TOOL_SHED_CONFIG_FILE:-none} tool_shed
echo "executing: $run_server $server_args"
eval $run_server $server_args
scripts/common_startup_functions.sh
+12 -24
View File
@@ -1,7 +1,5 @@
#!/bin/sh
uwsgi_args="--master --pythonpath=lib"
parse_common_args() {
INITIALIZE_TOOL_DEPENDENCIES=1
# Pop args meant for common_startup.sh
@@ -21,9 +19,9 @@ parse_common_args() {
common_startup_args="$common_startup_args $1"
shift
;;
--stop-daemon)
common_startup_args="$common_startup_args $1"
paster_args="$paster_args $1"
--stop-daemon|stop)
common_startup_args="$common_startup_args --stop-daemon"
paster_args="$paster_args --pid-file $PID_FILE --stop-daemon"
uwsgi_args="$uwsgi_args --stop $PID_FILE"
stop_daemon_arg_set=1
shift
@@ -39,8 +37,8 @@ parse_common_args() {
daemon_or_restart_arg_set=1
shift
;;
--daemon)
paster_args="$paster_args $1"
--daemon|start)
paster_args="$paster_args --pid-file $PID_FILE --log-file $LOG_FILE --daemon"
# --daemonize2 waits until after the application has loaded
# to daemonize, thus it stops if any errors are found
uwsgi_args="$uwsgi_args --daemonize2 $LOG_FILE --safe-pidfile $PID_FILE"
@@ -85,25 +83,13 @@ setup_python() {
python ./scripts/check_python.py || exit 1
}
find_uwsgi() {
# Look for uwsgi
if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then
UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi
elif command -v uwsgi >/dev/null 2>&1; then
UWSGI=uwsgi
else
echo 'ERROR: Could not find uwsgi executable'
exit 1
fi
}
find_server() {
server_config="$1"
server_config_style="ini-paste"
server_app="$2"
arg_getter_args=
default_webserver="paste"
case "$server_config" in
*.y*ml*)
server_config_style="yaml"
*.y*ml|''|none)
default_webserver="uwsgi" # paste incapable of this
;;
esac
@@ -120,10 +106,12 @@ find_server() {
echo 'ERROR: Could not find uwsgi executable'
exit 1
fi
[ "$server_config" != "none" ] && arg_getter_args="-c $server_config"
[ -n "$server_app" ] && arg_getter_args="--app $server_app"
run_server="$UWSGI"
server_args="--$server_config_style $server_config $uwsgi_args"
server_args="$(python ./scripts/get_uwsgi_args.py $arg_getter_args) $uwsgi_args"
else
run_server="python"
server_args="./scripts/paster.py serve $server_config $paster_args --pid-file $PID_FILE --log-file $LOG_FILE $paster_args"
server_args="./scripts/paster.py serve $server_config $paster_args"
fi
}
scripts/galaxy-main
-3
View File
@@ -189,7 +189,6 @@ class GalaxyConfigBuilder(object):
# Galaxy will attempt to setup logging if loggers is not present in
# ini config file - this handles that loggers block however if present
# (the way paste normally would)
from galaxy.web.stack import application_stack_log_filter
if not self.config_file:
return
if self.config_is_ini:
@@ -202,8 +201,6 @@ class GalaxyConfigBuilder(object):
dict(__file__=config_file, here=os.path.dirname(config_file))
)
root = logging.getLogger()
for h in root.handlers:
h.addFilter(application_stack_log_filter()())
def main():
scripts/get_uwsgi_args.py
+94 -32
View File
@@ -3,49 +3,111 @@ from __future__ import print_function
import os
import sys
from six import string_types
from six.moves import shlex_quote
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
from galaxy.script import main_factory
from galaxy.util.properties import nice_config_parser
from galaxy.util.path import get_ext
from galaxy.util.properties import load_app_properties, nice_config_parser
from galaxy.web.stack import get_app_kwds
DESCRIPTION = "Script to determine uWSGI command line arguments."
COMMAND_TEMPLATE = '{virtualenv} --ini-paste {config_file}{processes}{threads}{http}{pythonpath}{master}{static-map}{paste-logger}{die-on-term}{enable-threads}{py-call-osafterfork}{mule}{farm}'
DESCRIPTION = "Script to determine uWSGI command line arguments"
# socket is not an alias for http, but it is assumed that if you configure a socket in your uwsgi config you do not
# want to run the default http server (or you can configure it yourself)
ALIASES = {
'virtualenv': ('home', 'venv', 'pyhome'),
'pythonpath': ('python-path', 'pp'),
'http': ('httprouter', 'socket', 'uwsgi-socket', 'suwsgi-socket', 'ssl-socket'),
}
DEFAULT_ARGS = {
'_all_': ('virtualenv', 'pythonpath', 'master', 'threads', 'http', 'static-map', 'die-on-term', 'enable-threads'),
'galaxy': ('py-call-osafterfork', 'mule', 'farm'),
'reports': (),
'tool_shed': (),
}
DEFAULT_PORTS = {
'galaxy': 8080,
'reports': 9001,
'tool_shed': 9009,
}
def _get_uwsgi_args(args, kwargs):
config_file = args.config_file or kwargs['__file__']
def __arg_set(arg, kwargs):
if arg in kwargs:
return True
for alias in ALIASES.get(arg, ()):
if alias in kwargs:
return True
return False
def __add_arg(args, arg, value):
optarg = '--%s' % arg
if isinstance(value, bool):
if value is True:
args.append(optarg)
elif isinstance(value, string_types):
# the = in --optarg=value is usually, but not always, optional
if value.startswith('='):
args.append(shlex_quote(optarg + value))
else:
args.append(optarg)
args.append(shlex_quote(value))
else:
[__add_arg(args, arg, v) for v in value]
def __add_config_file_arg(args, config_file, app):
ext = None
if config_file:
ext = get_ext(config_file)
if ext in ('yaml', 'json'):
__add_arg(args, ext, config_file)
elif ext == 'ini':
config = nice_config_parser(config_file)
has_logging = config.has_section('loggers')
if config.has_section('app:main'):
# uWSGI does not have any way to set the app name when loading with paste.deploy:loadapp(), so hardcoding
# the name to `main` is fine
__add_arg(args, 'ini-paste' if not has_logging else 'ini-paste-logged', config_file)
return # do not add --module
else:
__add_arg(args, ext, config_file)
if has_logging:
__add_arg(args, 'paste-logger', True)
__add_arg(args, 'module', 'galaxy.webapps.{app}.buildapp:uwsgi_app()'.format(app=app))
def _get_uwsgi_args(cliargs, kwargs):
# it'd be nice if we didn't have to reparse here but we need things out of more than one section
config_file = cliargs.config_file or kwargs.get('__file__')
uwsgi_kwargs = load_app_properties(config_file=config_file, config_section='uwsgi')
handlerct = int(kwargs.get('job_handler_count', 1))
config = nice_config_parser(config_file)
config_defaults = {
'virtualenv': ' --virtualenv {venv}'.format(venv=os.environ.get('VIRTUAL_ENV', './.venv')),
'processes': ' --processes 1',
'threads': ' --threads 4',
'http': ' --http localhost:8080',
'pythonpath': ' --pythonpath lib',
'master': ' --master',
'static-map': (' --static-map /static/style={here}/static/style/blue'
' --static-map /static={here}/static'.format(here=os.getcwd())),
'paste-logger': ' --paste-logger' if config.has_section('formatters') else '',
'die-on-term': ' --die-on-term',
'enable-threads': ' --enable-threads',
'py-call-osafterfork': ' --py-call-osafterfork',
'mule': ' --mule=lib/galaxy/main.py' * handlerct,
'farm': ' --farm={name}:{mules}'.format(
args = []
__add_config_file_arg(args, config_file, cliargs.app)
defaults = {
'virtualenv': os.environ.get('VIRTUAL_ENV', './.venv'),
'pythonpath': 'lib',
'master': True,
'threads': '4',
'http': 'localhost:{port}'.format(port=DEFAULT_PORTS[cliargs.app]),
'static-map': ('/static/style={here}/static/style/blue'.format(here=os.getcwd()),
'/static={here}/static'.format(here=os.getcwd())),
'die-on-term': True,
'enable-threads': True,
'py-call-osafterfork': True,
'mule': ('=lib/galaxy/main.py',) * handlerct,
'farm': '={name}:{mules}'.format(
name=kwargs.get('job_handler_pool_name', 'job-handlers'),
mules=','.join([str(x) for x in range(1, handlerct + 1)])) if handlerct > 0 else '',
}
if not config.has_section('uwsgi'):
format_dict = config_defaults
else:
format_dict = {}
for opt in config_defaults.keys():
if config.has_option('uwsgi', opt):
format_dict[opt] = ''
else:
format_dict[opt] = config_defaults[opt]
format_dict['config_file'] = config_file
print(COMMAND_TEMPLATE.format(**format_dict))
for arg in DEFAULT_ARGS['_all_'] + DEFAULT_ARGS[cliargs.app]:
if not __arg_set(arg, uwsgi_kwargs):
__add_arg(args, arg, defaults[arg])
print(' '.join(args))
ACTIONS = {
test/base/driver_util.py
+2 -31
View File
@@ -25,6 +25,7 @@ from six.moves import shlex_quote
from functional import database_contexts
from galaxy.app import UniverseApplication as GalaxyUniverseApplication
from galaxy.config import LOGGING_CONFIG_DEFAULT
from galaxy.util import asbool, download_to_file
from galaxy.util.properties import load_app_properties
from galaxy.web import buildapp
@@ -51,34 +52,6 @@ MIGRATED_TOOL_PANEL_CONFIG = 'config/migrated_tools_conf.xml'
INSTALLED_TOOL_PANEL_CONFIGS = [
os.environ.get('GALAXY_TEST_SHED_TOOL_CONF', 'config/shed_tool_conf.xml')
]
LOGGING_CONFIG = {
'version': 1,
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'loggers': {
'galaxy': {
'handlers': ['console'],
'level': 'DEBUG',
'propagate': 0,
'qualname': 'galaxy',
},
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'generic',
'level': 'DEBUG',
'stream': 'ext://sys.stderr',
},
},
'formatters': {
'generic': {
'format': '%(name)s %(levelname)-5.5s %(asctime)s [p:%(process)s] [%(threadName)s] %(message)s'
},
},
}
DEFAULT_LOCALES = "en"
@@ -234,6 +207,7 @@ def setup_galaxy_config(
use_heartbeat=False,
user_library_import_dir=user_library_import_dir,
webhooks_dir=TEST_WEBHOOKS_DIR,
logging=LOGGING_CONFIG_DEFAULT,
)
config.update(database_conf(tmpdir))
config.update(install_database_conf(tmpdir, default_merged=default_install_db_merged))
@@ -247,9 +221,6 @@ def setup_galaxy_config(
# Used by shed's twill dependency stuff - todo read from
# Galaxy's config API.
os.environ["GALAXY_TEST_TOOL_DEPENDENCY_DIR"] = tool_dependency_dir
if log_format:
config['logging'] = LOGGING_CONFIG.copy()
config['logging']['formatters']['generic']['format'] = log_format
return config