diff --git a/lib/galaxy/config_watchers.py b/lib/galaxy/config_watchers.py index 23db380d6a2..2e16735ee7f 100644 --- a/lib/galaxy/config_watchers.py +++ b/lib/galaxy/config_watchers.py @@ -1,3 +1,4 @@ +import logging from os.path import dirname from galaxy.queue_worker import job_rule_modules @@ -7,6 +8,8 @@ from galaxy.tools.toolbox.watcher import ( ) from galaxy.util.watcher import get_watcher +log = logging.getLogger(__name__) + class ConfigWatchers(object): """Contains ToolConfWatcher, ToolWatcher and ToolDataWatcher objects.""" @@ -21,8 +24,20 @@ class ConfigWatchers(object): # If there are multiple ToolConfWatcher objects for the same handler or web process a race condition occurs between the two cache_cleanup functions. # If the reload_data_managers callback wins, the cache will miss the tools that had been removed from the cache # and will be blind to further changes in these tools. + + def reload_toolbox(): + save_integrated_tool_panel = False + try: + # Run and wait for toolbox reload on the process that watches the config files. + # The toolbpox reload will update the integrated_tool_panel_file + self.app.queue_worker.send_local_control_task('reload_toolbox', get_response=True), + except Exception: + save_integrated_tool_panel = True + log.exception("Exception occured while reloading toolbox") + self.app.queue_worker.send_control_task('reload_toolbox', noop_self=True, kwargs={'save_integrated_tool_panel': save_integrated_tool_panel}), + self.tool_config_watcher = get_tool_conf_watcher( - reload_callback=lambda: self.app.queue_worker.send_control_task('reload_toolbox'), + reload_callback=reload_toolbox, tool_cache=self.app.tool_cache, ) self.data_manager_config_watcher = get_tool_conf_watcher( diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 59335726060..87eeada1ecb 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -42,7 +42,7 @@ def send_local_control_task(app, task, get_response=False, kwargs=None): """ if kwargs is None: kwargs = {} - log.info("Queuing async task %s for %s." % (task, app.config.server_name)) + log.info("Queuing %s task %s for %s." % ("sync" if get_response else "async", task, app.config.server_name)) payload = {'task': task, 'kwargs': kwargs} routing_key = 'control.%s@%s' % (app.config.server_name, socket.gethostname()) @@ -114,7 +114,7 @@ class ControlTask(object): callback_queue = [self.callback_queue] self.correlation_id = uuid() try: - with producers[self.connection].acquire(block=True) as producer: + with producers[self.connection].acquire(block=True, timeout=10) as producer: producer.publish( payload, exchange=None if local else self.exchange, @@ -163,19 +163,19 @@ def reload_tool(app, **kwargs): log.error("Reload tool invoked without tool id.") -def reload_toolbox(app, **kwargs): +def reload_toolbox(app, save_integrated_tool_panel=True, **kwargs): reload_timer = util.ExecutionTimer() log.debug("Executing toolbox reload on '%s'", app.config.server_name) reload_count = app.toolbox._reload_count if hasattr(app, 'tool_cache'): app.tool_cache.cleanup() - _get_new_toolbox(app) + _get_new_toolbox(app, save_integrated_tool_panel) app.toolbox._reload_count = reload_count + 1 send_local_control_task(app, 'rebuild_toolbox_search_index') log.debug("Toolbox reload %s", reload_timer) -def _get_new_toolbox(app): +def _get_new_toolbox(app, save_integrated_tool_panel=True): """ Generate a new toolbox, by constructing a toolbox from the config files, and then adding pre-existing data managers from the old toolbox to the new toolbox. @@ -186,7 +186,7 @@ def _get_new_toolbox(app): app.tool_shed_repository_cache.rebuild() tool_configs = app.config.tool_configs - new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app) + new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app, save_integrated_tool_panel=save_integrated_tool_panel) new_toolbox.data_manager_tools = app.toolbox.data_manager_tools app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True) app.datatypes_registry.load_external_metadata_tool(new_toolbox) @@ -352,8 +352,8 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread): def send_control_task(self, task, noop_self=False, get_response=False, routing_key='control.*', kwargs=None): return send_control_task(app=self.app, task=task, noop_self=noop_self, get_response=get_response, routing_key=routing_key, kwargs=kwargs) - def send_local_control_task(self, task, kwargs=None): - return send_local_control_task(app=self.app, task=task, kwargs=kwargs) + def send_local_control_task(self, task, get_response=False, kwargs=None): + return send_local_control_task(app=self.app, get_response=get_response, task=task, kwargs=kwargs) @property def declare_queues(self): diff --git a/lib/galaxy/queues.py b/lib/galaxy/queues.py index abb9b993027..5b74c0e496d 100644 --- a/lib/galaxy/queues.py +++ b/lib/galaxy/queues.py @@ -32,7 +32,7 @@ def control_queues_from_config(config): """ hostname = socket.gethostname() process_name = "{server_name}@{hostname}".format(server_name=config.server_name, hostname=hostname) - exchange_queue = Queue("control.%s" % process_name, galaxy_exchange, routing_key='control.%s' % process_name) + exchange_queue = Queue("control.%s" % process_name, galaxy_exchange, routing_key='control.*') non_exchange_queue = Queue("control.%s" % process_name, routing_key='control.%s' % process_name) return exchange_queue, non_exchange_queue diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 711b52c977b..208fe5f5df4 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -241,7 +241,7 @@ class ToolBox(BaseGalaxyToolBox): how to construct them, action types, dependency management, etc.... """ - def __init__(self, config_filenames, tool_root_dir, app): + def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True): self._reload_count = 0 self.tool_location_fetcher = ToolLocationFetcher() # This is here to deal with the old default value, which doesn't make @@ -253,6 +253,7 @@ class ToolBox(BaseGalaxyToolBox): config_filenames=config_filenames, tool_root_dir=tool_root_dir, app=app, + save_integrated_tool_panel=save_integrated_tool_panel, ) def can_load_config_file(self, config_filename): diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 60ed3c0e925..a19a5f91b42 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -75,7 +75,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): workflows optionally in labelled sections. """ - def __init__(self, config_filenames, tool_root_dir, app): + def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True): """ Create a toolbox from the config files named by `config_filenames`, using `tool_root_dir` as the base directory for finding individual tool config files. @@ -119,7 +119,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: # Load self._tool_panel based on the order in self._integrated_tool_panel. self._load_tool_panel() - self._save_integrated_tool_panel() + if save_integrated_tool_panel: + self._save_integrated_tool_panel() def create_tool(self, config_file, tool_shed_repository=None, guid=None, **kwds): raise NotImplementedError() @@ -1189,8 +1190,8 @@ class BaseGalaxyToolBox(AbstractToolBox): shouldn't really depend on. """ - def __init__(self, config_filenames, tool_root_dir, app): - super(BaseGalaxyToolBox, self).__init__(config_filenames, tool_root_dir, app) + def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True): + super(BaseGalaxyToolBox, self).__init__(config_filenames, tool_root_dir, app, save_integrated_tool_panel) old_toolbox = getattr(app, 'toolbox', None) if old_toolbox: self.dependency_manager = old_toolbox.dependency_manager diff --git a/lib/galaxy/tools/toolbox/integrated_panel.py b/lib/galaxy/tools/toolbox/integrated_panel.py index 9e3020282bd..fa1b7949b5e 100644 --- a/lib/galaxy/tools/toolbox/integrated_panel.py +++ b/lib/galaxy/tools/toolbox/integrated_panel.py @@ -1,3 +1,4 @@ +import logging import os import shutil import string @@ -11,6 +12,8 @@ from .panel import ( ToolPanelElements ) +log = logging.getLogger(__name__) + INTEGRATED_TOOL_PANEL_DESCRIPTION = """ This is Galaxy's integrated tool panel and should be modified directly only for reordering tools inside a section. Each time Galaxy starts up, this file is @@ -52,6 +55,7 @@ class ManagesIntegratedToolPanelMixin(object): use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. """ destination = os.path.abspath(self._integrated_tool_panel_config) + log.debug("Writing integrated tool panel config file to '%s'", destination) tracking_directory = self._integrated_tool_panel_tracking_directory if tracking_directory: if not os.path.exists(tracking_directory):