Merge pull request #9487 from mvdbeek/save_integrated_tool_panel_only_once_during_reload

[20.01] Save integrated tool panel once on toolbox reload

lib/galaxy/config_watchers.py

@@ -1,3 +1,4 @@
+import logging
 from os.path import dirname
 
 from galaxy.queue_worker import job_rule_modules
@@ -7,6 +8,8 @@ from galaxy.tools.toolbox.watcher import (
 )
 from galaxy.util.watcher import get_watcher
 
+log = logging.getLogger(__name__)
+
 
 class ConfigWatchers(object):
     """Contains ToolConfWatcher, ToolWatcher and ToolDataWatcher objects."""
@@ -21,8 +24,20 @@ class ConfigWatchers(object):
         # If there are multiple ToolConfWatcher objects for the same handler or web process a race condition occurs between the two cache_cleanup functions.
         # If the reload_data_managers callback wins, the cache will miss the tools that had been removed from the cache
         # and will be blind to further changes in these tools.
+
+        def reload_toolbox():
+            save_integrated_tool_panel = False
+            try:
+                # Run and wait for toolbox reload on the process that watches the config files.
+                # The toolbpox reload will update the integrated_tool_panel_file
+                self.app.queue_worker.send_local_control_task('reload_toolbox', get_response=True),
+            except Exception:
+                save_integrated_tool_panel = True
+                log.exception("Exception occured while reloading toolbox")
+            self.app.queue_worker.send_control_task('reload_toolbox', noop_self=True, kwargs={'save_integrated_tool_panel': save_integrated_tool_panel}),
+
         self.tool_config_watcher = get_tool_conf_watcher(
-            reload_callback=lambda: self.app.queue_worker.send_control_task('reload_toolbox'),
+            reload_callback=reload_toolbox,
             tool_cache=self.app.tool_cache,
         )
         self.data_manager_config_watcher = get_tool_conf_watcher(

lib/galaxy/queue_worker.py

@@ -42,7 +42,7 @@ def send_local_control_task(app, task, get_response=False, kwargs=None):
     """
     if kwargs is None:
         kwargs = {}
-    log.info("Queuing async task %s for %s." % (task, app.config.server_name))
+    log.info("Queuing %s task %s for %s." % ("sync" if get_response else "async", task, app.config.server_name))
     payload = {'task': task,
                'kwargs': kwargs}
     routing_key = 'control.%s@%s' % (app.config.server_name, socket.gethostname())
@@ -114,7 +114,7 @@ class ControlTask(object):
             callback_queue = [self.callback_queue]
             self.correlation_id = uuid()
         try:
-            with producers[self.connection].acquire(block=True) as producer:
+            with producers[self.connection].acquire(block=True, timeout=10) as producer:
                 producer.publish(
                     payload,
                     exchange=None if local else self.exchange,
@@ -163,19 +163,19 @@ def reload_tool(app, **kwargs):
         log.error("Reload tool invoked without tool id.")
 
 
-def reload_toolbox(app, **kwargs):
+def reload_toolbox(app, save_integrated_tool_panel=True, **kwargs):
     reload_timer = util.ExecutionTimer()
     log.debug("Executing toolbox reload on '%s'", app.config.server_name)
     reload_count = app.toolbox._reload_count
     if hasattr(app, 'tool_cache'):
         app.tool_cache.cleanup()
-    _get_new_toolbox(app)
+    _get_new_toolbox(app, save_integrated_tool_panel)
     app.toolbox._reload_count = reload_count + 1
     send_local_control_task(app, 'rebuild_toolbox_search_index')
     log.debug("Toolbox reload %s", reload_timer)
 
 
-def _get_new_toolbox(app):
+def _get_new_toolbox(app, save_integrated_tool_panel=True):
     """
     Generate a new toolbox, by constructing a toolbox from the config files,
     and then adding pre-existing data managers from the old toolbox to the new toolbox.
@@ -186,7 +186,7 @@ def _get_new_toolbox(app):
         app.tool_shed_repository_cache.rebuild()
     tool_configs = app.config.tool_configs
 
-    new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app)
+    new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app, save_integrated_tool_panel=save_integrated_tool_panel)
     new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
     app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True)
     app.datatypes_registry.load_external_metadata_tool(new_toolbox)
@@ -352,8 +352,8 @@ class GalaxyQueueWorker(ConsumerProducerMixin, threading.Thread):
     def send_control_task(self, task, noop_self=False, get_response=False, routing_key='control.*', kwargs=None):
         return send_control_task(app=self.app, task=task, noop_self=noop_self, get_response=get_response, routing_key=routing_key, kwargs=kwargs)
 
-    def send_local_control_task(self, task, kwargs=None):
-        return send_local_control_task(app=self.app, task=task, kwargs=kwargs)
+    def send_local_control_task(self, task, get_response=False, kwargs=None):
+        return send_local_control_task(app=self.app, get_response=get_response, task=task, kwargs=kwargs)
 
     @property
     def declare_queues(self):

lib/galaxy/queues.py

@@ -32,7 +32,7 @@ def control_queues_from_config(config):
     """
     hostname = socket.gethostname()
     process_name = "{server_name}@{hostname}".format(server_name=config.server_name, hostname=hostname)
-    exchange_queue = Queue("control.%s" % process_name, galaxy_exchange, routing_key='control.%s' % process_name)
+    exchange_queue = Queue("control.%s" % process_name, galaxy_exchange, routing_key='control.*')
     non_exchange_queue = Queue("control.%s" % process_name, routing_key='control.%s' % process_name)
     return exchange_queue, non_exchange_queue
 

lib/galaxy/tools/__init__.py

@@ -241,7 +241,7 @@ class ToolBox(BaseGalaxyToolBox):
     how to construct them, action types, dependency management, etc....
     """
 
-    def __init__(self, config_filenames, tool_root_dir, app):
+    def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True):
         self._reload_count = 0
         self.tool_location_fetcher = ToolLocationFetcher()
         # This is here to deal with the old default value, which doesn't make
@@ -253,6 +253,7 @@ class ToolBox(BaseGalaxyToolBox):
             config_filenames=config_filenames,
             tool_root_dir=tool_root_dir,
             app=app,
+            save_integrated_tool_panel=save_integrated_tool_panel,
         )
 
     def can_load_config_file(self, config_filename):

lib/galaxy/tools/toolbox/base.py

@@ -75,7 +75,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
     workflows optionally in labelled sections.
     """
 
-    def __init__(self, config_filenames, tool_root_dir, app):
+    def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True):
         """
         Create a toolbox from the config files named by `config_filenames`, using
         `tool_root_dir` as the base directory for finding individual tool config files.
@@ -119,7 +119,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
         if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents:
             # Load self._tool_panel based on the order in self._integrated_tool_panel.
             self._load_tool_panel()
-        self._save_integrated_tool_panel()
+        if save_integrated_tool_panel:
+            self._save_integrated_tool_panel()
 
     def create_tool(self, config_file, tool_shed_repository=None, guid=None, **kwds):
         raise NotImplementedError()
@@ -1189,8 +1190,8 @@ class BaseGalaxyToolBox(AbstractToolBox):
     shouldn't really depend on.
     """
 
-    def __init__(self, config_filenames, tool_root_dir, app):
-        super(BaseGalaxyToolBox, self).__init__(config_filenames, tool_root_dir, app)
+    def __init__(self, config_filenames, tool_root_dir, app, save_integrated_tool_panel=True):
+        super(BaseGalaxyToolBox, self).__init__(config_filenames, tool_root_dir, app, save_integrated_tool_panel)
         old_toolbox = getattr(app, 'toolbox', None)
         if old_toolbox:
             self.dependency_manager = old_toolbox.dependency_manager

lib/galaxy/tools/toolbox/integrated_panel.py

@@ -1,3 +1,4 @@
+import logging
 import os
 import shutil
 import string
@@ -11,6 +12,8 @@ from .panel import (
     ToolPanelElements
 )
 
+log = logging.getLogger(__name__)
+
 INTEGRATED_TOOL_PANEL_DESCRIPTION = """
 This is Galaxy's integrated tool panel and should be modified directly only for
 reordering tools inside a section. Each time Galaxy starts up, this file is
@@ -52,6 +55,7 @@ class ManagesIntegratedToolPanelMixin(object):
         use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right.
         """
         destination = os.path.abspath(self._integrated_tool_panel_config)
+        log.debug("Writing integrated tool panel config file to '%s'", destination)
         tracking_directory = self._integrated_tool_panel_tracking_directory
         if tracking_directory:
             if not os.path.exists(tracking_directory):