From 1fb8203a0c1f6f941e0b13fa6357ec04ea677ffb Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Mon, 10 Dec 2018 17:16:15 -0500
Subject: [PATCH] Handler assignment documentation.

---
 config/galaxy.yml.sample                    |  5 +-
 config/job_conf.xml.sample_advanced         | 66 ++++++++++++++++++---
 config/workflow_schedulers_conf.xml.sample  | 27 +++++++--
 doc/source/admin/galaxy_options.rst         |  6 +-
 doc/source/admin/jobs.md                    |  2 +-
 doc/source/admin/scaling.md                 | 20 +++++--
 lib/galaxy/jobs/manager.py                  |  8 +--
 lib/galaxy/webapps/galaxy/config_schema.yml |  6 +-
 8 files changed, 105 insertions(+), 35 deletions(-)

diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample
index fb4351ba4be..12e57d2778f 100644
--- a/config/galaxy.yml.sample
+++ b/config/galaxy.yml.sample
@@ -1563,9 +1563,8 @@ galaxy:
   # failing jobs are just failed outright.
   #default_job_resubmission_condition: null
 
-  # In multiprocess configurations, notification between processes about
-  # new jobs must be done via the database.  In single process
-  # configurations, this can be done in memory, which is a bit quicker.
+  # This option is deprecated, use the `mem-self` handler assignment
+  # option in the job configuration instead.
   #track_jobs_in_database: true
 
   # This enables splitting of jobs into tasks, if specified by the
diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced
index 361d3693526..10787bbd7d1 100644
--- a/config/job_conf.xml.sample_advanced
+++ b/config/job_conf.xml.sample_advanced
@@ -332,24 +332,76 @@
              used by Galaxy is no environment variable of the specified name is found.
         -->
     </plugins>
-    <handlers default="handlers">
-        <!-- Additional job handlers - the id should match the name of a
-             [server:<id>] in galaxy.ini.
+    <handlers>
+        <!-- Job handler processes - for a full discussion of job handlers, see the documentation at:
+             https://docs.galaxyproject.org/en/latest/admin/scaling.html
+
+             The <handlers> container tag takes two optional attributes:
+
+               <handlers assign_with="method" default="id_or_tag"/>
+
+               - `assign_with` - How jobs should be assigned to handlers. The value can be a single method or a
+                 comma-separated list that will be tried in order. The default depends on whether any handlers and a job
+                 handler "pool" (such as uWSGI mules in a `job-handlers` farm) are configured. Valid methods are:
+
+                   - `mem-self` - Jobs are assigned to the web worker that received the tool execution request from the
+                     user via an internal in-memory queue. If a tool is configured to use a specific handler, that
+                     configuration is ignored; the process that creates the job *always* handles it. This can be
+                     slightly faster than `db-self` but only makes sense in single process environments without
+                     dedicated job handlers. This option replaces the former `track_jobs_in_database` option in
+                     galaxy.yml.
+
+                   - `db-self` - Like `mem-self` but assignment occurs by setting a new job's 'handler' column in the
+                     database to the process that created the job at the time it is created. Additionally, if a tool is
+                     configured to use a specific handler (ID or tag), that handler is assigned (tags by
+                     `db-preassign`). This is the default if no handlers are defined and no `job-handlers` pool is
+                     present (the default for a completely unconfigured Galaxy).
+
+                   - `db-preassign` - Jobs are assigned a handler by selecting one at random from the configured tag or
+                      default handlers. This occurs by setting a new job's 'handler' column in the database to the
+                      chosen handler ID (hence "preassign"ment). This is the default if handlers are defined and no
+                      `job-handlers` pool is present.
+
+                   - `uwsgi-mule-message` - Jobs are assigned a handler via uWSGI mule messaging (see uWSGI
+                     documentation). A mule in the `job-handlers` (if sent to the default tag) or `job-handlers.<tag>`
+                     farm will recieve the message and assign itself. This the default if a `job-handlers` pool is
+                     present and no handlers are configured.
+
+                 In the event that both a `job-handlers` pool is present and handlers are configured, the default is
+                 `uwsgi-mule-message,db-preassign`. At present, only `uwsgi-mule-message` is capable of deferring
+                 handler assignment to a later method (which would occur in the event that a tool is configured to use
+                 a tag for which there is not a matching pool).
+
+                 In all cases, if a tool is configured to use a specific handler (by ID, not tag), configured assignment
+                 methods are ignored and that handler is directly assigned in the job's 'handler' column at job creation
+                 time.
+
+               - `default` - An ID or tag of the handler(s) that should handle any jobs not assigned to a specific
+                 handler (which is probably most of them). If unset, the default is any untagged handlers plus any
+                 handlers in the `job-handlers` (no tag) pool.
+
+                 Note that in the event such a mixed configuration environment exists (both a `job-handlers` pool (farm)
+                 and untagged handlers are configured), the default value of
+                 `assign_with="uwsgi-mule-message,db-preassign"` would prevent any of the configured handlers from being
+                 assigned since `uwsgi-mule-message` is the preferred assignment method.
+        -->
+        <!-- Explicitly defined job handlers - the id should match the handler process's `server_name`. For webless
+             handlers, this is the value of the `server-name` argument to `galaxy-main`.
          -->
-        <handler id="handler0" tags="handlers"/>
-        <handler id="handler1" tags="handlers"/>
+        <handler id="handler0"/>
+        <handler id="handler1"/>
         <!-- Handlers will load all plugins defined in the <plugins> collection
              above by default, but can be limited to a subset using <plugin>
              tags. This is useful for heterogenous environments where the DRMAA
              plugin would need to be loaded more than once with different
              configs.
-         -->
+        -->
         <handler id="sge_handler">
             <plugin id="sge"/>
         </handler>
+        <!-- Handlers are grouped by defining (comma-separated) tags -->
         <handler id="special_handler0" tags="special_handlers"/>
         <handler id="special_handler1" tags="special_handlers"/>
-        <handler id="trackster_handler"/>
     </handlers>
     <destinations default="local">
         <!-- Destinations define details about remote resources and how jobs
diff --git a/config/workflow_schedulers_conf.xml.sample b/config/workflow_schedulers_conf.xml.sample
index e719453389e..1c83136f870 100644
--- a/config/workflow_schedulers_conf.xml.sample
+++ b/config/workflow_schedulers_conf.xml.sample
@@ -13,12 +13,29 @@
 
   <!-- Handlers (Galaxy server processes that perform the scheduling work) can
        be defined here in the same format as in job_conf.xml. By default, the
-       handlers defined in job_conf.xml will be used (or `main` if there is no
-       job_conf.xml).  -->
+       handlers defined in job_conf.xml will be used (or the web process that
+       receives the workflow scheduling request if handlers are not configured
+       in job_conf.xml).
+
+       The options here are the same as is documented for <handlers> in
+       job_conf.xml.sample_advanced with two exceptions:
+
+       - If a uWSGI farm named `workflow-schedulers` is present, it will be
+         preferred, followed by `job-handlers`. If any untagged handlers are
+         defined in this configuration they are eligible to schedule workflows
+         in addition to any matching mules.
+       - If uWSGI farms are present, the default assignment method is
+         `db-preassign` rather than `uwsgi-mule-message`, because `db-preassign`
+         is deterministic. All workflows scheduled in a single history will be
+         assigned to the same handler, ensuring they are scheduled serially
+         (preventing their outputs from being interleaved in the history). You
+         can override this by explicitly setting
+         `assign_with="uwsgi-mule-message"`.
+  -->
   <!--
-  <handlers default="handlers">
-    <handler id="handler0" tags="handlers"/>
-    <handler id="handler1" tags="handlers"/>
+  <handlers>
+    <handler id="handler0"/>
+    <handler id="handler1"/>
   </handlers>
   -->
 
diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index f96ee544854..a3c509888df 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -3277,10 +3277,8 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 :Description:
-    In multiprocess configurations, notification between processes
-    about new jobs must be done via the database.  In single process
-    configurations, this can be done in memory, which is a bit
-    quicker.
+    This option is deprecated, use the `mem-self` handler assignment
+    option in the job configuration instead.
 :Default: ``true``
 :Type: bool
 
diff --git a/doc/source/admin/jobs.md b/doc/source/admin/jobs.md
index 13423a546ca..a925afbc2d9 100644
--- a/doc/source/admin/jobs.md
+++ b/doc/source/admin/jobs.md
@@ -40,7 +40,7 @@ workers
 
 The `<handlers>` configuration elements defines which Galaxy server processes (when [running multiple server processes](scaling.html)) should be used for running jobs, and how to group those processes.
 
-The handlers configuration may define a ``default`` attribute. This is the the handler(s) that should be used if no explicit handler is defined for a job and is required if >1 handlers defined.
+The handlers configuration may define a ``default`` attribute. This is the the handler(s) that should be used if no explicit handler is defined for a job. If unset, any untagged handlers will be used by default.
 
 The collection contains `<handler>` elements.
 
diff --git a/doc/source/admin/scaling.md b/doc/source/admin/scaling.md
index 5af4d2ae955..32aa47100f4 100644
--- a/doc/source/admin/scaling.md
+++ b/doc/source/admin/scaling.md
@@ -94,8 +94,20 @@ Under this strategy, job handling is offloaded to dedicated non-web-serving proc
 directly by the master uWSGI process. As a benefit of using mule messaging, only job handlers that are alive will be
 selected to run jobs.
 
-This is the recommended deployment strategy for Galaxy servers that run web servers and job handlers **on the same
-host**.
+This is the recommended deployment strategy.
+
+```eval_rst
+.. important::
+
+   If using **Zerg Mode** or running more than one uWSGI *master* process, do not use **uWSGI + Mules**. Doing so can
+   can cause jobs to be executed by mutiple handlers when recovering unassigned jobs at Galaxy server startup.
+
+   Multiple master processes is a rare configuration and is typically only used in the case of load balancing the web
+   application across multiple hosts. Note that multiple master proceses is not the same thing as the ``processess``
+   uWSGI configuration option, which is perfectly safe to set when using job handler mules.
+
+   For these scenarios, **uWSGI + Webless** is the recommended deployment strategy.
+```
 
 ### uWSGI for web serving and Webless Galaxy applications as job handlers
 
@@ -110,8 +122,8 @@ Like mules, under this strategy, job handling is offloaded to dedicated non-web-
 are [managed by the administrator](#starting-and-stopping). Because the handler is randomly assigned by the web worker
 when the job is submitted via the UI/API, jobs may be assigned to dead handlers.
 
-This is the recommended deployment strategy for Galaxy servers that run web servers and job handlers **on different
-hosts**.
+This is the recommended deployment strategy when **Zerg Mode** is used, and for Galaxy servers that run web servers and
+job handlers **on different hosts**.
 
 ## Legacy Deployment Options
 
diff --git a/lib/galaxy/jobs/manager.py b/lib/galaxy/jobs/manager.py
index 325e7e26f12..891ff850543 100644
--- a/lib/galaxy/jobs/manager.py
+++ b/lib/galaxy/jobs/manager.py
@@ -26,15 +26,9 @@ class JobManager(object):
             self.job_handler = handler.JobHandler(app)
         else:
             self.job_handler = NoopHandler()
-        self.__check_jobs_at_startup()
+            self.__check_jobs_at_startup()
 
     def __check_jobs_at_startup(self):
-        """
-        TODO: It should be documented that starting two Galaxy uWSGI master processes simultaneously would result in a race condition that *could* cause two handlers to pick up the same job.
-
-        The recommended config for now will be webless handlers if running more than one uWSGI (master) process
-        """
-        # FIXME: test
         if self.app.job_config.use_messaging:
             jobs_at_startup = self.app.model.context.query(Job).enable_eagerloads(False) \
                 .filter((Job.state == Job.states.NEW) & (Job.handler == null())).all()
diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml
index 854ad3f8170..626fc0b14c6 100644
--- a/lib/galaxy/webapps/galaxy/config_schema.yml
+++ b/lib/galaxy/webapps/galaxy/config_schema.yml
@@ -2437,14 +2437,12 @@ mapping:
           failing jobs are just failed outright.
 
       track_jobs_in_database:
-        # FIXME: note deprecated, superceded by job_handler_assignment_method
         type: bool
         default: true
         required: false
         desc: |
-          In multiprocess configurations, notification between processes about new jobs
-          must be done via the database.  In single process configurations, this can be
-          done in memory, which is a bit quicker.
+          This option is deprecated, use the `mem-self` handler assignment option in the
+          job configuration instead.
 
       use_tasked_jobs:
         type: bool