New Feature: Administrative Job Lock.

Admins can now lock (and unlock) all job dispatching.  New jobs will remain in the job queue in the 'waiting' state until the lock is removed.  Existing dispatched jobs are unaffected, and jobs can still be submitted, but they will not dispatch.

lib/galaxy/jobs/__init__.py

@@ -73,6 +73,7 @@ class JobQueue( object ):
         """Start the job manager"""
         self.app = app
         self.sa_session = app.model.context
+        self.job_lock = False
         # Should we read jobs form the database, or use an in memory queue
         self.track_jobs_in_database = app.config.get_bool( 'track_jobs_in_database', False )
         # Keep track of the pid that started the job manager, only it
@@ -197,8 +198,13 @@ class JobQueue( object ):
                 elif job_state == JOB_INPUT_DELETED:
                     log.info( "job %d unable to run: one or more inputs deleted" % job.job_id )
                 elif job_state == JOB_READY:
-                    self.dispatcher.put( job )
-                    log.debug( "job %d dispatched" % job.job_id)
+                    if self.job_lock:
+                        log.info("Job dispatch attempted for %s, but prevented by administrative lock." % job.job_id)
+                        if not self.track_jobs_in_database:
+                            new_waiting.append( job )
+                    else:
+                        self.dispatcher.put( job )
+                        log.debug( "job %d dispatched" % job.job_id)
                 elif job_state == JOB_DELETED:
                     msg = "job %d deleted by user while still queued" % job.job_id
                     job.info = msg

lib/galaxy/web/base/controller.py

@@ -1404,7 +1404,8 @@ class Admin( object ):
 
     @web.expose
     @web.require_admin
-    def jobs( self, trans, stop = [], stop_msg = None, cutoff = 180, **kwd ):
+    def jobs( self, trans, stop = [], stop_msg = None, cutoff = 180, job_lock = None, **kwd ):
+        # DBTODO admin job lock.
         deleted = []
         msg = None
         status = None
@@ -1425,6 +1426,10 @@ class Admin( object ):
             msg += ' for deletion: '
             msg += ', '.join( deleted )
             status = 'done'
+        if job_lock == 'lock':
+            trans.app.job_manager.job_queue.job_lock = True
+        elif job_lock == 'unlock':
+            trans.app.job_manager.job_queue.job_lock = False
         cutoff_time = datetime.utcnow() - timedelta( seconds=int( cutoff ) )
         jobs = trans.sa_session.query( trans.app.model.Job ) \
                                .filter( and_( trans.app.model.Job.table.c.update_time < cutoff_time,
@@ -1445,7 +1450,8 @@ class Admin( object ):
                                     last_updated = last_updated,
                                     cutoff = cutoff,
                                     msg = msg,
-                                    status = status )
+                                    status = status,
+                                    job_lock = trans.app.job_manager.job_queue.job_lock )
 
 ## ---- Utility methods -------------------------------------------------------
         

templates/admin/jobs.mako

@@ -110,5 +110,30 @@
             </div>
         </div>
     </div>
+	<p/>
+    <div class="toolForm">
+        <div class="toolFormTitle">
+            Administrative Job Lock
+        </div>
+        <div class="toolFormBody">
+	%if job_lock==True:
+            <div class="form-row">
+				<p>All job execution is currently locked.  Click here to unlock.</p>
+				<input type='hidden' name='job_lock' value='unlock'/>
+            </div>
+            <div class="form-row">
+                <input type="submit" class="primary-button" name="submit" value="Unlock">
+            </div>
+	%else:
+		    <div class="form-row">
+				<p>To prevent new jobs from dispatching, you can lock down the job queue here.</p>
+				<input type='hidden' name='job_lock' value='lock'/>
+		    </div>
+		    <div class="form-row">
+		        <input type="submit" class="primary-button" name="submit" value="Lock">
+		    </div>
+	%endif
+        </div>
+    </div>
 
 </form>