Merge pull request #178 from natefoo/release_15.03

[STABLE] Fix job stopping under some conditions

lib/galaxy/jobs/handler.py

@@ -680,7 +680,10 @@ class JobHandlerStopQueue( object ):
         except Empty:
             pass
         for job, error_msg in jobs_to_check:
-            if job.state != job.states.DELETED_NEW and job.finished:
+            if ( job.state not in ( job.states.DELETED_NEW,
+                                    job.states.DELETED )
+                 and job.finished ):
+                # terminated before it got here
                 log.debug('Job %s already finished, not deleting or stopping', job.id)
                 continue
             final_state = job.states.DELETED
@@ -695,7 +698,8 @@ class JobHandlerStopQueue( object ):
                 self.dispatcher.stop( job )
 
     def put( self, job_id, error_msg=None ):
-        self.queue.put( ( job_id, error_msg ) )
+        if not self.app.config.track_jobs_in_database:
+            self.queue.put( ( job_id, error_msg ) )
 
     def shutdown( self ):
         """Attempts to gracefully shut down the worker thread"""

lib/galaxy/jobs/runners/local.py

@@ -134,9 +134,11 @@ class LocalJobRunner( BaseJobRunner ):
     def stop_job( self, job ):
         #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished
         job_ext_output_metadata = job.get_external_output_metadata()
-        if job_ext_output_metadata:
+        try:
             pid = job_ext_output_metadata[0].job_runner_external_pid  # every JobExternalOutputMetadata has a pid set, we just need to take from one of them
-        else:
+            assert pid not in [ None, '' ]
+        except Exception:
+            # metadata internal or job not complete yet
             pid = job.get_job_runner_external_id()
         if pid in [ None, '' ]:
             log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.get_id() )