From f0fc892089c2206358110c1081e25095176e662c Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 28 Apr 2015 11:43:18 -0400 Subject: [PATCH 1/2] Fix job stopping under some conditions: 1. track_jobs_in_database is False 2. Jobs being stopped were in the local runner 3. Jobs were deleted with the deleted state and not deleted_new --- lib/galaxy/jobs/handler.py | 8 ++++++-- lib/galaxy/jobs/runners/local.py | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 92d3b073e6d..d366d5f1f7c 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -680,7 +680,10 @@ class JobHandlerStopQueue( object ): except Empty: pass for job, error_msg in jobs_to_check: - if job.state != job.states.DELETED_NEW and job.finished: + if ( job.state not in ( job.states.DELETED_NEW, + job.states.DELETED ) + and job.finished ): + # terminated before it got here log.debug('Job %s already finished, not deleting or stopping', job.id) continue final_state = job.states.DELETED @@ -695,7 +698,8 @@ class JobHandlerStopQueue( object ): self.dispatcher.stop( job ) def put( self, job_id, error_msg=None ): - self.queue.put( ( job_id, error_msg ) ) + if not self.app.config.track_jobs_in_database: + self.queue.put( ( job_id, error_msg ) ) def shutdown( self ): """Attempts to gracefully shut down the worker thread""" diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index f06564711e3..360f98b09dd 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -134,9 +134,11 @@ class LocalJobRunner( BaseJobRunner ): def stop_job( self, job ): #if our local job has JobExternalOutputMetadata associated, then our primary job has to have already finished job_ext_output_metadata = job.get_external_output_metadata() - if job_ext_output_metadata: + try: pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them - else: + assert pid not in [ None, '' ] + except: + # metadata internal or job not complete yet pid = job.get_job_runner_external_id() if pid in [ None, '' ]: log.warning( "stop_job(): %s: no PID in database for job, unable to stop" % job.get_id() ) From 3211466d68d633849b5e930800b3ef3695a0e139 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 28 Apr 2015 15:26:29 -0400 Subject: [PATCH 2/2] Don't use a bare except --- lib/galaxy/jobs/runners/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index 360f98b09dd..475ea703bab 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -137,7 +137,7 @@ class LocalJobRunner( BaseJobRunner ): try: pid = job_ext_output_metadata[0].job_runner_external_pid # every JobExternalOutputMetadata has a pid set, we just need to take from one of them assert pid not in [ None, '' ] - except: + except Exception: # metadata internal or job not complete yet pid = job.get_job_runner_external_id() if pid in [ None, '' ]: