From 7c40c85034d918c50e8b22c1a3eb39746a6e043d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 10 Jun 2016 12:29:29 +0100 Subject: [PATCH 1/2] Fix for job handlers losing track of jobs when using DRMAA with internalexception_retries or invalidjobexception_retries --- lib/galaxy/jobs/runners/drmaa.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index b1dacde9c2b..fb46c9e51a1 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -268,6 +268,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): if retries < self.runner_params[ retry_param ]: # will retry check on next iteration setattr( ajs, retry_param, retries + 1 ) + new_watched.append( ajs ) continue if self.runner_params[ state_param ] == model.Job.states.OK: log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e ) From 933f42619e2fb2fb518702397aa2f11ea2cfb8e0 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 10 Jun 2016 12:34:48 +0100 Subject: [PATCH 2/2] Log all failed job status checks --- lib/galaxy/jobs/runners/drmaa.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index fb46c9e51a1..02a993116d6 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -257,13 +257,11 @@ class DRMAAJobRunner( AsynchronousJobRunner ): assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id ) state = self.ds.jobStatus( external_job_id ) except ( drmaa.InternalException, drmaa.InvalidJobException ) as e: - if isinstance( e , drmaa.InvalidJobException ): - ecn = "InvalidJobException".lower() - else: - ecn = "InternalException".lower() + ecn = type(e).__name__ retry_param = ecn.lower() + '_retries' state_param = ecn.lower() + '_state' retries = getattr( ajs, retry_param, 0 ) + log.warning("(%s/%s) unable to check job status because of %s exception for %d tries: %s", galaxy_id_tag, external_job_id, ecn, retries + 1, e) if self.runner_params[ retry_param ] > 0: if retries < self.runner_params[ retry_param ]: # will retry check on next iteration @@ -271,10 +269,10 @@ class DRMAAJobRunner( AsynchronousJobRunner ): new_watched.append( ajs ) continue if self.runner_params[ state_param ] == model.Job.states.OK: - log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e ) + log.warning( "(%s/%s) job will now be finished OK", galaxy_id_tag, external_job_id ) self.work_queue.put( ( self.finish_job, ajs ) ) elif self.runner_params[ state_param ] == model.Job.states.ERROR: - log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e ) + log.warning( "(%s/%s) job will now be errored", galaxy_id_tag, external_job_id ) self.work_queue.put( ( self.fail_job, ajs ) ) else: raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] ) @@ -285,7 +283,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): continue except Exception as e: # so we don't kill the monitor thread - log.exception( "(%s/%s) Unable to check job status: %s" % ( galaxy_id_tag, external_job_id, str( e ) ) ) + log.exception( "(%s/%s) unable to check job status: %s" % ( galaxy_id_tag, external_job_id, e ) ) log.warning( "(%s/%s) job will now be errored" % ( galaxy_id_tag, external_job_id ) ) ajs.fail_message = "Cluster could not complete job" self.work_queue.put( ( self.fail_job, ajs ) )