opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Log all failed job status checks

Browse Source
This commit is contained in:
Nicola Soranzo
2016-06-10 12:34:48 +01:00
parent 7c40c85034
commit 933f42619e
1 changed files with 5 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/jobs/runners/drmaa.py
+5 -7
View File
@@ -257,13 +257,11 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
assert external_job_id not in ( None, 'None' ), '(%s/%s) Invalid job id' % ( galaxy_id_tag, external_job_id )
state = self.ds.jobStatus( external_job_id )
except ( drmaa.InternalException, drmaa.InvalidJobException ) as e:
if isinstance( e , drmaa.InvalidJobException ):
ecn = "InvalidJobException".lower()
else:
ecn = "InternalException".lower()
ecn = type(e).__name__
retry_param = ecn.lower() + '_retries'
state_param = ecn.lower() + '_state'
retries = getattr( ajs, retry_param, 0 )
log.warning("(%s/%s) unable to check job status because of %s exception for %d tries: %s", galaxy_id_tag, external_job_id, ecn, retries + 1, e)
if self.runner_params[ retry_param ] > 0:
if retries < self.runner_params[ retry_param ]:
# will retry check on next iteration
@@ -271,10 +269,10 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
new_watched.append( ajs )
continue
if self.runner_params[ state_param ] == model.Job.states.OK:
log.info( "(%s/%s) job left DRM queue with following message: %s", galaxy_id_tag, external_job_id, e )
log.warning( "(%s/%s) job will now be finished OK", galaxy_id_tag, external_job_id )
self.work_queue.put( ( self.finish_job, ajs ) )
elif self.runner_params[ state_param ] == model.Job.states.ERROR:
log.info( "(%s/%s) job check resulted in %s after %s tries: %s", galaxy_id_tag, external_job_id, ecn, retries, e )
log.warning( "(%s/%s) job will now be errored", galaxy_id_tag, external_job_id )
self.work_queue.put( ( self.fail_job, ajs ) )
else:
raise Exception( "%s is set to an invalid value (%s), this should not be possible. See galaxy.jobs.drmaa.__init__()", state_param, self.runner_params[ state_param ] )
@@ -285,7 +283,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ):
continue
except Exception as e:
# so we don't kill the monitor thread
log.exception( "(%s/%s) Unable to check job status: %s" % ( galaxy_id_tag, external_job_id, str( e ) ) )
log.exception( "(%s/%s) unable to check job status: %s" % ( galaxy_id_tag, external_job_id, e ) )
log.warning( "(%s/%s) job will now be errored" % ( galaxy_id_tag, external_job_id ) )
ajs.fail_message = "Cluster could not complete job"
self.work_queue.put( ( self.fail_job, ajs ) )