From 3005bba5e389c86fccabe129326a1f3a675c6911 Mon Sep 17 00:00:00 2001 From: Lance Parsons Date: Wed, 4 May 2016 16:17:07 +0100 Subject: [PATCH 1/3] Remove SLURM memory limit warning from stderr if the job was successful --- lib/galaxy/jobs/runners/slurm.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 9b7f7a1be7f..ad8c11c2418 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -13,7 +13,8 @@ log = logging.getLogger( __name__ ) __all__ = [ 'SlurmJobRunner' ] -SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' +SLURM_MEMORY_LIMIT_EXCEEDED_MSGS = ['slurmstepd: error: Exceeded job memory limit at some point. Job may have been partially swapped out to disk.', + 'slurmstepd: error: Exceeded step memory limit at some point. Step may have been partially swapped out to disk.'] class SlurmJobRunner( DRMAAJobRunner ): @@ -83,6 +84,18 @@ class SlurmJobRunner( DRMAAJobRunner ): except Exception as e: log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state ) + elif drmaa_state == self.drmaa_job_states.DONE: + log.debug( '(%s/%s) Job completed, checking for SLURM Exceeded memory warnings', ajs.job_wrapper.get_id_tag(), ajs.job_id ) + f = open(ajs.error_file,"r+") + d = f.readlines() + f.seek(0) + for i in d: + if i.strip() not in SLURM_MEMORY_LIMIT_EXCEEDED_MSGS: + f.write(i) + else: + log.debug( '(%s/%s) Job completed, removing SLURM exceeded memory warning: (%s)', ajs.job_wrapper.get_id_tag(), ajs.job_id, i ) + f.truncate() + f.close() # by default, finish as if the job was successful. super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state ) @@ -104,7 +117,7 @@ class SlurmJobRunner( DRMAAJobRunner ): f.seek(-pos + 1, 2) bof = True - if (bof or f.read(1) == '\n') and f.readline().strip() == SLURM_MEMORY_LIMIT_EXCEEDED_MSG: + if (bof or f.read(1) == '\n') and f.readline().strip() in SLURM_MEMORY_LIMIT_EXCEEDED_MSGS: return True if bof: From 338d25403109244f0b228f9a0df23e02e1de7578 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 4 May 2016 17:39:04 +0100 Subject: [PATCH 2/3] Use partial messages to match SLURM memory warnings --- lib/galaxy/jobs/runners/slurm.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index ad8c11c2418..91323441983 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -13,8 +13,9 @@ log = logging.getLogger( __name__ ) __all__ = [ 'SlurmJobRunner' ] -SLURM_MEMORY_LIMIT_EXCEEDED_MSGS = ['slurmstepd: error: Exceeded job memory limit at some point. Job may have been partially swapped out to disk.', - 'slurmstepd: error: Exceeded step memory limit at some point. Step may have been partially swapped out to disk.'] +SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' +SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.', + ': Exceeded step memory limit at some point.'] class SlurmJobRunner( DRMAAJobRunner ): @@ -85,17 +86,16 @@ class SlurmJobRunner( DRMAAJobRunner ): log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state ) elif drmaa_state == self.drmaa_job_states.DONE: - log.debug( '(%s/%s) Job completed, checking for SLURM Exceeded memory warnings', ajs.job_wrapper.get_id_tag(), ajs.job_id ) - f = open(ajs.error_file,"r+") - d = f.readlines() - f.seek(0) - for i in d: - if i.strip() not in SLURM_MEMORY_LIMIT_EXCEEDED_MSGS: - f.write(i) - else: - log.debug( '(%s/%s) Job completed, removing SLURM exceeded memory warning: (%s)', ajs.job_wrapper.get_id_tag(), ajs.job_id, i ) - f.truncate() - f.close() + with open(ajs.error_file, 'r+') as f: + lines = f.readlines() + f.seek(0) + for line in lines: + stripped_line = line.strip() + if any([_ in stripped_line for _ in SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS]): + log.debug( '(%s/%s) Job completed, removing SLURM exceeded memory warning: "%s"', ajs.job_wrapper.get_id_tag(), ajs.job_id, stripped_line ) + else: + f.write(line) + f.truncate() # by default, finish as if the job was successful. super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state ) @@ -117,7 +117,7 @@ class SlurmJobRunner( DRMAAJobRunner ): f.seek(-pos + 1, 2) bof = True - if (bof or f.read(1) == '\n') and f.readline().strip() in SLURM_MEMORY_LIMIT_EXCEEDED_MSGS: + if (bof or f.read(1) == '\n') and f.readline().strip() == SLURM_MEMORY_LIMIT_EXCEEDED_MSG: return True if bof: From ad0647d677cb1979b9841cf0c56ba26d7ec5ca34 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 4 May 2016 20:12:03 +0100 Subject: [PATCH 3/3] Change elif to if because drmaa_state can be changed inside the if --- lib/galaxy/jobs/runners/slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 91323441983..2cc028284eb 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -85,7 +85,7 @@ class SlurmJobRunner( DRMAAJobRunner ): except Exception as e: log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e ) super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state ) - elif drmaa_state == self.drmaa_job_states.DONE: + if drmaa_state == self.drmaa_job_states.DONE: with open(ajs.error_file, 'r+') as f: lines = f.readlines() f.seek(0)