Merge pull request #2309 from nsoranzo/release_16.01_slurm_mem_limit

[16.01] Remove SLURM memory limit warning from stderr if the job was successful

lib/galaxy/jobs/runners/slurm.py

@@ -14,6 +14,8 @@ log = logging.getLogger( __name__ )
 __all__ = [ 'SlurmJobRunner' ]
 
 SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit'
+SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.',
+                                                ': Exceeded step memory limit at some point.']
 
 
 class SlurmJobRunner( DRMAAJobRunner ):
@@ -83,6 +85,17 @@ class SlurmJobRunner( DRMAAJobRunner ):
             except Exception as e:
                 log.exception( '(%s/%s) Unable to inspect failed slurm job using scontrol, job will be unconditionally failed: %s', ajs.job_wrapper.get_id_tag(), ajs.job_id, e )
                 super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state )
+        if drmaa_state == self.drmaa_job_states.DONE:
+            with open(ajs.error_file, 'r+') as f:
+                lines = f.readlines()
+                f.seek(0)
+                for line in lines:
+                    stripped_line = line.strip()
+                    if any([_ in stripped_line for _ in SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS]):
+                        log.debug( '(%s/%s) Job completed, removing SLURM exceeded memory warning: "%s"', ajs.job_wrapper.get_id_tag(), ajs.job_id, stripped_line )
+                    else:
+                        f.write(line)
+                f.truncate()
         # by default, finish as if the job was successful.
         super( SlurmJobRunner, self )._complete_terminal_job( ajs, drmaa_state=drmaa_state )