From 214e757d81623e9c12d171f3dcfc516c7dcc5e32 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 31 May 2016 11:42:36 -0400 Subject: [PATCH 1/2] Only scan the last 16 MB of a job's stderr for slurm memory warning messages. --- lib/galaxy/jobs/runners/slurm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index 7349586ec8c..bab11589341 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -2,6 +2,7 @@ SLURM job control via the DRMAA API. """ +import os import time import logging import subprocess @@ -16,6 +17,7 @@ __all__ = [ 'SlurmJobRunner' ] SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.', ': Exceeded step memory limit at some point.'] +SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB class SlurmJobRunner( DRMAAJobRunner ): @@ -88,6 +90,8 @@ class SlurmJobRunner( DRMAAJobRunner ): return if drmaa_state == self.drmaa_job_states.DONE: with open(ajs.error_file, 'r+') as f: + if os.path.getsize(ajs.error_file) > SLURM_MEMORY_LIMIT_SCAN_SIZE: + f.seek(-SLURM_MEMORY_LIMIT_SCAN_SIZE, os.SEEK_END) lines = f.readlines() f.seek(0) for line in lines: From dcb28e095012fef3be769f677fa572f13d4ed88a Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 31 May 2016 11:58:57 -0400 Subject: [PATCH 2/2] Fix linting error --- lib/galaxy/jobs/runners/slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py index bab11589341..834e289b43b 100644 --- a/lib/galaxy/jobs/runners/slurm.py +++ b/lib/galaxy/jobs/runners/slurm.py @@ -17,7 +17,7 @@ __all__ = [ 'SlurmJobRunner' ] SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit' SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.', ': Exceeded step memory limit at some point.'] -SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB +SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB class SlurmJobRunner( DRMAAJobRunner ):