From 7e2a77e8af152a501f124a6d0f702a66502e0dd3 Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Wed, 12 Sep 2018 13:59:26 -0400
Subject: [PATCH 1/2] Catch additional permutations of Slurm cgroup creation
 warnings

---
 lib/galaxy/jobs/runners/slurm.py | 46 +++++++++++++++++++++++++++-----
 1 file changed, 39 insertions(+), 7 deletions(-)

diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py
index 5c7856f1dfc..32cd36880c5 100644
--- a/lib/galaxy/jobs/runners/slurm.py
+++ b/lib/galaxy/jobs/runners/slurm.py
@@ -29,6 +29,13 @@ SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at
                                                 ': Exceeded step memory limit at some point.']
 SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024  # 16MB
 SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE = re.compile(r"""slurmstepd: error: task/cgroup: unable to add task\[pid=\d+\] to memory cg '\(null\)'$""")
+SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: xcgroup_instantiate: unable to create cgroup '[^']+' : No space left on device$""")
+SLURM_UNABLE_TO_INSTANCIATE_JOB_MSG_RE = re.compile(r"""slurmstepd: error: jobacct_gather/cgroup: unable to instanciate job \d+ memory cgroup$""")
+SLURM_TOP_WARNING_RES = (
+    SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE,
+    SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE,
+    SLURM_UNABLE_TO_INSTANCIATE_JOB_MSG_RE
+)
 
 # These messages are returned to the user
 OUT_OF_MEMORY_MSG = 'This job was terminated because it used more memory than it was allocated.'
@@ -154,13 +161,7 @@ class SlurmJobRunner(DRMAAJobRunner):
                     return
             if drmaa_state == self.drmaa_job_states.DONE:
                 with open(ajs.error_file, 'r') as rfh:
-                    first_line = rfh.readline()
-                    if SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE.match(first_line):
-                        with tempfile.NamedTemporaryFile('w', delete=False) as wfh:
-                            shutil.copyfileobj(rfh, wfh)
-                            wf_name = wfh.name
-                        shutil.move(wf_name, ajs.error_file)
-                        log.debug('(%s/%s) Job completed, removing SLURM spurious warning: "%s"', ajs.job_wrapper.get_id_tag(), ajs.job_id, first_line)
+                    _remove_spurious_top_lines(rfh, ajs)
                 with open(ajs.error_file, 'r+') as f:
                     if os.path.getsize(ajs.error_file) > SLURM_MEMORY_LIMIT_SCAN_SIZE:
                         f.seek(-SLURM_MEMORY_LIMIT_SCAN_SIZE, os.SEEK_END)
@@ -201,3 +202,34 @@ class SlurmJobRunner(DRMAAJobRunner):
             log.exception('Error reading end of %s:', efile_path)
 
         return False
+
+
+def _remove_spurious_top_lines(rfh, ajs, maxlines=2):
+    bad = []
+    putback = None
+    i = 0
+    while i < maxlines:
+        line = rfh.readline()
+        log.debug('#### checking line: %s', line)
+        for pattern in SLURM_TOP_WARNING_RES:
+            if pattern.match(line):
+                bad.append(line)
+                # found a match, stop checking REs and check next line
+                break
+        else:
+            if bad:
+                # no match found on this line so line is now a good line, but previous bad lines are found, so it needs to be put back
+                putback = line
+            # no match on this line, stop looking
+            break
+        # check next line
+        i += 1
+    if bad:
+        with tempfile.NamedTemporaryFile('w', delete=False) as wfh:
+            if putback:
+                wfh.write(putback)
+            shutil.copyfileobj(rfh, wfh)
+            wf_name = wfh.name
+        shutil.move(wf_name, ajs.error_file)
+        for line in bad:
+            log.debug('(%s/%s) Job completed, removing SLURM spurious warning: "%s"', ajs.job_wrapper.get_id_tag(), ajs.job_id, line)

From 3d508a01d3d58908b667e428f125df91d8c599c1 Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Fri, 14 Sep 2018 14:39:01 -0400
Subject: [PATCH 2/2] Improvements from @nsoranzo

---
 lib/galaxy/jobs/runners/slurm.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/jobs/runners/slurm.py b/lib/galaxy/jobs/runners/slurm.py
index 32cd36880c5..0abf61734ad 100644
--- a/lib/galaxy/jobs/runners/slurm.py
+++ b/lib/galaxy/jobs/runners/slurm.py
@@ -207,10 +207,9 @@ class SlurmJobRunner(DRMAAJobRunner):
 def _remove_spurious_top_lines(rfh, ajs, maxlines=2):
     bad = []
     putback = None
-    i = 0
-    while i < maxlines:
+    for i in range(maxlines):
         line = rfh.readline()
-        log.debug('#### checking line: %s', line)
+        log.trace('checking line: %s', line)
         for pattern in SLURM_TOP_WARNING_RES:
             if pattern.match(line):
                 bad.append(line)
@@ -223,10 +222,9 @@ def _remove_spurious_top_lines(rfh, ajs, maxlines=2):
             # no match on this line, stop looking
             break
         # check next line
-        i += 1
     if bad:
         with tempfile.NamedTemporaryFile('w', delete=False) as wfh:
-            if putback:
+            if putback is not None:
                 wfh.write(putback)
             shutil.copyfileobj(rfh, wfh)
             wf_name = wfh.name