Merge pull request #7766 from natefoo/slurmstepd

[19.01] Catch and filter *all* Slurm cgroup-related messages
This commit is contained in:
Nicola Soranzo
2019-05-07 15:57:31 +01:00
committed by GitHub
+2 -6
View File
@@ -28,13 +28,9 @@ SLURM_MEMORY_LIMIT_EXCEEDED_MSG = 'slurmstepd: error: Exceeded job memory limit'
SLURM_MEMORY_LIMIT_EXCEEDED_PARTIAL_WARNINGS = [': Exceeded job memory limit at some point.',
': Exceeded step memory limit at some point.']
SLURM_MEMORY_LIMIT_SCAN_SIZE = 16 * 1024 * 1024 # 16MB
SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE = re.compile(r"""slurmstepd: error: task/cgroup: unable to add task\[pid=\d+\] to memory cg '\(null\)'$""")
SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: xcgroup_instantiate: unable to create cgroup '[^']+' : No space left on device$""")
SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE = re.compile(r"""slurmstepd: error: jobacct_gather/cgroup: unable to instanciate (job|user) \d+ memory cgroup$""")
SLURM_CGROUP_RE = re.compile(r"""slurmstepd: .*cgroup.*$""")
SLURM_TOP_WARNING_RES = (
SLURM_UNABLE_TO_ADD_TASK_TO_MEMORY_CG_MSG_RE,
SLURM_UNABLE_TO_CREATE_CGROUP_MSG_RE,
SLURM_UNABLE_TO_INSTANCIATE_CGROUP_MSG_RE
SLURM_CGROUP_RE,
)
# These messages are returned to the user