From be3e07fb393abbc10c8bfefab29f653c0c9fadb0 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 21 Jan 2020 19:12:59 +0100 Subject: [PATCH] fix Popen/py3 related bugs for real user setups - Popen returns bytes in python3 - strings are needed analoguous to what is happening in slurm I use unicodify here --- lib/galaxy/jobs/__init__.py | 1 + lib/galaxy/jobs/runners/drmaa.py | 5 +++-- lib/galaxy/jobs/runners/univa.py | 9 +++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 28c223102c5..597fca3a0e0 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -2092,6 +2092,7 @@ class JobWrapper(HasResourceParameters): log.debug('(%s) Changing ownership of working directory with: %s' % (job.id, ' '.join(cmd))) p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() + stdout, stderr = unicodify(stdout).strip(), unicodify(stdout).strip() if p.returncode != 0: log.error('external script failed.') log.error('stdout was: %s' % stdout) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 9699aa7920b..5f20d18c049 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -18,7 +18,7 @@ from galaxy.jobs.runners import ( AsynchronousJobRunner, AsynchronousJobState ) -from galaxy.util import asbool +from galaxy.util import asbool, unicodify drmaa = None @@ -407,7 +407,8 @@ class DRMAAJobRunner(AsynchronousJobRunner): log.info("Running command %s" % command) p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdoutdata, stderrdata) = p.communicate() + stdoutdata, stderrdata = p.communicate() + stdoutdata, stderrdata = unicodify(stdoutdata).strip(), unicodify(stderrdata).strip() exitcode = p.returncode # os.unlink(jobtemplate_filename) if exitcode != 0: diff --git a/lib/galaxy/jobs/runners/univa.py b/lib/galaxy/jobs/runners/univa.py index 51aa43b5821..d1353b28fae 100644 --- a/lib/galaxy/jobs/runners/univa.py +++ b/lib/galaxy/jobs/runners/univa.py @@ -30,7 +30,7 @@ import signal import subprocess import time -from galaxy import util +from galaxy.util import unicodify, size_to_bytes from galaxy.jobs.runners.drmaa import DRMAAJobRunner log = logging.getLogger(__name__) @@ -155,6 +155,7 @@ class UnivaJobRunner(DRMAAJobRunner): cmd = ['qstat', '-u', '"*"'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() + stdout, stderr = unicodify(stdout).strip(), unicodify(stderr).strip() if p.returncode != 0 or stderr != "": log.exception('`%s` returned %d, stderr: %s' % (' '.join(cmd), p.returncode, stderr)) raise self.drmaa.InternalException() @@ -198,7 +199,7 @@ class UnivaJobRunner(DRMAAJobRunner): while True: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() - stderr = stderr.strip() + stdout, stderr = unicodify(stdout).strip(), unicodify(stderr).strip() if p.returncode != 0: if slp <= 32 and "job id {jobid} not found".format(jobid=job_id) in stderr: # log.debug('`%s` returned %s, stderr: %s => retry after %ds' % (' '.join(cmd), p.returncode, stderr, slp)) @@ -243,7 +244,7 @@ class UnivaJobRunner(DRMAAJobRunner): # qdel 100 137 user@mail extinfo["time_wasted"] = _parse_time(qacct["wallclock"]) - extinfo["memory_wasted"] = util.size_to_bytes(qacct["maxvmem"]) + extinfo["memory_wasted"] = size_to_bytes(qacct["maxvmem"]) extinfo["slots"] = int(qacct["slots"]) # deleted_by @@ -569,7 +570,7 @@ def _parse_native_specs(job_id, native_spec): # parse memory m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec) if m is not None: - mem = util.size_to_bytes(m.group(1)) + mem = size_to_bytes(m.group(1)) # mem = _parse_mem(m.group(1)) if mem is None: log.error("DRMAAUniva: job {job_id} has unparsable memory native spec {spec}".format(job_id=job_id, spec=native_spec))