fix Popen/py3 related bugs for real user setups

- Popen returns bytes in python3
- strings are needed

analoguous to what is happening in slurm I use unicodify here

lib/galaxy/jobs/__init__.py

@@ -2092,6 +2092,7 @@ class JobWrapper(HasResourceParameters):
             log.debug('(%s) Changing ownership of working directory with: %s' % (job.id, ' '.join(cmd)))
             p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             stdout, stderr = p.communicate()
+            stdout, stderr = unicodify(stdout).strip(), unicodify(stdout).strip()
             if p.returncode != 0:
                 log.error('external script failed.')
                 log.error('stdout was: %s' % stdout)

lib/galaxy/jobs/runners/drmaa.py

@@ -18,7 +18,7 @@ from galaxy.jobs.runners import (
     AsynchronousJobRunner,
     AsynchronousJobState
 )
-from galaxy.util import asbool
+from galaxy.util import asbool, unicodify
 
 drmaa = None
 
@@ -407,7 +407,8 @@ class DRMAAJobRunner(AsynchronousJobRunner):
         log.info("Running command %s" % command)
         p = subprocess.Popen(command,
                              shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        (stdoutdata, stderrdata) = p.communicate()
+        stdoutdata, stderrdata = p.communicate()
+        stdoutdata, stderrdata = unicodify(stdoutdata).strip(), unicodify(stderrdata).strip()
         exitcode = p.returncode
         # os.unlink(jobtemplate_filename)
         if exitcode != 0:

lib/galaxy/jobs/runners/univa.py

@@ -30,7 +30,7 @@ import signal
 import subprocess
 import time
 
-from galaxy import util
+from galaxy.util import unicodify, size_to_bytes
 from galaxy.jobs.runners.drmaa import DRMAAJobRunner
 log = logging.getLogger(__name__)
 
@@ -155,6 +155,7 @@ class UnivaJobRunner(DRMAAJobRunner):
         cmd = ['qstat', '-u', '"*"']
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = p.communicate()
+        stdout, stderr = unicodify(stdout).strip(), unicodify(stderr).strip()
         if p.returncode != 0 or stderr != "":
             log.exception('`%s` returned %d, stderr: %s' % (' '.join(cmd), p.returncode, stderr))
             raise self.drmaa.InternalException()
@@ -198,7 +199,7 @@ class UnivaJobRunner(DRMAAJobRunner):
         while True:
             p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             stdout, stderr = p.communicate()
-            stderr = stderr.strip()
+            stdout, stderr = unicodify(stdout).strip(), unicodify(stderr).strip()
             if p.returncode != 0:
                 if slp <= 32 and "job id {jobid} not found".format(jobid=job_id) in stderr:
                     # log.debug('`%s` returned %s, stderr: %s => retry after %ds' % (' '.join(cmd), p.returncode, stderr, slp))
@@ -243,7 +244,7 @@ class UnivaJobRunner(DRMAAJobRunner):
         # qdel       100     137          user@mail
 
         extinfo["time_wasted"] = _parse_time(qacct["wallclock"])
-        extinfo["memory_wasted"] = util.size_to_bytes(qacct["maxvmem"])
+        extinfo["memory_wasted"] = size_to_bytes(qacct["maxvmem"])
         extinfo["slots"] = int(qacct["slots"])
 
         # deleted_by
@@ -569,7 +570,7 @@ def _parse_native_specs(job_id, native_spec):
     # parse memory
     m = re.search(r"mem=([\d.]+[KGMT]?)[\s,]*", native_spec)
     if m is not None:
-        mem = util.size_to_bytes(m.group(1))
+        mem = size_to_bytes(m.group(1))
         # mem = _parse_mem(m.group(1))
         if mem is None:
             log.error("DRMAAUniva: job {job_id} has unparsable memory native spec {spec}".format(job_id=job_id, spec=native_spec))