From 60b6c47b403fc7db4147a4cb7da13bc12e24ddfe Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 19 Jun 2022 15:51:32 +0200 Subject: [PATCH] Wait for wait --- lib/galaxy/datatypes/text.py | 4 +-- lib/galaxy/tool_util/deps/conda_util.py | 4 +-- .../tool_util/deps/mulled/mulled_build.py | 4 +-- lib/galaxy/util/__init__.py | 7 +++++ lib/galaxy/util/path/__init__.py | 2 +- test/integration/test_kubernetes_runner.py | 28 +++++++++++++------ 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 812be9198ac..7fd68d82444 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -6,7 +6,6 @@ import json import logging import os import re -import shlex import subprocess import tempfile @@ -28,6 +27,7 @@ from galaxy.datatypes.sniff import ( ) from galaxy.util import ( nice_size, + shlex_join, string_as_bool, unicodify, ) @@ -219,7 +219,7 @@ class Ipynb(Json): ofilename = dataset.file_name log.exception( 'Command "%s" failed. Could not convert the Jupyter Notebook to HTML, defaulting to plain text.', - " ".join(map(shlex.quote, cmd)), + shlex_join(cmd), ) return open(ofilename, mode="rb"), headers diff --git a/lib/galaxy/tool_util/deps/conda_util.py b/lib/galaxy/tool_util/deps/conda_util.py index 0fca80f8832..4b3dcb77731 100644 --- a/lib/galaxy/tool_util/deps/conda_util.py +++ b/lib/galaxy/tool_util/deps/conda_util.py @@ -4,7 +4,6 @@ import json import logging import os import re -import shlex import shutil import sys import tempfile @@ -18,6 +17,7 @@ import packaging.version from galaxy.util import ( commands, listify, + shlex_join, smart_str, which, ) @@ -237,7 +237,7 @@ class CondaContext(installable.InstallableContext): env = {} if self.condarc_override: env["CONDARC"] = self.condarc_override - cmd_string = " ".join(map(shlex.quote, cmd)) + cmd_string = shlex_join(cmd) kwds = dict() try: if stdout_path: diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index ba0b5fa022d..beb8b693f1f 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -12,7 +12,6 @@ Build a mulled image with: import json import logging import os -import shlex import shutil import stat import string @@ -31,6 +30,7 @@ from galaxy.tool_util.deps.docker_util import command_list as docker_command_lis from galaxy.util import ( commands, safe_makedirs, + shlex_join, unicodify, ) from ._cli import arg_parser @@ -301,7 +301,7 @@ def mull_targets( involucro_args.insert(6, "-set") involucro_args.insert(7, f"TEST_BINDS={','.join(test_bind)}") cmd = involucro_context.build_command(involucro_args) - print(f"Executing: {' '.join(shlex.quote(_) for _ in cmd)}") + print(f"Executing: {shlex_join(cmd)}") if dry_run: return 0 ensure_installed(involucro_context, True) diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 190abb575d8..401942f9b29 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -13,6 +13,7 @@ import json import os import random import re +import shlex import shutil import smtplib import stat @@ -77,6 +78,12 @@ from .path import ( # noqa: F401 safe_relpath, ) +try: + shlex_join = shlex.join # type: ignore[attr-defined] +except AttributeError: + # Python < 3.8 + shlex_join = lambda split_command: " ".join(map(shlex.quote, split_command)) # noqa: E731 + inflector = Inflector() log = get_logger(__name__) diff --git a/lib/galaxy/util/path/__init__.py b/lib/galaxy/util/path/__init__.py index 8842807931b..89bf02b5cc6 100644 --- a/lib/galaxy/util/path/__init__.py +++ b/lib/galaxy/util/path/__init__.py @@ -365,7 +365,7 @@ def external_chown(path, pwent, external_chown_script, description="file"): cmd = shlex.split(external_chown_script) cmd.extend([path, pwent[0], str(pwent[3])]) - log.debug(f"Changing ownership of {path} with: {' '.join(map(shlex.quote, cmd))}") + log.debug(f"Changing ownership of {path} with: '{galaxy.util.shlex_join(cmd)}'") galaxy.util.commands.execute(cmd) return True except galaxy.util.commands.CommandLineException as e: diff --git a/test/integration/test_kubernetes_runner.py b/test/integration/test_kubernetes_runner.py index bbf9ca9f9d1..d1b90b87cff 100644 --- a/test/integration/test_kubernetes_runner.py +++ b/test/integration/test_kubernetes_runner.py @@ -11,10 +11,14 @@ import time import pytest -from galaxy.util import unicodify +from galaxy.util import ( + shlex_join, + unicodify, +) from galaxy_test.base.populators import ( DatasetPopulator, skip_without_tool, + wait_on, ) from galaxy_test.driver import integration_util from .test_containerized_jobs import MulledJobTestCases @@ -323,13 +327,21 @@ class BaseKubernetesIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase, M self._wait_for_external_state(sa_session=sa_session, job=job, expected=app.model.Job.states.RUNNING) external_id = job.job_runner_external_id - log_cmd = ["kubectl", "logs", "-l", f"job-name={external_id}"] - p = subprocess.run(log_cmd, capture_output=True, text=True) - if p.returncode: - raise Exception( - f"Command '{' '.join(log_cmd)}' failed with exit code: {p.returncode}.\nstdout: {p.stdout}\nstderr: {p.stderr}" - ) - output = p.stdout + + def get_kubectl_logs(allow_wait=True): + log_cmd = ["kubectl", "logs", "-l", f"job-name={external_id}"] + p = subprocess.run(log_cmd, capture_output=True, text=True) + if p.returncode: + if allow_wait and "is waiting to start" in p.stderr: + return None + raise Exception( + f"Command '{shlex_join}' failed with exit code: {p.returncode}.\nstdout: {p.stdout}\nstderr: {p.stderr}" + ) + return p.stdout + + wait_on(get_kubectl_logs, "k8s logs") + output = get_kubectl_logs(allow_wait=False) + EXPECTED_STDOUT = "The bool is not true" EXPECTED_STDERR = "The bool is very not true" assert EXPECTED_STDOUT in output