From 3b427177fc684c8e6243c8f413bb10bb55c45489 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 28 Sep 2020 01:34:58 +0530 Subject: [PATCH 1/2] Ensure that k8s job ids are unique --- lib/galaxy/jobs/runners/kubernetes.py | 4 ++-- lib/galaxy/jobs/runners/util/pykube_util.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 9fcec554f81..d6f319ba4c3 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -226,7 +226,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): (see pod selector) and an appropriate restart policy.""" k8s_spec_template = { "metadata": { - "labels": {"app": self.__produce_unique_k8s_job_name(ajs.job_wrapper.get_id_tag())} + "labels": {"app": self.__produce_unique_k8s_job_name(ajs.job_wrapper.get_id_tag())[:-5]} }, "spec": { "volumes": self.runner_params['k8s_mountable_volumes'], @@ -526,7 +526,7 @@ class KubernetesJobRunner(AsynchronousJobRunner): """Attempts to delete a dispatched job to the k8s cluster""" job = job_wrapper.get_job() try: - name = self.__produce_unique_k8s_job_name(job.get_id_tag()) + name = job.job_runner_external_id namespace = self.runner_params['k8s_namespace'] job_to_delete = find_job_object_by_name(self._pykube_api, name, namespace) if job_to_delete: diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index f2e06f78c59..a4db31b70f6 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -1,7 +1,9 @@ """Interface layer for pykube library shared between Galaxy and Pulsar.""" import logging import os +import random import re +import string import uuid try: @@ -57,7 +59,9 @@ def produce_unique_k8s_job_name(app_prefix=None, instance_id=None, job_id=None): if instance_id and len(instance_id) > 0: job_name += "%s-" % instance_id - return job_name + job_id + unique_id = ''.join(random.choices(string.ascii_lowercase, k=4)) + + return job_name + job_id + "-" + unique_id def pull_policy(params): From 488baee52a02efd1a4985fe4ad27b7208f399036 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 2 Oct 2020 17:13:42 +0530 Subject: [PATCH 2/2] Use generated uuid instead of galaxy job id for k8s job name Co-authored-by: Marius van den Beek --- lib/galaxy/jobs/runners/util/pykube_util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index a4db31b70f6..b64e98ec519 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -1,9 +1,7 @@ """Interface layer for pykube library shared between Galaxy and Pulsar.""" import logging import os -import random import re -import string import uuid try: @@ -59,9 +57,7 @@ def produce_unique_k8s_job_name(app_prefix=None, instance_id=None, job_id=None): if instance_id and len(instance_id) > 0: job_name += "%s-" % instance_id - unique_id = ''.join(random.choices(string.ascii_lowercase, k=4)) - - return job_name + job_id + "-" + unique_id + return "{}{}-{}".format(job_name, job_id, uuid.uuid4()) def pull_policy(params):