From 4a7708b7e0702fadd64873e1e912b2abe32a57d0 Mon Sep 17 00:00:00 2001
From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com>
Date: Mon, 3 Jun 2019 14:36:44 +0530
Subject: [PATCH] Fix wording for pod retries while maintaining backward
 compatibility

---
 config/job_conf.xml.sample_advanced   |  4 ++--
 lib/galaxy/jobs/runners/kubernetes.py | 23 +++++++++++++++--------
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced
index c7fbea173e7..e1e72044eb6 100644
--- a/config/job_conf.xml.sample_advanced
+++ b/config/job_conf.xml.sample_advanced
@@ -950,12 +950,12 @@
                 <param id="docker_image_default">my-tool</param>
                 <param id="docker_tag_default">latest</param>
             -->
-            <param id="max_pod_retrials">3</param>
+            <param id="max_pod_retries">3</param>
             <!-- Allows pods to retry up to this number of times, before marking the galaxy job failed. k8s is a state
                  setter essentially, so by default it will try to take a job submitted to successful completion. A job
                  submits pods, until the number of successes (1 in this use case) is achieved, assuming that whatever is
                  making the pods fail will be fixed (such as a stale disk or a dead node that it is being restarted).
-                 This option sets a limit of retrials, so that after that number of failed pods, the job is re-scaled to
+                 This option sets a limit of retries, so that after that number of failed pods, the job is re-scaled to
                  zero (no execution) and the stderr/stdout of the k8s job is reported in galaxy (and the galaxy job set
                  to failed).
 
diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py
index 3594e044cb7..782c4a3dc29 100644
--- a/lib/galaxy/jobs/runners/kubernetes.py
+++ b/lib/galaxy/jobs/runners/kubernetes.py
@@ -59,6 +59,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             k8s_default_requests_memory=dict(map=str, default=None),
             k8s_default_limits_cpu=dict(map=str, default=None),
             k8s_default_limits_memory=dict(map=str, default=None),
+            k8s_pod_retries=dict(map=int, valid=lambda x: int >= 0, default=3),
             k8s_pod_retrials=dict(map=int, valid=lambda x: int >= 0, default=3))
 
         if 'runner_param_specs' not in kwargs:
@@ -396,11 +397,17 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             active = 0
             failed = 0
 
-            max_pod_retrials = 1
-            if 'k8s_pod_retrials' in self.runner_params:
-                max_pod_retrials = int(self.runner_params['k8s_pod_retrials'])
-            if 'max_pod_retrials' in job_destination.params:
-                max_pod_retrials = int(job_destination.params['max_pod_retrials'])
+            max_pod_retries = 1
+            if 'max_pod_retries' in job_destination.params:
+                max_pod_retries = int(job_destination.params['max_pod_retries'])
+            elif 'k8s_pod_retries' in self.runner_params:
+                max_pod_retries = int(self.runner_params['k8s_pod_retries'])
+            elif 'max_pod_retrials' in job_destination.params:
+                # For backward compatibility
+                max_pod_retries = int(job_destination.params['max_pod_retrials'])
+            elif 'k8s_pod_retrials' in self.runner_params:
+                # For backward compatibility
+                max_pod_retries = int(self.runner_params['max_pod_retrials'])
 
             if 'succeeded' in job.obj['status']:
                 succeeded = job.obj['status']['succeeded']
@@ -416,12 +423,12 @@ class KubernetesJobRunner(AsynchronousJobRunner):
                 return None
             elif failed > 0 and self.__job_failed_due_to_low_memory(job_state):
                 return self._handle_job_failure(job, job_state, reason="OOM")
-            elif active > 0 and failed <= max_pod_retrials:
+            elif active > 0 and failed <= max_pod_retries:
                 if not job_state.running:
                     job_state.running = True
                     job_state.job_wrapper.change_state(model.Job.states.RUNNING)
                 return job_state
-            elif failed > max_pod_retrials:
+            elif failed > max_pod_retries:
                 return self._handle_job_failure(job, job_state)
             elif job_state.job_wrapper.get_job().state == model.Job.states.DELETED:
                 # Job has been deleted via stop_job, cleanup and remove from watched_jobs by returning `None`
@@ -443,7 +450,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             # there is more than one job associated to the expected unique job id used as selector.
             log.error("More than one Kubernetes Job associated to job id '%s'", job_state.job_id)
             with open(job_state.error_file, 'w') as error_file:
-                error_file.write("More than one Kubernetes Job associated to job id '%s'\n" % job_state.job_id)
+                error_file.write("More than one Kubernetes Job associated with job id '%s'\n" % job_state.job_id)
             self.mark_as_failed(job_state)
             return job_state