opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8082 from nuwang/fix_k8s_runner_wording

Browse Source 
Fix wording for pod retries while maintaining backward compatibility
This commit is contained in:
John Chilton
2019-06-03 09:56:38 -04:00
committed by GitHub
parent 84063a2bfc 4a7708b7e0
commit 6f88c4455f
2 changed files with 17 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/job_conf.xml.sample_advanced
+2 -2
View File
@@ -950,12 +950,12 @@
<param id="docker_image_default">my-tool</param>
<param id="docker_tag_default">latest</param>
-->
<param id="max_pod_retrials">3</param>
<param id="max_pod_retries">3</param>
<!-- Allows pods to retry up to this number of times, before marking the galaxy job failed. k8s is a state
setter essentially, so by default it will try to take a job submitted to successful completion. A job
submits pods, until the number of successes (1 in this use case) is achieved, assuming that whatever is
making the pods fail will be fixed (such as a stale disk or a dead node that it is being restarted).
This option sets a limit of retrials, so that after that number of failed pods, the job is re-scaled to
This option sets a limit of retries, so that after that number of failed pods, the job is re-scaled to
zero (no execution) and the stderr/stdout of the k8s job is reported in galaxy (and the galaxy job set
to failed).
lib/galaxy/jobs/runners/kubernetes.py
+15 -8
View File
@@ -59,6 +59,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
k8s_default_requests_memory=dict(map=str, default=None),
k8s_default_limits_cpu=dict(map=str, default=None),
k8s_default_limits_memory=dict(map=str, default=None),
k8s_pod_retries=dict(map=int, valid=lambda x: int >= 0, default=3),
k8s_pod_retrials=dict(map=int, valid=lambda x: int >= 0, default=3))
if 'runner_param_specs' not in kwargs:
@@ -396,11 +397,17 @@ class KubernetesJobRunner(AsynchronousJobRunner):
active = 0
failed = 0
max_pod_retrials = 1
if 'k8s_pod_retrials' in self.runner_params:
max_pod_retrials = int(self.runner_params['k8s_pod_retrials'])
if 'max_pod_retrials' in job_destination.params:
max_pod_retrials = int(job_destination.params['max_pod_retrials'])
max_pod_retries = 1
if 'max_pod_retries' in job_destination.params:
max_pod_retries = int(job_destination.params['max_pod_retries'])
elif 'k8s_pod_retries' in self.runner_params:
max_pod_retries = int(self.runner_params['k8s_pod_retries'])
elif 'max_pod_retrials' in job_destination.params:
# For backward compatibility
max_pod_retries = int(job_destination.params['max_pod_retrials'])
elif 'k8s_pod_retrials' in self.runner_params:
# For backward compatibility
max_pod_retries = int(self.runner_params['max_pod_retrials'])
if 'succeeded' in job.obj['status']:
succeeded = job.obj['status']['succeeded']
@@ -416,12 +423,12 @@ class KubernetesJobRunner(AsynchronousJobRunner):
return None
elif failed > 0 and self.__job_failed_due_to_low_memory(job_state):
return self._handle_job_failure(job, job_state, reason="OOM")
elif active > 0 and failed <= max_pod_retrials:
elif active > 0 and failed <= max_pod_retries:
if not job_state.running:
job_state.running = True
job_state.job_wrapper.change_state(model.Job.states.RUNNING)
return job_state
elif failed > max_pod_retrials:
elif failed > max_pod_retries:
return self._handle_job_failure(job, job_state)
elif job_state.job_wrapper.get_job().state == model.Job.states.DELETED:
# Job has been deleted via stop_job, cleanup and remove from watched_jobs by returning `None`
@@ -443,7 +450,7 @@ class KubernetesJobRunner(AsynchronousJobRunner):
# there is more than one job associated to the expected unique job id used as selector.
log.error("More than one Kubernetes Job associated to job id '%s'", job_state.job_id)
with open(job_state.error_file, 'w') as error_file:
error_file.write("More than one Kubernetes Job associated to job id '%s'\n" % job_state.job_id)
error_file.write("More than one Kubernetes Job associated with job id '%s'\n" % job_state.job_id)
self.mark_as_failed(job_state)
return job_state