From b476cc4c57ad86c507eba22c88221584a62954a5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 23 Nov 2017 11:50:36 +0100 Subject: [PATCH 1/4] Fail job if job output collection fails We now set a non-zero exit code to actually set a job to failed. I belive this is a reasonable action, and it avoids green, empty datasets with log messages like: ``` galaxy.jobs.runners ERROR 2017-11-14 13:32:06,469 (19020/860246.torque6.curie.fr) Job output not returned from cluster: [Errno 2] No such file or directory: '/data/users/mvandenb/gx 124/tmp_nfs/jwd/019/19020/galaxy_19020.o' galaxy.jobs.runners DEBUG 2017-11-14 13:32:06,491 (19020/860246.torque6.curie.fr) Unable to cleanup /data/users/mvandenb/gx124/tmp_nfs/jwd/019/19020/galaxy_19020.o: [Errno 2] No suc h file or directory: '/data/users/mvandenb/gx124/tmp_nfs/jwd/019/19020/galaxy_19020.o' galaxy.jobs.runners DEBUG 2017-11-14 13:32:06,503 (19020/860246.torque6.curie.fr) Unable to cleanup /data/users/mvandenb/gx124/tmp_nfs/jwd/019/19020/galaxy_19020.e: [Errno 2] No suc h file or directory: '/data/users/mvandenb/gx124/tmp_nfs/jwd/019/19020/galaxy_19020.e' galaxy.jobs.runners DEBUG 2017-11-14 13:32:06,513 (19020/860246.torque6.curie.fr) Unable to cleanup /data/users/mvandenb/gx124/tmp_nfs/jwd/019/19020/galaxy_19020.ec: [Errno 2] No su ch file or directory: '/data/users/mvandenb/gx124/tmp_nfs/jwd/019/19020/galaxy_19020.ec' ``` --- lib/galaxy/jobs/runners/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index b29e6e57f7d..6bbeedd3ffa 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -623,13 +623,14 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner): stdout = '' stderr = 'Job output not returned from cluster' log.error('(%s/%s) %s: %s' % (galaxy_id_tag, external_job_id, stderr, str(e))) + exit_code_str = "123" # Indicate that job probaly failed, given that we couldn't collect the Job output else: time.sleep(1) which_try += 1 try: # This should be an 8-bit exit code, but read ahead anyway: - exit_code_str = open(job_state.exit_code_file, "r").read(32) + exit_code_str = open(job_state.exit_code_file, "r").read(32) if not exit_code_str else exit_code_str except Exception: # By default, the exit code is 0, which typically indicates success. exit_code_str = "0" From 73454e12df8b8fe05aefc8ae120286e976985db8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 27 Nov 2017 17:54:25 +0100 Subject: [PATCH 2/4] Fix undefined exit_code_str problem (thx @nsoranzo) --- lib/galaxy/jobs/runners/__init__.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 6bbeedd3ffa..3bfd146d0c3 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -41,6 +41,7 @@ STOP_SIGNAL = object() JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE = "Invalid job runner parameter for this plugin: %s" JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s' could not be converted to the correct type" JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" +JOB_COLLECTION_FAILED_EXIT_CODE = "123" GALAXY_LIB_ADJUST_TEMPLATE = """GALAXY_LIB="%s"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi;""" GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;""" @@ -613,6 +614,7 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner): # wait for the files to appear which_try = 0 + collect_output_success = True while which_try < self.app.config.retry_job_output_collection + 1: try: stdout = shrink_stream_by_size(open(job_state.output_file, "r"), DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True) @@ -623,17 +625,20 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner): stdout = '' stderr = 'Job output not returned from cluster' log.error('(%s/%s) %s: %s' % (galaxy_id_tag, external_job_id, stderr, str(e))) - exit_code_str = "123" # Indicate that job probaly failed, given that we couldn't collect the Job output + collect_output_success = False else: time.sleep(1) which_try += 1 - try: - # This should be an 8-bit exit code, but read ahead anyway: - exit_code_str = open(job_state.exit_code_file, "r").read(32) if not exit_code_str else exit_code_str - except Exception: - # By default, the exit code is 0, which typically indicates success. - exit_code_str = "0" + if collect_output_success: + try: + # This should be an 8-bit exit code, but read ahead anyway: + exit_code_str = open(job_state.exit_code_file, "r").read(32) + except Exception: + # By default, the exit code is 0, which typically indicates success. + exit_code_str = "0" + else: + exit_code_str = JOB_COLLECTION_FAILED_EXIT_CODE try: # Decode the exit code. If it's bogus, then just use 0. From cacd6eadc5b95686173c9394f635d5a25527f503 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 28 Nov 2017 10:44:05 +0100 Subject: [PATCH 3/4] Mark job wrapper as failed in a more direct way --- lib/galaxy/jobs/runners/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index 3bfd146d0c3..d0e624a9228 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -41,7 +41,6 @@ STOP_SIGNAL = object() JOB_RUNNER_PARAMETER_UNKNOWN_MESSAGE = "Invalid job runner parameter for this plugin: %s" JOB_RUNNER_PARAMETER_MAP_PROBLEM_MESSAGE = "Job runner parameter '%s' value '%s' could not be converted to the correct type" JOB_RUNNER_PARAMETER_VALIDATION_FAILED_MESSAGE = "Job runner parameter %s failed validation" -JOB_COLLECTION_FAILED_EXIT_CODE = "123" GALAXY_LIB_ADJUST_TEMPLATE = """GALAXY_LIB="%s"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi;""" GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;""" @@ -630,15 +629,17 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner): time.sleep(1) which_try += 1 - if collect_output_success: - try: - # This should be an 8-bit exit code, but read ahead anyway: - exit_code_str = open(job_state.exit_code_file, "r").read(32) - except Exception: - # By default, the exit code is 0, which typically indicates success. - exit_code_str = "0" - else: - exit_code_str = JOB_COLLECTION_FAILED_EXIT_CODE + if not collect_output_success: + job_state.fail_message = stderr + self.mark_as_failed(job_state) + return + + try: + # This should be an 8-bit exit code, but read ahead anyway: + exit_code_str = open(job_state.exit_code_file, "r").read(32) + except Exception: + # By default, the exit code is 0, which typically indicates success. + exit_code_str = "0" try: # Decode the exit code. If it's bogus, then just use 0. From 13f6eeecf099316acc586282773316f3858c0549 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 28 Nov 2017 12:50:54 +0100 Subject: [PATCH 4/4] Add new runner state for 'Job output not returned from cluster' This would allow adimns to write resubmit conditions for this specific failure. --- lib/galaxy/jobs/runners/__init__.py | 4 +++- lib/galaxy/jobs/runners/state_handlers/resubmit.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index d0e624a9228..c95919e2ed6 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -414,6 +414,7 @@ class JobState(object): runner_states = Bunch( WALLTIME_REACHED='walltime_reached', MEMORY_LIMIT_REACHED='memory_limit_reached', + JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER='Job output not returned from cluster', UNKNOWN_ERROR='unknown_error', GLOBAL_WALLTIME_REACHED='global_walltime_reached', OUTPUT_SIZE_LIMIT='output_size_limit' @@ -622,7 +623,7 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner): except Exception as e: if which_try == self.app.config.retry_job_output_collection: stdout = '' - stderr = 'Job output not returned from cluster' + stderr = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER log.error('(%s/%s) %s: %s' % (galaxy_id_tag, external_job_id, stderr, str(e))) collect_output_success = False else: @@ -631,6 +632,7 @@ class AsynchronousJobRunner(Monitors, BaseJobRunner): if not collect_output_success: job_state.fail_message = stderr + job_state.runner_state = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER self.mark_as_failed(job_state) return diff --git a/lib/galaxy/jobs/runners/state_handlers/resubmit.py b/lib/galaxy/jobs/runners/state_handlers/resubmit.py index 6ded2f6eb4f..55c972d6393 100644 --- a/lib/galaxy/jobs/runners/state_handlers/resubmit.py +++ b/lib/galaxy/jobs/runners/state_handlers/resubmit.py @@ -68,6 +68,7 @@ def failure(app, job_runner, job_state): runner_state = getattr(job_state, 'runner_state', None) or JobState.runner_states.UNKNOWN_ERROR if (runner_state not in (JobState.runner_states.WALLTIME_REACHED, JobState.runner_states.MEMORY_LIMIT_REACHED, + JobState.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER, JobState.runner_states.UNKNOWN_ERROR)): # not set or not a handleable runner state return