From c421f29d851c8ebdbd29d87fe378ef06a78aaecf Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 28 Sep 2017 08:07:55 +0000 Subject: [PATCH 1/5] Log more information on command execution So we can tell *why* it failed on not just that it failed. --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 2c717c8f13d..4204d9a17e2 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -1,3 +1,4 @@ +import logging import os import subprocess import time @@ -8,6 +9,7 @@ from six import text_type from galaxy.util import unicodify +log = logging.getLogger(__name__) DEFAULT_SHELL = '/bin/bash' DEFAULT_JOB_FILE_TEMPLATE = Template( @@ -131,11 +133,14 @@ def _handle_script_integrity(path, config): # These have occurred both in Docker containers and on EC2 clusters # under high load. subprocess.check_call(INTEGRITY_SYNC_COMMAND) - except Exception: - pass + except Exception as e: + log.debug("Error syncing the filesystem %s", e) + time.sleep(sleep_amt) - except Exception: - pass + except OSError as ose: + log.debug("Script not available yet %s", ose) + except Exception as exc: + log.debug("Script not available yet %s", exc) if not script_integrity_verified: raise Exception("Failed to write job script, could not verify job script integrity.") From 955206b05208f920f0db60473c90a0e20ca6e181 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 28 Sep 2017 08:08:53 +0000 Subject: [PATCH 2/5] Another debugging statement --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 4204d9a17e2..070292a715e 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -126,6 +126,8 @@ def _handle_script_integrity(path, config): script_integrity_verified = True break + log.debug("Script integrity error: returncode was %d", returncode) + # Else we will sync and wait to see if the script becomes # executable. try: From 85533747050494cd5c2ddb749506b75e40e6bff0 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 28 Sep 2017 08:11:01 +0000 Subject: [PATCH 3/5] Unicodify anything logged due to the french --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 070292a715e..7469a7d6a53 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -136,13 +136,13 @@ def _handle_script_integrity(path, config): # under high load. subprocess.check_call(INTEGRITY_SYNC_COMMAND) except Exception as e: - log.debug("Error syncing the filesystem %s", e) + log.debug("Error syncing the filesystem %s", unicodify(e)) time.sleep(sleep_amt) except OSError as ose: - log.debug("Script not available yet %s", ose) + log.debug("Script not available yet %s", unicodify(ose)) except Exception as exc: - log.debug("Script not available yet %s", exc) + log.debug("Script not available yet %s", unicodify(exc)) if not script_integrity_verified: raise Exception("Failed to write job script, could not verify job script integrity.") From 7e7369c168991302f4a3b434cc25d8da934f5506 Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 28 Sep 2017 11:52:04 +0200 Subject: [PATCH 4/5] collapse logging + : --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 7469a7d6a53..10741d43e61 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -136,13 +136,11 @@ def _handle_script_integrity(path, config): # under high load. subprocess.check_call(INTEGRITY_SYNC_COMMAND) except Exception as e: - log.debug("Error syncing the filesystem %s", unicodify(e)) + log.debug("Error syncing the filesystem: %s", unicodify(e)) time.sleep(sleep_amt) - except OSError as ose: - log.debug("Script not available yet %s", unicodify(ose)) except Exception as exc: - log.debug("Script not available yet %s", unicodify(exc)) + log.debug("Script not available yet: %s", unicodify(exc)) if not script_integrity_verified: raise Exception("Failed to write job script, could not verify job script integrity.") From 08b90d699d72358c89104643f1859869c5d4dffc Mon Sep 17 00:00:00 2001 From: E Rasche Date: Thu, 28 Sep 2017 14:27:01 +0200 Subject: [PATCH 5/5] Move sleep --- lib/galaxy/jobs/runners/util/job_script/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 10741d43e61..20aae2fe86f 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -138,10 +138,11 @@ def _handle_script_integrity(path, config): except Exception as e: log.debug("Error syncing the filesystem: %s", unicodify(e)) - time.sleep(sleep_amt) except Exception as exc: log.debug("Script not available yet: %s", unicodify(exc)) + time.sleep(sleep_amt) + if not script_integrity_verified: raise Exception("Failed to write job script, could not verify job script integrity.")