From 22a565d1a25a5eda466d49b69df3c3ab436e8b82 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Wed, 7 Oct 2015 12:50:34 +0100
Subject: [PATCH] Try harder to verify script integrity before running job
 scripts.

These "Text file busy" errors are occasionally encountered in the wild in production settings and more commonly when running Galaxy in Docker containers.

Simply sycning the file system has proven insufficient to prevent the problem and I do not want to add an arbitrary sleep. So this new strategy is used.

 - Inject special functionality into each job script that when executed with a certain environment variable set causes the script to exit with a specific return code.
 - Have Galaxy rerun this script in that configuration until it can succeed in executing the script.
 - In between each iteration sleep for a small amount of time and try to cause a file system sync.

This entire is ... insane ... so I have added a configuration option to disable it entirely check_job_script_integrity=False.
---
 config/galaxy.ini.sample                      |  4 ++
 lib/galaxy/config.py                          |  1 +
 lib/galaxy/jobs/runners/__init__.py           | 37 +++++++++++++++----
 .../job_script/DEFAULT_JOB_FILE_TEMPLATE.sh   |  8 ++++
 4 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index ffd41aa8a0d..f6fb9e4ef55 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -262,6 +262,10 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # used for the cache
 #template_cache_path = database/compiled_templates
 
+# Set to false to disable various checks Galaxy will do to ensure it
+# can run job scripts before attempting to execute or submit them.
+#check_job_script_integrity = True
+
 # Citation related caching.  Tool citations information maybe fetched from
 # external sources such as http://dx.doi.org/ by Galaxy - the following
 # parameters can be used to control the caching used to store this information.
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index 4ccc6beaa8c..2a9dc3f73f5 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -172,6 +172,7 @@ class Configuration( object ):
         self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
         self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )
         self.retry_job_output_collection = int( kwargs.get( 'retry_job_output_collection', 0 ) )
+        self.check_job_script_integrity = string_as_bool( kwargs.get( "check_job_script_integrity", True ) )
         self.job_walltime = kwargs.get( 'job_walltime', None )
         self.job_walltime_delta = None
         if self.job_walltime is not None:
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index 8c507bdbd00..0b2c0a5099a 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -313,13 +313,36 @@ class BaseJobRunner( object ):
         with open( path, 'w' ) as f:
             f.write( contents )
         os.chmod( path, mode )
-        try:
-            # sync file system to avoid "Text file busy" problems.
-            # These have occurred both in Docker containers and on EC2 clusters
-            # under high load.
-            subprocess.check_call(["sync"])
-        except Exception:
-            pass
+        self._handle_script_integrity( path )
+
+    def _handle_script_integrity( self, path ):
+        if not getattr( self.app.config, "check_job_script_integrity", True ):
+            return
+
+        script_integrity_verified = False
+        for i in range(5):
+            try:
+                proc = subprocess.Popen( [path], shell=True, env={"ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ": "1"} )
+                proc.wait()
+                if proc.returncode == 42:
+                    script_integrity_verified = True
+                    break
+
+                # Else we will sync and wait to see if the script becomes
+                # executable.
+                try:
+                    # sync file system to avoid "Text file busy" problems.
+                    # These have occurred both in Docker containers and on EC2 clusters
+                    # under high load.
+                    subprocess.check_call(["/bin/sync"])
+                except Exception:
+                    pass
+                time.sleep( .1 )
+            except Exception:
+                pass
+
+        if not script_integrity_verified:
+            raise Exception("Failed to write job script, could not verify job script integrity.")
 
     def _complete_terminal_job( self, ajs, **kwargs ):
         if ajs.job_wrapper.get_state() != model.Job.states.DELETED:
diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh
index 28af9118358..393f1d55557 100644
--- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh
+++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh
@@ -1,4 +1,12 @@
 #!/bin/sh
+
+# The following block can be used by the job creation system
+# to ensure this script is runnable before running it directly
+# or submitting it to a cluster manager.
+if [ ! -z "$ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ" ]; then
+    exit 42
+fi
+
 $headers
 $slots_statement
 export GALAXY_SLOTS