From 4e35ff04a116a4a9ca1d96e9b3c5842be9fa9f7f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 5 Oct 2015 09:34:12 +0100 Subject: [PATCH 1/2] More improved file handling for job code. Another stab in the dark that won't actually fix the testing issues with Docker. --- lib/galaxy/jobs/command_factory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 92044dfa8aa..927143d1660 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -73,9 +73,9 @@ def build_command( def __externalize_commands(job_wrapper, commands_builder, remote_command_params, script_name="tool_script.sh"): local_container_script = join( job_wrapper.working_directory, script_name ) - fh = file( local_container_script, "w" ) - fh.write( "#!%s\n%s" % (DEFAULT_SHELL, commands_builder.build())) - fh.close() + with open( local_container_script, "w" ) as f: + script_contents = "#!%s\n%s" % (DEFAULT_SHELL, commands_builder.build()) + f.write( script_contents ) chmod( local_container_script, 0755 ) commands = local_container_script From b75175f18d15000a2dc5826f49424eba6dc7d489 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 5 Oct 2015 11:00:46 +0100 Subject: [PATCH 2/2] Centralize logic for writing job scripts. Runners were using various permissions, some didn't close file handles, etc.... In addition, run sync after script is written to avoid "Text file busy" problems in all job script scenerios. --- lib/galaxy/jobs/runners/__init__.py | 12 ++++++++++++ lib/galaxy/jobs/runners/cli.py | 4 +--- lib/galaxy/jobs/runners/condor.py | 5 +---- lib/galaxy/jobs/runners/drmaa.py | 5 +---- lib/galaxy/jobs/runners/local.py | 4 +--- lib/galaxy/jobs/runners/pbs.py | 5 +---- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index dcc2a31d345..6fe41458530 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -311,6 +311,18 @@ class BaseJobRunner( object ): options.update(**kwds) return job_script(**options) + def write_executable_script( self, path, contents, mode=0o755 ): + with open( path, 'w' ) as f: + f.write( contents ) + os.chmod( path, mode ) + try: + # sync file system to avoid "Text file busy" problems. + # These have occurred both in Docker containers and on EC2 clusters + # under high load. + subprocess.check_call(["sync"]) + except Exception: + pass + def _complete_terminal_job( self, ajs, **kwargs ): if ajs.job_wrapper.get_state() != model.Job.states.DELETED: self.work_queue.put( ( self.finish_job, ajs ) ) diff --git a/lib/galaxy/jobs/runners/cli.py b/lib/galaxy/jobs/runners/cli.py index 795202e8c30..7d8b5477957 100644 --- a/lib/galaxy/jobs/runners/cli.py +++ b/lib/galaxy/jobs/runners/cli.py @@ -72,9 +72,7 @@ class ShellJobRunner( AsynchronousJobRunner ): ) try: - fh = file(ajs.job_file, "w") - fh.write(script) - fh.close() + self.write_executable_script( ajs.job_file, script ) except: log.exception("(%s) failure writing job script" % galaxy_id_tag ) job_wrapper.fail("failure preparing job script", exception=True) diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py index 6ae20f94396..4b14bbb5e9f 100644 --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -90,10 +90,7 @@ class CondorJobRunner( AsynchronousJobRunner ): slots_statement=galaxy_slots_statement, ) try: - fh = file( executable, "w" ) - fh.write( script ) - fh.close() - os.chmod( executable, 0o750 ) + self.write_executable_script( executable.job_file, script ) except: job_wrapper.fail( "failure preparing job script", exception=True ) log.exception( "(%s) failure preparing job script" % galaxy_id_tag ) diff --git a/lib/galaxy/jobs/runners/drmaa.py b/lib/galaxy/jobs/runners/drmaa.py index 88727620838..944cd55f969 100644 --- a/lib/galaxy/jobs/runners/drmaa.py +++ b/lib/galaxy/jobs/runners/drmaa.py @@ -147,10 +147,7 @@ class DRMAAJobRunner( AsynchronousJobRunner ): # fill in the DRM's job run template script = self.get_job_file(job_wrapper, exit_code_path=ajs.exit_code_file) try: - fh = file( ajs.job_file, "w" ) - fh.write( script ) - fh.close() - os.chmod( ajs.job_file, 0o755 ) + self.write_executable_script( ajs.job_file, script ) except: job_wrapper.fail( "failure preparing job script", exception=True ) log.exception( "(%s) failure writing job script" % galaxy_id_tag ) diff --git a/lib/galaxy/jobs/runners/local.py b/lib/galaxy/jobs/runners/local.py index 40cbbfa9f05..ecd5038264b 100644 --- a/lib/galaxy/jobs/runners/local.py +++ b/lib/galaxy/jobs/runners/local.py @@ -68,9 +68,7 @@ class LocalJobRunner( BaseJobRunner ): 'working_directory': job_wrapper.working_directory, } job_file_contents = self.get_job_file( job_wrapper, **job_script_props ) - with open( job_file, 'w' ) as f: - f.write( job_file_contents ) - os.chmod( job_file, 0o755 ) + self.write_executable_script( job_file, job_file_contents ) return job_file, exit_code_path def queue_job( self, job_wrapper ): diff --git a/lib/galaxy/jobs/runners/pbs.py b/lib/galaxy/jobs/runners/pbs.py index b93b4156b90..c01809de3bb 100644 --- a/lib/galaxy/jobs/runners/pbs.py +++ b/lib/galaxy/jobs/runners/pbs.py @@ -296,10 +296,7 @@ class PBSJobRunner( AsynchronousJobRunner ): env_setup_commands = [ stage_commands ] script = self.get_job_file(job_wrapper, exit_code_path=ecfile, env_setup_commands=env_setup_commands) job_file = "%s/%s.sh" % (self.app.config.cluster_files_directory, job_wrapper.job_id) - fh = file(job_file, "w") - fh.write(script) - fh.close() - + self.write_executable_script( job_file, script ) # job was deleted while we were preparing it if job_wrapper.get_state() == model.Job.states.DELETED: log.debug( "Job %s deleted by user before it entered the PBS queue" % job_wrapper.job_id )