Merge pull request #16696 from natefoo/clean-working

Run the tool working dir backup/restore on Pulsar

lib/galaxy/jobs/command_factory.py

@@ -34,13 +34,6 @@ CAPTURE_RETURN_CODE = "return_code=$?"
 YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"'
 SETUP_GALAXY_FOR_METADATA = """
 [ "$GALAXY_VIRTUAL_ENV" = "None" ] && GALAXY_VIRTUAL_ENV="$_GALAXY_VIRTUAL_ENV"; _galaxy_setup_environment True"""
-PREPARE_DIRS = """mkdir -p working outputs configs
-if [ -d _working ]; then
-    rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
-else
-    cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
-fi
-cd working"""
 
 
 def build_command(
@@ -126,12 +119,9 @@ def build_command(
     # Don't need to create a separate tool working directory for Pulsar
     # jobs - that is handled by Pulsar.
     if create_tool_working_directory:
-        # usually working will already exist, but it will not for task
-        # split jobs.
-
-        # Copy working and outputs before job submission so that these can be restored on resubmission
-        # xref https://github.com/galaxyproject/galaxy/issues/3289
-        commands_builder.prepend_command(PREPARE_DIRS)
+        # Working (and outputs, configs) are backed up and restored in the job script for both Galaxy and Pulsar jobs,
+        # but Pulsar automatically changes into the working dir, whereas Galaxy does not.
+        commands_builder.prepend_command("cd working")
 
     __handle_remote_command_line_building(commands_builder, job_wrapper, for_pulsar=for_pulsar)
 

lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh

@@ -58,6 +58,7 @@ export TEMP
 export TMPDIR
 
 GALAXY_PYTHON=`command -v python`
+$prepare_dirs_statement
 cd $working_directory
 $memory_statement
 $instrument_pre_commands

lib/galaxy/jobs/runners/util/job_script/__init__.py

@@ -41,6 +41,16 @@ if [ -n "$ABC_TEST_JOB_SCRIPT_INTEGRITY_XYZ" ]; then
 fi
 """
 
+# Copy working, outputs, and configs before tool execution so that these can be restored on job resubmission
+# xref https://github.com/galaxyproject/galaxy/issues/3289
+PREPARE_DIRS = """mkdir -p working outputs configs
+if [ -d _working ]; then
+    rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+    cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+"""
+
 INTEGRITY_SYNC_COMMAND = "/bin/sync"
 DEFAULT_INTEGRITY_CHECK = True
 DEFAULT_INTEGRITY_COUNT = 35
@@ -58,6 +68,7 @@ OPTIONAL_TEMPLATE_PARAMS: Dict[str, Any] = {
     "shell": DEFAULT_SHELL,
     "preserve_python_environment": True,
     "tmp_dir_creation_statement": '""',
+    "prepare_dirs_statement": PREPARE_DIRS,
 }
 
 

test/unit/app/jobs/test_command_factory.py

@@ -9,7 +9,6 @@ from typing import (
 
 from galaxy.jobs.command_factory import (
     build_command,
-    PREPARE_DIRS,
     SETUP_GALAXY_FOR_METADATA,
 )
 from galaxy.model import Dataset
@@ -186,7 +185,7 @@ class TestCommandFactory(TestCase):
         return build_command(**kwds)
 
     def _surround_command(self, command, post_command=""):
-        command = f'''{PREPARE_DIRS};{self.TEE_LOG}{command} {self.CAPTURE_AND_REDIRECT}{post_command}; sh -c "exit $return_code"'''
+        command = f'''cd working;{self.TEE_LOG}{command} {self.CAPTURE_AND_REDIRECT}{post_command}; sh -c "exit $return_code"'''
         return command.replace("galaxy_1.ec", os.path.join(self.job_wrapper.working_directory, "galaxy_1.ec"), 1)