More Pulsar fixes.

- Fix working directory outputs for a2e3140375 (giving tools clean working directories).
 - Fix copy action when target directory doesn't already exist (https://github.com/galaxyproject/pulsar/pull/107 thanks to @natefoo).
 - Fix handling of Pulsar jobs that fail during submission (https://github.com/galaxyproject/pulsar/pull/103 thanks to @mvdbeek).
 - Fix PBS status reporting (https://github.com/galaxyproject/pulsar/pull/102 thanks to @mvdbeek).
 - Fix for JetStream that allows setting staging directories on a per-manager basis (https://github.com/galaxyproject/pulsar/commit/7d4030412d1aa58d0c3daa86d0ad7113c79fee81).

lib/galaxy/dependencies/pinned-requirements.txt

@@ -43,7 +43,7 @@ anyjson==0.3.3
 
 # Pulsar requirements
 psutil==4.1.0
-pulsar-galaxy-lib==0.7.0.dev3
+pulsar-galaxy-lib==0.7.0.dev4
 
 # sqlalchemy-migrate and dependencies
 sqlalchemy-migrate==0.10.0

lib/galaxy/dependencies/requirements.txt

@@ -41,7 +41,7 @@ kombu
 
 # Pulsar requirements
 psutil
-pulsar-galaxy-lib==0.7.0.dev3
+pulsar-galaxy-lib>=0.7.0.dev4
 
 # sqlalchemy-migrate and dependencies
 sqlalchemy-migrate

lib/galaxy/jobs/command_factory.py

@@ -22,6 +22,7 @@ def build_command(
     container=None,
     include_metadata=False,
     include_work_dir_outputs=True,
+    create_tool_working_directory=True,
     remote_command_params={},
     metadata_directory=None,
 ):
@@ -72,9 +73,12 @@ def build_command(
         else:
             commands_builder = CommandsBuilder( externalized_commands )
 
-    # usually working will already exist, but it will not for task
-    # split jobs.
-    commands_builder.prepend_command("mkdir -p working; cd working")
+    # Don't need to create a separate tool working directory for Pulsar
+    # jobs - that is handled by Pulsar.
+    if create_tool_working_directory:
+        # usually working will already exist, but it will not for task
+        # split jobs.
+        commands_builder.prepend_command("mkdir -p working; cd working")
 
     if include_work_dir_outputs:
         __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params)

lib/galaxy/jobs/runners/pulsar.py

@@ -337,12 +337,15 @@ class PulsarJobRunner( AsynchronousJobRunner ):
             if remote_metadata:
                 metadata_directory = remote_job_config['metadata_directory']
 
+            # Pulsar handles ``create_tool_working_directory`` and
+            # ``include_work_dir_outputs`` details.
             command_line = build_command(
                 self,
                 job_wrapper=job_wrapper,
                 container=container,
                 include_metadata=remote_metadata,
                 metadata_directory=metadata_directory,
+                create_tool_working_directory=False,
                 include_work_dir_outputs=False,
                 remote_command_params=remote_command_params,
             )

test/integration/test_pulsar_embedded.py

@@ -22,3 +22,6 @@ class EmbeddedPulsarIntegrationTestCase(integration_util.IntegrationTestCase):
 
     def test_multi_data_param(self):
         self._run_tool_test("multi_data_param")
+
+    def test_work_dir_outputs(self):
+        self._run_tool_test("output_filter")