From 49a7328186bff37fc55725eb4b9af8dbda4ecdb6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 18 Apr 2016 09:16:28 -0400 Subject: [PATCH] More Pulsar fixes. - Fix working directory outputs for a2e31403753c407e7f6966b69b157e32f5f3ef8a (giving tools clean working directories). - Fix copy action when target directory doesn't already exist (https://github.com/galaxyproject/pulsar/pull/107 thanks to @natefoo). - Fix handling of Pulsar jobs that fail during submission (https://github.com/galaxyproject/pulsar/pull/103 thanks to @mvdbeek). - Fix PBS status reporting (https://github.com/galaxyproject/pulsar/pull/102 thanks to @mvdbeek). - Fix for JetStream that allows setting staging directories on a per-manager basis (https://github.com/galaxyproject/pulsar/commit/7d4030412d1aa58d0c3daa86d0ad7113c79fee81). --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- lib/galaxy/dependencies/requirements.txt | 2 +- lib/galaxy/jobs/command_factory.py | 10 +++++++--- lib/galaxy/jobs/runners/pulsar.py | 3 +++ test/integration/test_pulsar_embedded.py | 3 +++ 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 53d879d53f0..c12c5799f01 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -43,7 +43,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev3 +pulsar-galaxy-lib==0.7.0.dev4 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/dependencies/requirements.txt b/lib/galaxy/dependencies/requirements.txt index f4fdee1eb2c..675cbc0964e 100644 --- a/lib/galaxy/dependencies/requirements.txt +++ b/lib/galaxy/dependencies/requirements.txt @@ -41,7 +41,7 @@ kombu # Pulsar requirements psutil -pulsar-galaxy-lib==0.7.0.dev3 +pulsar-galaxy-lib>=0.7.0.dev4 # sqlalchemy-migrate and dependencies sqlalchemy-migrate diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 8c7d408b43b..d57cd77d199 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -22,6 +22,7 @@ def build_command( container=None, include_metadata=False, include_work_dir_outputs=True, + create_tool_working_directory=True, remote_command_params={}, metadata_directory=None, ): @@ -72,9 +73,12 @@ def build_command( else: commands_builder = CommandsBuilder( externalized_commands ) - # usually working will already exist, but it will not for task - # split jobs. - commands_builder.prepend_command("mkdir -p working; cd working") + # Don't need to create a separate tool working directory for Pulsar + # jobs - that is handled by Pulsar. + if create_tool_working_directory: + # usually working will already exist, but it will not for task + # split jobs. + commands_builder.prepend_command("mkdir -p working; cd working") if include_work_dir_outputs: __handle_work_dir_outputs(commands_builder, job_wrapper, runner, remote_command_params) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index bd62b216b72..55ffa7f131e 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -337,12 +337,15 @@ class PulsarJobRunner( AsynchronousJobRunner ): if remote_metadata: metadata_directory = remote_job_config['metadata_directory'] + # Pulsar handles ``create_tool_working_directory`` and + # ``include_work_dir_outputs`` details. command_line = build_command( self, job_wrapper=job_wrapper, container=container, include_metadata=remote_metadata, metadata_directory=metadata_directory, + create_tool_working_directory=False, include_work_dir_outputs=False, remote_command_params=remote_command_params, ) diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py index 43fd7ce8b3c..03998092d61 100644 --- a/test/integration/test_pulsar_embedded.py +++ b/test/integration/test_pulsar_embedded.py @@ -22,3 +22,6 @@ class EmbeddedPulsarIntegrationTestCase(integration_util.IntegrationTestCase): def test_multi_data_param(self): self._run_tool_test("multi_data_param") + + def test_work_dir_outputs(self): + self._run_tool_test("output_filter")