From 2495ceae267da873813d2dc1aae145b686d18672 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 21 Sep 2017 14:02:53 -0400 Subject: [PATCH 01/10] Remove overly chatty debug statement in workflow output cleanup -- this happens pretty often and much of the time is not a problem at all. If we have to track down any new issues with workflow output cleanup, it's easy enough to temporarily add this logging. --- lib/galaxy/jobs/actions/post.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 2f7402f399c..545328bbeb1 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -308,7 +308,6 @@ class DeleteIntermediatesAction(DefaultJobAction): safe_to_delete = True for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]: if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]: - log.debug("Workflow Intermediates cleanup attempted, but non-terminal state '%s' detected for job %s" % (job_to_check.state, job_to_check.id)) safe_to_delete = False if safe_to_delete: # Support purging here too. From 244fc834cd4dd3980cc3edb755d22ca5a9372421 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 11 Sep 2017 12:09:04 -0400 Subject: [PATCH 02/10] Explicitly include PyPI as an --extra-index-url to pip so we can install from PyPI even when a package exists (but the correct version or architecture does not exist) on wheels.galaxyproject.org. Previously, PyPI was only checked after wheels.galaxyproject.org because pypiserver performs an automatic redirect to PyPI for any packages that do not exist in its index. --- scripts/common_startup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 679a88c7a2c..a1e64df09b8 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -139,9 +139,9 @@ if [ $REPLACE_PIP -eq 1 ]; then fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" + pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))") - [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" + [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi fi if [ $FETCH_WHEELS -eq 1 -a $DEV_WHEELS -eq 1 ]; then From 74ad0acb52031355c7efe28b00d1d9b0205174f2 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 14 Sep 2017 23:47:31 -0400 Subject: [PATCH 03/10] Remove repetition in Galaxy env setup in the job script and fix issues with remote metadata on Pulsar. --- .../dependencies/pinned-requirements.txt | 2 +- lib/galaxy/jobs/command_factory.py | 14 +------- lib/galaxy/jobs/runners/pulsar.py | 32 ++++++++++++----- .../job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 35 +++++++++++-------- 4 files changed, 46 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 4c4f7601c3b..1bd08e183cb 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev5 +pulsar-galaxy-lib==0.7.5 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index abf1a0e84de..fb2db1201ac 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -17,19 +17,7 @@ log = getLogger(__name__) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' SETUP_GALAXY_FOR_METADATA = """ -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi -GALAXY_PYTHON=`command -v python` +_galaxy_setup_environment True """ diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..956d8eb88da 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -51,7 +51,10 @@ __all__ = ( 'PulsarEmbeddedJobRunner', ) -MINIMUM_PULSAR_VERSION = LooseVersion("0.7.0.dev3") +MINIMUM_PULSAR_VERSIONS = { + '_default_': LooseVersion("0.7.0.dev3"), + 'remote_metadata': LooseVersion("0.7.5"), +} NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." NO_REMOTE_DATATYPES_CONFIG = "Pulsar client is configured to use remote datatypes configuration when setting metadata externally, but Pulsar is not configured with this information. Defaulting to datatypes_conf.xml." @@ -59,7 +62,7 @@ GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server." FAILED_REMOTE_ERROR = "Remote job server indicated a problem running or monitoring this job." LOST_REMOTE_ERROR = "Remote job server could not determine this job's state." -UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." % MINIMUM_PULSAR_VERSION +UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." # Is there a good way to infer some default for this? Can only use # url_for from web threads. https://gist.github.com/jmchilton/9098762 @@ -310,6 +313,11 @@ class PulsarJobRunner(AsynchronousJobRunner): pulsar_job_state.job_destination = job_destination self.monitor_job(pulsar_job_state) + def __needed_features(self, client): + return { + 'remote_metadata': PulsarJobRunner.__remote_metadata(client), + } + def __prepare_job(self, job_wrapper, job_destination): """Build command-line and Pulsar client for this job.""" command_line = None @@ -319,8 +327,9 @@ class PulsarJobRunner(AsynchronousJobRunner): try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool - remote_job_config = client.setup(tool.id, tool.version) - PulsarJobRunner.check_job_config(remote_job_config) + remote_job_config = client.setup(tool.id, tool.version, tool.requires_galaxy_python_environment) + needed_features = self.__needed_features(client) + PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features) rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client) prepare_kwds = {} if rewrite_parameters: @@ -593,12 +602,17 @@ class PulsarJobRunner(AsynchronousJobRunner): return client_outputs @staticmethod - def check_job_config(remote_job_config): + def check_job_config(remote_job_config, check_features=None): + check_features = check_features or {} # 0.6.0 was newest Pulsar version that did not report it's version. pulsar_version = LooseVersion(remote_job_config.get('pulsar_version', "0.6.0")) + needed_version = LooseVersion("0.0.0") log.info("pulsar_version is %s" % pulsar_version) - if pulsar_version < MINIMUM_PULSAR_VERSION: - raise UnsupportedPulsarException() + for feature in list(check_features.keys()) + ['_default_']: + if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]: + needed_version = max(needed_version, MINIMUM_PULSAR_VERSIONS[feature]) + if pulsar_version < needed_version: + raise UnsupportedPulsarException(needed_version) @staticmethod def __dependencies_description(pulsar_client, job_wrapper): @@ -854,5 +868,5 @@ class PulsarComputeEnvironment(ComputeEnvironment): class UnsupportedPulsarException(Exception): - def __init__(self): - super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR) + def __init__(self, needed): + super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR % needed) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 801df6acbc1..2a83d133b89 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -1,25 +1,32 @@ #!$shell +_galaxy_setup_environment() { + local _use_framework_galaxy="$1" + if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH + fi + $env_setup_commands + if [ "$GALAXY_VIRTUAL_ENV" != "None" -a "$_use_framework_galaxy" = "True" \ + -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ + -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then + . "$GALAXY_VIRTUAL_ENV/bin/activate" + fi +} + $headers $integrity_injection $slots_statement export GALAXY_SLOTS +GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" -if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -$env_setup_commands -GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi +_galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" +GALAXY_PYTHON=`command -v python` $instrument_pre_commands cd $working_directory $command From 276af4db2ab5e81f4af8f35b7d09f48d9fed8dcd Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 15 Sep 2017 16:58:08 -0400 Subject: [PATCH 04/10] Job script enhancements based on feedback. --- lib/galaxy/jobs/command_factory.py | 2 +- .../runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index fb2db1201ac..f8426aeefe6 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -17,7 +17,7 @@ log = getLogger(__name__) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' SETUP_GALAXY_FOR_METADATA = """ -_galaxy_setup_environment True +[ "$GALAXY_VIRTUAL_ENV" = "None" ] && GALAXY_VIRTUAL_ENV="$_GALAXY_VIRTUAL_ENV"; _galaxy_setup_environment True """ diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 2a83d133b89..68da1b3e362 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -1,5 +1,7 @@ #!$shell +$headers + _galaxy_setup_environment() { local _use_framework_galaxy="$1" if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then @@ -11,18 +13,17 @@ _galaxy_setup_environment() { export PYTHONPATH fi $env_setup_commands - if [ "$GALAXY_VIRTUAL_ENV" != "None" -a "$_use_framework_galaxy" = "True" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ + if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate" fi } -$headers $integrity_injection $slots_statement export GALAXY_SLOTS GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" +_GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" _galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" From 53095d688be81a36c31b9721caebf58b383cfe72 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 19:33:02 -0400 Subject: [PATCH 05/10] Fix remote datatypes_conf.xml and registry.xml param rewriting with Pulsar --- lib/galaxy/jobs/__init__.py | 10 ++++++---- lib/galaxy/jobs/command_factory.py | 2 ++ lib/galaxy/jobs/runners/pulsar.py | 2 +- lib/galaxy/model/metadata.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 1e21b45c9db..f6bd6093947 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1648,7 +1648,8 @@ class JobWrapper(object, HasResourceParameters): def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, config_root=None, - config_file=None, resolve_metadata_dependencies=False, + config_file=None, datatypes_config=None, + resolve_metadata_dependencies=False, set_extension=True, **kwds): # extension could still be 'auto' if this is the upload tool. job = self.get_job() @@ -1667,8 +1668,9 @@ class JobWrapper(object, HasResourceParameters): config_root = self.app.config.root if config_file is None: config_file = self.app.config.config_file - datatypes_config = os.path.join(self.working_directory, 'registry.xml') - self.app.datatypes_registry.to_xml_file(path=datatypes_config) + if datatypes_config is None: + datatypes_config = os.path.join(self.working_directory, 'registry.xml') + self.app.datatypes_registry.to_xml_file(path=datatypes_config) command = self.external_output_metadata.setup_external_metadata([output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets + job.output_library_datasets], @@ -1977,7 +1979,7 @@ class TaskWrapper(JobWrapper): pass def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, - config_root=None, config_file=None, + config_root=None, config_file=None, datatypes_config=None, set_extension=True, **kwds): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index f8426aeefe6..7922531ee8b 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -186,6 +186,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para output_fnames = metadata_kwds.get('output_fnames', job_wrapper.get_output_fnames()) config_root = metadata_kwds.get('config_root', None) config_file = metadata_kwds.get('config_file', None) + datatypes_config = metadata_kwds.get('datatypes_config', None) compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None) resolve_metadata_dependencies = job_wrapper.commands_in_new_shell metadata_command = job_wrapper.setup_external_metadata( @@ -196,6 +197,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para set_extension=False, config_root=config_root, config_file=config_file, + datatypes_config=datatypes_config, compute_tmp_dir=compute_tmp_dir, resolve_metadata_dependencies=resolve_metadata_dependencies, kwds={'overwrite': False} diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 956d8eb88da..aabb38b1d5e 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -701,7 +701,7 @@ class PulsarJobRunner(AsynchronousJobRunner): remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') metadata_kwds['datatypes_config'] = remote_datatypes_config else: - datatypes_config = os.path.join(configs_directory, 'registry.xml') + datatypes_config = os.path.join(job_wrapper.working_directory, 'registry.xml') self.app.datatypes_registry.to_xml_file(path=datatypes_config) # Ensure this file gets pushed out to the remote config dir. job_wrapper.extra_filenames.append(datatypes_config) diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index a1dd075ffd8..fa6e9b3537a 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -836,7 +836,7 @@ class JobExternalOutputMetadataWrapper(object): sa_session.add(metadata_files) sa_session.flush() metadata_files_list.append(metadata_files) - args = '"%s" "%s" %s %s' % (datatypes_config, + args = '"%s" "%s" %s %s' % (metadata_path_on_compute(datatypes_config), job_metadata, " ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)), max_metadata_value_size) From db5185cca0bb9b5f59d8fcf218eb4c5d81cb5911 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 21 Sep 2017 16:12:46 -0400 Subject: [PATCH 06/10] Pulsar release was 0.8.0 not 0.7.5 --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- lib/galaxy/jobs/runners/pulsar.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 1bd08e183cb..80b2bcffa29 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.5 +pulsar-galaxy-lib==0.8.0 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index aabb38b1d5e..459e41fd1fc 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -53,7 +53,7 @@ __all__ = ( MINIMUM_PULSAR_VERSIONS = { '_default_': LooseVersion("0.7.0.dev3"), - 'remote_metadata': LooseVersion("0.7.5"), + 'remote_metadata': LooseVersion("0.8.0"), } NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." From 2669ab022ed2f90faac54c510014f87783efeaa0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 14:52:24 -0400 Subject: [PATCH 07/10] Pass job output file unqualified names to Pulsar so that it can create them before running the job. --- lib/galaxy/jobs/__init__.py | 10 ++++++++++ lib/galaxy/jobs/runners/pulsar.py | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 1e21b45c9db..8d5f263d9d6 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1571,6 +1571,9 @@ class JobWrapper(object, HasResourceParameters): paths.append(DatasetPath(da.id, real_path=real_path, false_path=false_path, mutable=False)) return paths + def get_output_basenames(self): + return map(os.path.basename, map(str, self.get_output_fnames())) + def get_output_fnames(self): if self.output_paths is None: self.compute_outputs() @@ -1997,6 +2000,10 @@ class ComputeEnvironment(object): compute server. """ + @abstractmethod + def output_names(self): + """ Output unqualified filenames defined by job. """ + @abstractmethod def output_paths(self): """ Output DatasetPaths defined by job. """ @@ -2062,6 +2069,9 @@ class SharedComputeEnvironment(SimpleComputeEnvironment): self.job_wrapper = job_wrapper self.job = job + def output_names(self): + return self.job_wrapper.get_output_basenames() + def output_paths(self): return self.job_wrapper.get_output_fnames() diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..f1b34681e03 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -277,8 +277,10 @@ class PulsarJobRunner(AsynchronousJobRunner): dependencies_description = PulsarJobRunner.__dependencies_description(client, job_wrapper) rewrite_paths = not PulsarJobRunner.__rewrite_parameters(client) unstructured_path_rewrites = {} + output_names = [] if compute_environment: unstructured_path_rewrites = compute_environment.unstructured_path_rewrites + output_names = compute_environment.output_names() client_job_description = ClientJobDescription( command_line=command_line, @@ -292,6 +294,7 @@ class PulsarJobRunner(AsynchronousJobRunner): env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, + touch_outputs=output_names, ) job_id = pulsar_submit_job(client, client_job_description, remote_job_config) log.info("Pulsar job submitted with job_id %s" % job_id) @@ -783,6 +786,10 @@ class PulsarComputeEnvironment(ComputeEnvironment): version_path = new_version_path self._version_path = version_path + def output_names(self): + # Maybe this should use the path mapper, but the path mapper just uses basenames + return self.job_wrapper.get_output_basenames() + def output_paths(self): local_output_paths = self._wrapper_output_paths From 3344c77a188cd92ffb2e7150a4f4a2f4f7facedb Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Sep 2017 15:56:37 -0400 Subject: [PATCH 08/10] Bump pulsar lib version to 0.8.0 for output touch support. --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 4c4f7601c3b..80b2bcffa29 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev5 +pulsar-galaxy-lib==0.8.0 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 From c7699cd3a0e16b9522af334cf2ea1a107fbe5f57 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 21 Sep 2017 18:23:31 -0400 Subject: [PATCH 09/10] Fix lint error --- lib/galaxy/jobs/runners/pulsar.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 459e41fd1fc..72efe1f0e1f 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -670,7 +670,6 @@ class PulsarJobRunner(AsynchronousJobRunner): raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) metadata_kwds['exec_dir'] = remote_galaxy_home outputs_directory = remote_job_config['outputs_directory'] - configs_directory = remote_job_config['configs_directory'] working_directory = remote_job_config['working_directory'] metadata_directory = remote_job_config['metadata_directory'] # For metadata calculation, we need to build a list of of output From 41013563e5e15745fd2bf796340578f6d758fbe8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 24 Sep 2017 21:18:50 +0200 Subject: [PATCH 10/10] Update CITATION file with 2016 paper --- CITATION | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CITATION b/CITATION index 2151602534b..49be43c17d1 100644 --- a/CITATION +++ b/CITATION @@ -1,6 +1,13 @@ If you use or extend Galaxy in your published work, please cite each of the following publications: +- Afgan E, Baker D, van den Beek M, Blankenberg D, Bouvier D, Čech M, Chilton + J, Clements D, Coraor N, Eberhard C, Grüning B, Guerler A, Hillman-Jackson + J,Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks + J. "The Galaxy platform for accessible, reproducible and collaborative + biomedical analyses: 2016 update." + Nucleic acids research, 44(W1), W3-W10. + - Goecks, J, Nekrutenko, A, Taylor, J and The Galaxy Team. "Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences." @@ -22,6 +29,17 @@ See also: https://galaxyproject.org/citing-galaxy BibTeX format: +@article{afgan2016galaxy, + title={The Galaxy platform for accessible, reproducible and collaborative biomedical analyses: 2016 update}, + author={Afgan, Enis and Baker, Dannon and Van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and Čech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and others}, + journal={Nucleic acids research}, + volume={44}, + number={W1}, + pages={W3--W10}, + year={2016}, + publisher={Oxford University Press} +} + @article{goecks2010galaxy, title={Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences}, author={Goecks, Jeremy and Nekrutenko, Anton and Taylor, James and The Galaxy Team},