diff --git a/CITATION b/CITATION index 2151602534b..49be43c17d1 100644 --- a/CITATION +++ b/CITATION @@ -1,6 +1,13 @@ If you use or extend Galaxy in your published work, please cite each of the following publications: +- Afgan E, Baker D, van den Beek M, Blankenberg D, Bouvier D, Čech M, Chilton + J, Clements D, Coraor N, Eberhard C, Grüning B, Guerler A, Hillman-Jackson + J,Von Kuster G, Rasche E, Soranzo N, Turaga N, Taylor J, Nekrutenko A, Goecks + J. "The Galaxy platform for accessible, reproducible and collaborative + biomedical analyses: 2016 update." + Nucleic acids research, 44(W1), W3-W10. + - Goecks, J, Nekrutenko, A, Taylor, J and The Galaxy Team. "Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences." @@ -22,6 +29,17 @@ See also: https://galaxyproject.org/citing-galaxy BibTeX format: +@article{afgan2016galaxy, + title={The Galaxy platform for accessible, reproducible and collaborative biomedical analyses: 2016 update}, + author={Afgan, Enis and Baker, Dannon and Van den Beek, Marius and Blankenberg, Daniel and Bouvier, Dave and Čech, Martin and Chilton, John and Clements, Dave and Coraor, Nate and Eberhard, Carl and others}, + journal={Nucleic acids research}, + volume={44}, + number={W1}, + pages={W3--W10}, + year={2016}, + publisher={Oxford University Press} +} + @article{goecks2010galaxy, title={Galaxy: a comprehensive approach for supporting accessible, reproducible, and transparent computational research in the life sciences}, author={Goecks, Jeremy and Nekrutenko, Anton and Taylor, James and The Galaxy Team}, diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 4c4f7601c3b..80b2bcffa29 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -57,7 +57,7 @@ anyjson==0.3.3 # Pulsar requirements psutil==4.1.0 -pulsar-galaxy-lib==0.7.0.dev5 +pulsar-galaxy-lib==0.8.0 # sqlalchemy-migrate and dependencies sqlalchemy-migrate==0.10.0 diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 1e21b45c9db..08457df9aa2 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1571,6 +1571,9 @@ class JobWrapper(object, HasResourceParameters): paths.append(DatasetPath(da.id, real_path=real_path, false_path=false_path, mutable=False)) return paths + def get_output_basenames(self): + return map(os.path.basename, map(str, self.get_output_fnames())) + def get_output_fnames(self): if self.output_paths is None: self.compute_outputs() @@ -1648,7 +1651,8 @@ class JobWrapper(object, HasResourceParameters): def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, config_root=None, - config_file=None, resolve_metadata_dependencies=False, + config_file=None, datatypes_config=None, + resolve_metadata_dependencies=False, set_extension=True, **kwds): # extension could still be 'auto' if this is the upload tool. job = self.get_job() @@ -1667,8 +1671,9 @@ class JobWrapper(object, HasResourceParameters): config_root = self.app.config.root if config_file is None: config_file = self.app.config.config_file - datatypes_config = os.path.join(self.working_directory, 'registry.xml') - self.app.datatypes_registry.to_xml_file(path=datatypes_config) + if datatypes_config is None: + datatypes_config = os.path.join(self.working_directory, 'registry.xml') + self.app.datatypes_registry.to_xml_file(path=datatypes_config) command = self.external_output_metadata.setup_external_metadata([output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets + job.output_library_datasets], @@ -1977,7 +1982,7 @@ class TaskWrapper(JobWrapper): pass def setup_external_metadata(self, exec_dir=None, tmp_dir=None, dataset_files_path=None, - config_root=None, config_file=None, + config_root=None, config_file=None, datatypes_config=None, set_extension=True, **kwds): # There is no metadata setting for tasks. This is handled after the merge, at the job level. return "" @@ -1997,6 +2002,10 @@ class ComputeEnvironment(object): compute server. """ + @abstractmethod + def output_names(self): + """ Output unqualified filenames defined by job. """ + @abstractmethod def output_paths(self): """ Output DatasetPaths defined by job. """ @@ -2062,6 +2071,9 @@ class SharedComputeEnvironment(SimpleComputeEnvironment): self.job_wrapper = job_wrapper self.job = job + def output_names(self): + return self.job_wrapper.get_output_basenames() + def output_paths(self): return self.job_wrapper.get_output_fnames() diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 2f7402f399c..545328bbeb1 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -308,7 +308,6 @@ class DeleteIntermediatesAction(DefaultJobAction): safe_to_delete = True for job_to_check in [d_j.job for d_j in input_dataset.dependent_jobs]: if job_to_check != job and job_to_check.state not in [job.states.OK, job.states.DELETED]: - log.debug("Workflow Intermediates cleanup attempted, but non-terminal state '%s' detected for job %s" % (job_to_check.state, job_to_check.id)) safe_to_delete = False if safe_to_delete: # Support purging here too. diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index abf1a0e84de..7922531ee8b 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -17,19 +17,7 @@ log = getLogger(__name__) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' SETUP_GALAXY_FOR_METADATA = """ -if [ "$GALAXY_LIB" != "None" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi -GALAXY_PYTHON=`command -v python` +[ "$GALAXY_VIRTUAL_ENV" = "None" ] && GALAXY_VIRTUAL_ENV="$_GALAXY_VIRTUAL_ENV"; _galaxy_setup_environment True """ @@ -198,6 +186,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para output_fnames = metadata_kwds.get('output_fnames', job_wrapper.get_output_fnames()) config_root = metadata_kwds.get('config_root', None) config_file = metadata_kwds.get('config_file', None) + datatypes_config = metadata_kwds.get('datatypes_config', None) compute_tmp_dir = metadata_kwds.get('compute_tmp_dir', None) resolve_metadata_dependencies = job_wrapper.commands_in_new_shell metadata_command = job_wrapper.setup_external_metadata( @@ -208,6 +197,7 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para set_extension=False, config_root=config_root, config_file=config_file, + datatypes_config=datatypes_config, compute_tmp_dir=compute_tmp_dir, resolve_metadata_dependencies=resolve_metadata_dependencies, kwds={'overwrite': False} diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 9722ea32736..6dfbc243cd0 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -51,7 +51,10 @@ __all__ = ( 'PulsarEmbeddedJobRunner', ) -MINIMUM_PULSAR_VERSION = LooseVersion("0.7.0.dev3") +MINIMUM_PULSAR_VERSIONS = { + '_default_': LooseVersion("0.7.0.dev3"), + 'remote_metadata': LooseVersion("0.8.0"), +} NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE = "Pulsar misconfiguration - Pulsar client configured to set metadata remotely, but remote Pulsar isn't properly configured with a galaxy_home directory." NO_REMOTE_DATATYPES_CONFIG = "Pulsar client is configured to use remote datatypes configuration when setting metadata externally, but Pulsar is not configured with this information. Defaulting to datatypes_conf.xml." @@ -59,7 +62,7 @@ GENERIC_REMOTE_ERROR = "Failed to communicate with remote job server." FAILED_REMOTE_ERROR = "Remote job server indicated a problem running or monitoring this job." LOST_REMOTE_ERROR = "Remote job server could not determine this job's state." -UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." % MINIMUM_PULSAR_VERSION +UPGRADE_PULSAR_ERROR = "Galaxy is misconfigured, please contact administrator. The target Pulsar server is unsupported, this version of Galaxy requires Pulsar version %s or newer." # Is there a good way to infer some default for this? Can only use # url_for from web threads. https://gist.github.com/jmchilton/9098762 @@ -277,8 +280,10 @@ class PulsarJobRunner(AsynchronousJobRunner): dependencies_description = PulsarJobRunner.__dependencies_description(client, job_wrapper) rewrite_paths = not PulsarJobRunner.__rewrite_parameters(client) unstructured_path_rewrites = {} + output_names = [] if compute_environment: unstructured_path_rewrites = compute_environment.unstructured_path_rewrites + output_names = compute_environment.output_names() client_job_description = ClientJobDescription( command_line=command_line, @@ -292,6 +297,7 @@ class PulsarJobRunner(AsynchronousJobRunner): env=client.env, rewrite_paths=rewrite_paths, arbitrary_files=unstructured_path_rewrites, + touch_outputs=output_names, ) job_id = pulsar_submit_job(client, client_job_description, remote_job_config) log.info("Pulsar job submitted with job_id %s" % job_id) @@ -310,6 +316,11 @@ class PulsarJobRunner(AsynchronousJobRunner): pulsar_job_state.job_destination = job_destination self.monitor_job(pulsar_job_state) + def __needed_features(self, client): + return { + 'remote_metadata': PulsarJobRunner.__remote_metadata(client), + } + def __prepare_job(self, job_wrapper, job_destination): """Build command-line and Pulsar client for this job.""" command_line = None @@ -319,8 +330,9 @@ class PulsarJobRunner(AsynchronousJobRunner): try: client = self.get_client_from_wrapper(job_wrapper) tool = job_wrapper.tool - remote_job_config = client.setup(tool.id, tool.version) - PulsarJobRunner.check_job_config(remote_job_config) + remote_job_config = client.setup(tool.id, tool.version, tool.requires_galaxy_python_environment) + needed_features = self.__needed_features(client) + PulsarJobRunner.check_job_config(remote_job_config, check_features=needed_features) rewrite_parameters = PulsarJobRunner.__rewrite_parameters(client) prepare_kwds = {} if rewrite_parameters: @@ -593,12 +605,17 @@ class PulsarJobRunner(AsynchronousJobRunner): return client_outputs @staticmethod - def check_job_config(remote_job_config): + def check_job_config(remote_job_config, check_features=None): + check_features = check_features or {} # 0.6.0 was newest Pulsar version that did not report it's version. pulsar_version = LooseVersion(remote_job_config.get('pulsar_version', "0.6.0")) + needed_version = LooseVersion("0.0.0") log.info("pulsar_version is %s" % pulsar_version) - if pulsar_version < MINIMUM_PULSAR_VERSION: - raise UnsupportedPulsarException() + for feature in list(check_features.keys()) + ['_default_']: + if pulsar_version < MINIMUM_PULSAR_VERSIONS[feature]: + needed_version = max(needed_version, MINIMUM_PULSAR_VERSIONS[feature]) + if pulsar_version < needed_version: + raise UnsupportedPulsarException(needed_version) @staticmethod def __dependencies_description(pulsar_client, job_wrapper): @@ -656,7 +673,6 @@ class PulsarJobRunner(AsynchronousJobRunner): raise Exception(NO_REMOTE_GALAXY_FOR_METADATA_MESSAGE) metadata_kwds['exec_dir'] = remote_galaxy_home outputs_directory = remote_job_config['outputs_directory'] - configs_directory = remote_job_config['configs_directory'] working_directory = remote_job_config['working_directory'] metadata_directory = remote_job_config['metadata_directory'] # For metadata calculation, we need to build a list of of output @@ -687,7 +703,7 @@ class PulsarJobRunner(AsynchronousJobRunner): remote_datatypes_config = os.path.join(remote_galaxy_home, 'datatypes_conf.xml') metadata_kwds['datatypes_config'] = remote_datatypes_config else: - datatypes_config = os.path.join(configs_directory, 'registry.xml') + datatypes_config = os.path.join(job_wrapper.working_directory, 'registry.xml') self.app.datatypes_registry.to_xml_file(path=datatypes_config) # Ensure this file gets pushed out to the remote config dir. job_wrapper.extra_filenames.append(datatypes_config) @@ -783,6 +799,10 @@ class PulsarComputeEnvironment(ComputeEnvironment): version_path = new_version_path self._version_path = version_path + def output_names(self): + # Maybe this should use the path mapper, but the path mapper just uses basenames + return self.job_wrapper.get_output_basenames() + def output_paths(self): local_output_paths = self._wrapper_output_paths @@ -854,5 +874,5 @@ class PulsarComputeEnvironment(ComputeEnvironment): class UnsupportedPulsarException(Exception): - def __init__(self): - super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR) + def __init__(self, needed): + super(UnsupportedPulsarException, self).__init__(UPGRADE_PULSAR_ERROR % needed) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 801df6acbc1..68da1b3e362 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -1,25 +1,33 @@ #!$shell $headers + +_galaxy_setup_environment() { + local _use_framework_galaxy="$1" + if [ "$GALAXY_LIB" != "None" -a "$_use_framework_galaxy" = "True" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH + fi + $env_setup_commands + if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \ + -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then + . "$GALAXY_VIRTUAL_ENV/bin/activate" + fi +} + $integrity_injection $slots_statement export GALAXY_SLOTS +GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" +_GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" -if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - if [ -n "$PYTHONPATH" ]; then - PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" - else - PYTHONPATH="$GALAXY_LIB" - fi - export PYTHONPATH -fi -$env_setup_commands -GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" -if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi +_galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" +GALAXY_PYTHON=`command -v python` $instrument_pre_commands cd $working_directory $command diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index a1dd075ffd8..fa6e9b3537a 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -836,7 +836,7 @@ class JobExternalOutputMetadataWrapper(object): sa_session.add(metadata_files) sa_session.flush() metadata_files_list.append(metadata_files) - args = '"%s" "%s" %s %s' % (datatypes_config, + args = '"%s" "%s" %s %s' % (metadata_path_on_compute(datatypes_config), job_metadata, " ".join(map(__metadata_files_list_to_cmd_line, metadata_files_list)), max_metadata_value_size) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 679a88c7a2c..a1e64df09b8 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -139,9 +139,9 @@ if [ $REPLACE_PIP -eq 1 ]; then fi if [ $FETCH_WHEELS -eq 1 ]; then - pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" + pip install -r requirements.txt --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))") - [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" + [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url https://pypi.python.org/pypi fi if [ $FETCH_WHEELS -eq 1 -a $DEV_WHEELS -eq 1 ]; then