diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 5a231a1957b..8028f9917c3 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -1192,6 +1192,18 @@ use_interactive = True # option (Linux) or -actimeo=0 (Solaris). #retry_job_output_collection = 0 +# In the past Galaxy would preserve its Python environment when running jobs ( +# and still does for internal tools packaged with Galaxy). This behavior exposes +# Galaxy internals to tools and could result in problems when activating +# Python environments for tools (such as with Conda packaging). The default +# legacy_only will restrict this behavior to tools identified by the Galaxy +# team as requiring this environment. Set this to "always" to restore the +# previous behavior (and potentially break Conda dependency resolution for many +# tools). Set this to legacy_and_local to preserve the environment for legacy +# tools and locally managed tools (this might be useful for instance if you are +# installing software into Galaxy's virtualenv for tool development). +#python_environment_problem = legacy_only + # Clean up various bits of jobs left on the filesystem after completion. These # bits include the job working directory, external metadata temporary files, # and DRM stdout and stderr files (if using a DRM). Possible values are: diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 1f62447cf10..7d6a754e772 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -175,6 +175,11 @@ class Configuration( object ): self.jobs_directory = resolve_path( kwargs.get( "jobs_directory", default_jobs_directory ), self.root ) self.default_job_shell = kwargs.get( "default_job_shell", "/bin/bash" ) self.cleanup_job = kwargs.get( "cleanup_job", "always" ) + preserve_python_environment = kwargs.get( "preserve_python_environment", "legacy_only" ) + if preserve_python_environment not in ["legacy_only", "legacy_and_local", "always"]: + log.warn("preserve_python_environment set to unknown value [%s], defaulting to legacy_only") + preserve_python_environment = "legacy_only" + self.preserve_python_environment = preserve_python_environment self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) ) self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) ) self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) ) diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index 4f7f0aaaee0..cca621057d9 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -16,6 +16,21 @@ log = getLogger( __name__ ) CAPTURE_RETURN_CODE = "return_code=$?" YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"' +SETUP_GALAXY_FOR_METADATA = """ +if [ "$GALAXY_LIB" != "None" ]; then + if [ -n "$PYTHONPATH" ]; then + PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" + else + PYTHONPATH="$GALAXY_LIB" + fi + export PYTHONPATH +fi +if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ + -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then + . "$GALAXY_VIRTUAL_ENV/bin/activate" +fi +GALAXY_PYTHON=`which python` +""" def build_command( @@ -194,6 +209,8 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para ) or '' metadata_command = metadata_command.strip() if metadata_command: + # Place Galaxy and its dependencies in environment for metadata regardless of tool. + metadata_command = "%s%s" % (SETUP_GALAXY_FOR_METADATA, metadata_command) commands_builder.capture_return_code() commands_builder.append_command(metadata_command) diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py index e46c34a6160..ba186503144 100644 --- a/lib/galaxy/jobs/runners/__init__.py +++ b/lib/galaxy/jobs/runners/__init__.py @@ -314,6 +314,7 @@ class BaseJobRunner( object ): working_directory=os.path.abspath( job_wrapper.working_directory ), command=command_line, shell=job_wrapper.shell, + preserve_python_environment=job_wrapper.tool.requires_galaxy_python_environment, ) # Additional logging to enable if debugging from_work_dir handling, metadata # commands, etc... (or just peak in the job script.) diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index a2205509935..801df6acbc1 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -4,8 +4,9 @@ $headers $integrity_injection $slots_statement export GALAXY_SLOTS +PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" -if [ "$GALAXY_LIB" != "None" ]; then +if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" else @@ -16,10 +17,9 @@ fi $env_setup_commands GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \ - -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then + -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate" fi -GALAXY_PYTHON=`which python` $instrument_pre_commands cd $working_directory $command diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 1bcc391fc6b..77af249e5c4 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -45,6 +45,7 @@ OPTIONAL_TEMPLATE_PARAMS = { 'instrument_post_commands': '', 'integrity_injection': INTEGRITY_INJECTION, 'shell': DEFAULT_SHELL, + 'preserve_python_environment': True, } diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 7eea8700528..702cecee69c 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -73,6 +73,45 @@ log = logging.getLogger( __name__ ) HELP_UNINITIALIZED = threading.Lock() MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__)) +# Tools that require Galaxy's Python environment to be preserved. +GALAXY_LIB_TOOLS = [ + "upload1", + # Legacy tools bundled with Galaxy. + "vcf_to_maf_customtrack1", + "laj_1", + "meme_fimo", + "secure_hash_message_digest", + "join1", + "gff2bed1", + "gff_filter_by_feature_count", + "Extract genomic DNA 1", + "aggregate_scores_in_intervals2", + "Interval_Maf_Merged_Fasta2", + "maf_stats1", + "Interval2Maf1", + "MAF_To_Interval1", + "MAF_filter", + "MAF_To_Fasta1", + "MAF_Reverse_Complement_1", + "MAF_split_blocks_by_species1", + "maf_limit_size1", + "maf_by_block_number1", + # Tools improperly migrated to the tool shed (devteam) + "lastz_wrapper_2", + "qualityFilter", + "winSplitter", + "pileup_interval", + "count_gff_features", + "Convert characters1", + "lastz_paired_reads_wrapper", + "subRate1", + "substitutions1", + "sam_pileup", + "find_diag_hits", + "cufflinks", + # Tools improperly migrated to the tool shed (iuc) + "tabular_to_dbnsfp", +] class ToolErrorLog: @@ -404,10 +443,17 @@ class Tool( object, Dictifiable ): """Indicates this tool's runtime requires Galaxy's Python environment.""" # All special tool types (data source, history import/export, etc...) # seem to require Galaxy's Python. - return self.tool_type != "default" or self.id in [ - "__SET_METADATA__", - "upload1", - ] + if self.tool_type != "default": + return True + + config = self.app.config + preserve_python_environment = config.preserve_python_environment + if preserve_python_environment == "always": + return True + elif preserve_python_environment == "legacy_and_local" and self.repository_id is None: + return True + else: + return self.old_id in GALAXY_LIB_TOOLS def __get_job_tool_configuration(self, job_params=None): """Generalized method for getting this tool's job configuration. diff --git a/test/functional/tools/python_environment_problem.py b/test/functional/tools/python_environment_problem.py new file mode 100644 index 00000000000..19bc7305b01 --- /dev/null +++ b/test/functional/tools/python_environment_problem.py @@ -0,0 +1,12 @@ +import sys + +import galaxy.eggs # NOQA + + +def main(): + with open(sys.argv[1], "w") as f: + f.write("out_file1") + + +if __name__ == '__main__': + main() diff --git a/test/functional/tools/python_environment_problem.xml b/test/functional/tools/python_environment_problem.xml new file mode 100644 index 00000000000..dcc2961e1bc --- /dev/null +++ b/test/functional/tools/python_environment_problem.xml @@ -0,0 +1,20 @@ + + + + python $__tool_directory__/python_environment_problem.py '$out_file1' + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 6940f128af5..ccef907c6a1 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -97,6 +97,7 @@ + diff --git a/test/unit/jobs/test_command_factory.py b/test/unit/jobs/test_command_factory.py index 9166a489c50..cfa9be34da3 100644 --- a/test/unit/jobs/test_command_factory.py +++ b/test/unit/jobs/test_command_factory.py @@ -4,7 +4,7 @@ from os import getcwd from tempfile import mkdtemp from unittest import TestCase -from galaxy.jobs.command_factory import build_command +from galaxy.jobs.command_factory import build_command, SETUP_GALAXY_FOR_METADATA from galaxy.util.bunch import Bunch MOCK_COMMAND_LINE = "/opt/galaxy/tools/bowtie /mnt/galaxyData/files/000/input000.dat" @@ -87,7 +87,7 @@ class TestCommandFactory(TestCase): self.include_metadata = True self.include_work_dir_outputs = False self.job_wrapper.metadata_line = TEST_METADATA_LINE - expected_command = _surrond_command("%s; return_code=$?; cd '%s'; %s" % (MOCK_COMMAND_LINE, self.job_dir, TEST_METADATA_LINE)) + expected_command = _surrond_command("%s; return_code=$?; cd '%s'; %s%s" % (MOCK_COMMAND_LINE, self.job_dir, SETUP_GALAXY_FOR_METADATA, TEST_METADATA_LINE)) self.__assert_command_is( expected_command ) def test_empty_metadata(self): diff --git a/test/unit/tools_support.py b/test/unit/tools_support.py index 51f23cb4ccc..3a6fe71c00b 100644 --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -120,6 +120,7 @@ class MockApp( object ): builds_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'builds.txt.sample' ), migrated_tools_config=os.path.join(test_directory, "migrated_tools_conf.xml"), server_name="test_server", + preserve_python_environment="always", ) # Setup some attributes for downstream extension by specific tests.