Merge pull request #3364 from jmchilton/clean_python

Limit adjusting Galaxy's Python environment to legacy tools.

config/galaxy.ini.sample

@@ -1192,6 +1192,18 @@ use_interactive = True
 # option (Linux) or -actimeo=0 (Solaris).
 #retry_job_output_collection = 0
 
+# In the past Galaxy would preserve its Python environment when running jobs (
+# and still does for internal tools packaged with Galaxy). This behavior exposes
+# Galaxy internals to tools and could result in problems when activating
+# Python environments for tools (such as with Conda packaging). The default
+# legacy_only will restrict this behavior to tools identified by the Galaxy
+# team as requiring this environment. Set this to "always" to restore the
+# previous behavior (and potentially break Conda dependency resolution for many
+# tools). Set this to legacy_and_local to preserve the environment for legacy
+# tools and locally managed tools (this might be useful for instance if you are
+# installing software into Galaxy's virtualenv for tool development).
+#python_environment_problem = legacy_only
+
 # Clean up various bits of jobs left on the filesystem after completion.  These
 # bits include the job working directory, external metadata temporary files,
 # and DRM stdout and stderr files (if using a DRM).  Possible values are:

lib/galaxy/config.py

@@ -175,6 +175,11 @@ class Configuration( object ):
         self.jobs_directory = resolve_path( kwargs.get( "jobs_directory", default_jobs_directory ), self.root )
         self.default_job_shell = kwargs.get( "default_job_shell", "/bin/bash" )
         self.cleanup_job = kwargs.get( "cleanup_job", "always" )
+        preserve_python_environment = kwargs.get( "preserve_python_environment", "legacy_only" )
+        if preserve_python_environment not in ["legacy_only", "legacy_and_local", "always"]:
+            log.warn("preserve_python_environment set to unknown value [%s], defaulting to legacy_only")
+            preserve_python_environment = "legacy_only"
+        self.preserve_python_environment = preserve_python_environment
         self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) )
         self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )
         self.output_size_limit = int( kwargs.get( 'output_size_limit', 0 ) )

lib/galaxy/jobs/command_factory.py

@@ -16,6 +16,21 @@ log = getLogger( __name__ )
 
 CAPTURE_RETURN_CODE = "return_code=$?"
 YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"'
+SETUP_GALAXY_FOR_METADATA = """
+if [ "$GALAXY_LIB" != "None" ]; then
+    if [ -n "$PYTHONPATH" ]; then
+        PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
+    else
+        PYTHONPATH="$GALAXY_LIB"
+    fi
+    export PYTHONPATH
+fi
+if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
+     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then
+    . "$GALAXY_VIRTUAL_ENV/bin/activate"
+fi
+GALAXY_PYTHON=`which python`
+"""
 
 
 def build_command(
@@ -194,6 +209,8 @@ def __handle_metadata(commands_builder, job_wrapper, runner, remote_command_para
     ) or ''
     metadata_command = metadata_command.strip()
     if metadata_command:
+        # Place Galaxy and its dependencies in environment for metadata regardless of tool.
+        metadata_command = "%s%s" % (SETUP_GALAXY_FOR_METADATA, metadata_command)
         commands_builder.capture_return_code()
         commands_builder.append_command(metadata_command)
 

lib/galaxy/jobs/runners/__init__.py

@@ -314,6 +314,7 @@ class BaseJobRunner( object ):
             working_directory=os.path.abspath( job_wrapper.working_directory ),
             command=command_line,
             shell=job_wrapper.shell,
+            preserve_python_environment=job_wrapper.tool.requires_galaxy_python_environment,
         )
         # Additional logging to enable if debugging from_work_dir handling, metadata
         # commands, etc... (or just peak in the job script.)

lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh

@@ -4,8 +4,9 @@ $headers
 $integrity_injection
 $slots_statement
 export GALAXY_SLOTS
+PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment"
 GALAXY_LIB="$galaxy_lib"
-if [ "$GALAXY_LIB" != "None" ]; then
+if [ "$GALAXY_LIB" != "None" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then
     if [ -n "$PYTHONPATH" ]; then
         PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"
     else
@@ -16,10 +17,9 @@ fi
 $env_setup_commands
 GALAXY_VIRTUAL_ENV="$galaxy_virtual_env"
 if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" \
-     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then
+     -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" -a "$PRESERVE_GALAXY_ENVIRONMENT" = "True" ]; then
     . "$GALAXY_VIRTUAL_ENV/bin/activate"
 fi
-GALAXY_PYTHON=`which python`
 $instrument_pre_commands
 cd $working_directory
 $command

lib/galaxy/jobs/runners/util/job_script/__init__.py

@@ -45,6 +45,7 @@ OPTIONAL_TEMPLATE_PARAMS = {
     'instrument_post_commands': '',
     'integrity_injection': INTEGRITY_INJECTION,
     'shell': DEFAULT_SHELL,
+    'preserve_python_environment': True,
 }
 
 

lib/galaxy/tools/__init__.py

@@ -73,6 +73,45 @@ log = logging.getLogger( __name__ )
 
 HELP_UNINITIALIZED = threading.Lock()
 MODEL_TOOLS_PATH = os.path.abspath(os.path.dirname(__file__))
+# Tools that require Galaxy's Python environment to be preserved.
+GALAXY_LIB_TOOLS = [
+    "upload1",
+    # Legacy tools bundled with Galaxy.
+    "vcf_to_maf_customtrack1",
+    "laj_1",
+    "meme_fimo",
+    "secure_hash_message_digest",
+    "join1",
+    "gff2bed1",
+    "gff_filter_by_feature_count",
+    "Extract genomic DNA 1",
+    "aggregate_scores_in_intervals2",
+    "Interval_Maf_Merged_Fasta2",
+    "maf_stats1",
+    "Interval2Maf1",
+    "MAF_To_Interval1",
+    "MAF_filter",
+    "MAF_To_Fasta1",
+    "MAF_Reverse_Complement_1",
+    "MAF_split_blocks_by_species1",
+    "maf_limit_size1",
+    "maf_by_block_number1",
+    # Tools improperly migrated to the tool shed (devteam)
+    "lastz_wrapper_2",
+    "qualityFilter",
+    "winSplitter",
+    "pileup_interval",
+    "count_gff_features",
+    "Convert characters1",
+    "lastz_paired_reads_wrapper",
+    "subRate1",
+    "substitutions1",
+    "sam_pileup",
+    "find_diag_hits",
+    "cufflinks",
+    # Tools improperly migrated to the tool shed (iuc)
+    "tabular_to_dbnsfp",
+]
 
 
 class ToolErrorLog:
@@ -404,10 +443,17 @@ class Tool( object, Dictifiable ):
         """Indicates this tool's runtime requires Galaxy's Python environment."""
         # All special tool types (data source, history import/export, etc...)
         # seem to require Galaxy's Python.
-        return self.tool_type != "default" or self.id in [
-            "__SET_METADATA__",
-            "upload1",
-        ]
+        if self.tool_type != "default":
+            return True
+
+        config = self.app.config
+        preserve_python_environment = config.preserve_python_environment
+        if preserve_python_environment == "always":
+            return True
+        elif preserve_python_environment == "legacy_and_local" and self.repository_id is None:
+            return True
+        else:
+            return self.old_id in GALAXY_LIB_TOOLS
 
     def __get_job_tool_configuration(self, job_params=None):
         """Generalized method for getting this tool's job configuration.

test/functional/tools/python_environment_problem.py

@@ -0,0 +1,12 @@
+import sys
+
+import galaxy.eggs  # NOQA
+
+
+def main():
+    with open(sys.argv[1], "w") as f:
+        f.write("out_file1")
+
+
+if __name__ == '__main__':
+    main()

test/functional/tools/python_environment_problem.xml

@@ -0,0 +1,20 @@
+<tool id="python_environment_problem" name="python_environment_problem" version="0.1.0">
+    <!--
+        This script imports galaxy.eggs which is no longer available to tools
+        by default. Only declared legacy tools have this available.
+    -->
+    <command>
+        python $__tool_directory__/python_environment_problem.py '$out_file1'
+    </command>
+    <inputs>
+    </inputs>
+    <outputs>
+        <data name="out_file1" />
+    </outputs>
+    <tests>
+        <test expect_failure="true">
+        </test>
+    </tests>
+    <help>
+    </help>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -97,6 +97,7 @@
   <tool file="cheetah_problem_unbound_var.xml" />
   <tool file="cheetah_problem_unbound_var_input.xml" />
   <tool file="cheetah_problem_syntax_error.xml" />
+  <tool file="python_environment_problem.xml" />
 
   <tool file="multiple_versions_v01.xml" />
   <tool file="multiple_versions_v02.xml" />

test/unit/jobs/test_command_factory.py

@@ -4,7 +4,7 @@ from os import getcwd
 from tempfile import mkdtemp
 from unittest import TestCase
 
-from galaxy.jobs.command_factory import build_command
+from galaxy.jobs.command_factory import build_command, SETUP_GALAXY_FOR_METADATA
 from galaxy.util.bunch import Bunch
 
 MOCK_COMMAND_LINE = "/opt/galaxy/tools/bowtie /mnt/galaxyData/files/000/input000.dat"
@@ -87,7 +87,7 @@ class TestCommandFactory(TestCase):
         self.include_metadata = True
         self.include_work_dir_outputs = False
         self.job_wrapper.metadata_line = TEST_METADATA_LINE
-        expected_command = _surrond_command("%s; return_code=$?; cd '%s'; %s" % (MOCK_COMMAND_LINE, self.job_dir, TEST_METADATA_LINE))
+        expected_command = _surrond_command("%s; return_code=$?; cd '%s'; %s%s" % (MOCK_COMMAND_LINE, self.job_dir, SETUP_GALAXY_FOR_METADATA, TEST_METADATA_LINE))
         self.__assert_command_is( expected_command )
 
     def test_empty_metadata(self):

test/unit/tools_support.py

@@ -120,6 +120,7 @@ class MockApp( object ):
             builds_file_path=os.path.join( 'tool-data', 'shared', 'ucsc', 'builds.txt.sample' ),
             migrated_tools_config=os.path.join(test_directory, "migrated_tools_conf.xml"),
             server_name="test_server",
+            preserve_python_environment="always",
         )
 
         # Setup some attributes for downstream extension by specific tests.