Merge pull request #1473 from jmchilton/job_shell

[16.01] Allow override of job shell (for conda resolver).

config/galaxy.ini.sample

@@ -307,6 +307,14 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 #check_job_script_integrity_sleep = .25
 
 
+# Set the default shell used by jobs Galaxy-wide. This defaults to
+# sh and can be overidden at the destination level for
+# heterogenous clusters. conda job resolution requires bash or zsh
+# so if this is not switched to /bin/bash for instance - conda resolution
+# should be disabled. (16.01 will likely be the last release with a
+# default of /bin/sh instead of /bin/bash).
+#default_job_shell = /bin/sh
+
 # Citation related caching.  Tool citations information maybe fetched from
 # external sources such as http://dx.doi.org/ by Galaxy - the following
 # parameters can be used to control the caching used to store this information.

lib/galaxy/config.py

@@ -169,6 +169,7 @@ class Configuration( object ):
         self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") )
         self.cluster_files_directory = os.path.abspath( kwargs.get( "cluster_files_directory", "database/pbs" ) )
         self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root )
+        self.default_job_shell = kwargs.get( "default_job_shell", "/bin/sh" )
         self.cleanup_job = kwargs.get( "cleanup_job", "always" )
         self.container_image_cache_path = self.resolve_path( kwargs.get( "container_image_cache_path", "database/container_images" ) )
         self.outputs_to_working_directory = string_as_bool( kwargs.get( 'outputs_to_working_directory', False ) )

lib/galaxy/jobs/__init__.py

@@ -43,6 +43,9 @@ log = logging.getLogger( __name__ )
 # and should eventually become API'd
 TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
 
+# Override with config.default_job_shell.
+DEFAULT_JOB_SHELL = '/bin/sh'
+
 
 class JobDestination( Bunch ):
     """
@@ -55,6 +58,7 @@ class JobDestination( Bunch ):
         self['runner'] = None
         self['legacy'] = False
         self['converted'] = False
+        self['shell'] = None
         self['env'] = []
         self['resubmit'] = []
         # dict is appropriate (rather than a bunch) since keys may not be valid as attributes
@@ -785,6 +789,10 @@ class JobWrapper( object ):
     def get_parallelism(self):
         return self.tool.parallelism
 
+    @property
+    def shell(self):
+        return self.job_destination.shell or getattr(self.app.config, 'default_job_shell', DEFAULT_JOB_SHELL)
+
     @property
     def commands_in_new_shell(self):
         return self.app.config.commands_in_new_shell

lib/galaxy/jobs/command_factory.py

@@ -14,7 +14,6 @@ log = getLogger( __name__ )
 
 CAPTURE_RETURN_CODE = "return_code=$?"
 YIELD_CAPTURED_CODE = 'sh -c "exit $return_code"'
-DEFAULT_SHELL = "/bin/sh"
 
 
 def build_command(
@@ -23,7 +22,7 @@ def build_command(
     container=None,
     include_metadata=False,
     include_work_dir_outputs=True,
-    remote_command_params={}
+    remote_command_params={},
 ):
     """
     Compose the sequence of commands necessary to execute a job. This will
@@ -34,6 +33,7 @@ def build_command(
         - command line taken from job wrapper
         - commands to set metadata (if include_metadata is True)
     """
+    shell = job_wrapper.shell
     base_command_line = job_wrapper.get_command_line()
     # job_id = job_wrapper.job_id
     # log.debug( 'Tool evaluation for job (%s) produced command-line: %s' % ( job_id, base_command_line ) )
@@ -56,7 +56,7 @@ def build_command(
         __handle_dependency_resolution(commands_builder, job_wrapper, remote_command_params)
 
     if container or job_wrapper.commands_in_new_shell:
-        externalized_commands = __externalize_commands(job_wrapper, commands_builder, remote_command_params)
+        externalized_commands = __externalize_commands(job_wrapper, shell, commands_builder, remote_command_params)
         if container:
             # Stop now and build command before handling metadata and copying
             # working directory files back. These should always happen outside
@@ -80,18 +80,18 @@ def build_command(
     return commands_builder.build()
 
 
-def __externalize_commands(job_wrapper, commands_builder, remote_command_params, script_name="tool_script.sh"):
+def __externalize_commands(job_wrapper, shell, commands_builder, remote_command_params, script_name="tool_script.sh"):
     local_container_script = join( job_wrapper.working_directory, script_name )
     tool_commands = commands_builder.build()
     config = job_wrapper.app.config
     integrity_injection = ""
     if check_script_integrity(config):
         integrity_injection = INTEGRITY_INJECTION
-    script_contents = u"#!%s\n%s%s" % (DEFAULT_SHELL, integrity_injection, tool_commands)
+    script_contents = u"#!%s\n%s%s" % (shell, integrity_injection, tool_commands)
     write_script(local_container_script, script_contents, config)
     commands = local_container_script
     if 'working_directory' in remote_command_params:
-        commands = "%s %s" % (DEFAULT_SHELL, join(remote_command_params['working_directory'], script_name))
+        commands = "%s %s" % (shell, join(remote_command_params['working_directory'], script_name))
     log.info("Built script [%s] for tool command[%s]" % (local_container_script, tool_commands))
     return commands
 

lib/galaxy/jobs/runners/__init__.py

@@ -167,7 +167,7 @@ class BaseJobRunner( object ):
             job_wrapper.runner_command_line = self.build_command_line(
                 job_wrapper,
                 include_metadata=include_metadata,
-                include_work_dir_outputs=include_work_dir_outputs
+                include_work_dir_outputs=include_work_dir_outputs,
             )
         except:
             log.exception("(%s) Failure preparing job" % job_id)
@@ -298,6 +298,7 @@ class BaseJobRunner( object ):
             env_setup_commands=env_setup_commands,
             working_directory=os.path.abspath( job_wrapper.working_directory ),
             command=command_line,
+            shell=job_wrapper.shell,
         )
         # Additional logging to enable if debugging from_work_dir handling, metadata
         # commands, etc... (or just peak in the job script.)

lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!$shell
 
 $integrity_injection
 $headers

lib/galaxy/jobs/runners/util/job_script/__init__.py

@@ -6,6 +6,8 @@ from pkg_resources import resource_string
 
 from galaxy.util import unicodify
 
+DEFAULT_SHELL = '/bin/sh'
+
 DEFAULT_JOB_FILE_TEMPLATE = Template(
     resource_string(__name__, 'DEFAULT_JOB_FILE_TEMPLATE.sh').decode('UTF-8')
 )
@@ -40,6 +42,7 @@ OPTIONAL_TEMPLATE_PARAMS = {
     'instrument_pre_commands': '',
     'instrument_post_commands': '',
     'integrity_injection': INTEGRITY_INJECTION,
+    'shell': DEFAULT_SHELL,
 }
 
 

lib/galaxy/tools/deps/resolvers/conda.py

@@ -177,7 +177,7 @@ class CondaDepenency(Dependency):
         return self._exact
 
     def shell_commands(self, requirement):
-        return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || source %s '%s'""" % (
+        return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s'""" % (
             self.environment_path,
             self.activate,
             self.environment_path

test/functional/tools/output_action_change_format.xml

@@ -1,6 +1,6 @@
 <tool id="output_action_change_format" name="output_action_change_format" version="1.0.0">
     <command>
-        echo "1\t2" > out1;
+        printf "1\t2\n" > out1;
     </command>
     <inputs>
         <conditional name="input_cond">

test/unit/jobs/test_command_factory.py

@@ -168,6 +168,7 @@ class MockJobWrapper(object):
                 check_job_script_integrity=False,
             )
         )
+        self.shell = "/bin/sh"
 
     def get_command_line(self):
         return self.command_line

test/unit/jobs/test_runner_local.py

@@ -121,6 +121,7 @@ class MockJobWrapper( object ):
         self.mock_metadata_path = os.path.abspath( os.path.join( test_directory, "METADATA_SET" ) )
         self.metadata_command = "touch %s" % self.mock_metadata_path
         self.galaxy_virtual_env = None
+        self.shell = "/bin/bash"
 
         # Cruft for setting metadata externally, axe at some point.
         self.external_output_metadata = bunch.Bunch(