Merge pull request #5193 from jmchilton/structured_job_environment_1

Improved home and temp directory handling.

config/job_conf.xml.sample_advanced

@@ -542,6 +542,23 @@
                itself.
           -->
         </destination>
+        <!-- Following three destinations demonstrate setting up per-job temp directory handling.
+             In these cases TEMP, TMP, and TMPDIR will be set for each job dispatched to these
+             destinations.
+             The first simply tells Galaxy to create a temp directory in the job directory, the
+             other forms can be used to issue shell commands before the job runs on the worker node to
+             allocate a temp directory. In these other cases, Galaxy will not clean up these
+             directories so either use directories managed by the job resource manager or setup
+             tooling to clean old temp directories up outside of Galaxy. -->
+        <destination id="clean_tmp_by_job" runner="drmma">
+          <param id="tmp_dir">True</param>
+        </destination>
+        <destination id="clean_tmp_drm" runner="drmma">
+          <param id="tmp_dir">"$DRM_SET_VARIABLES_FOR_THIS_JOB"</param>
+        </destination>
+        <destination id="clean_tmp_fast_scratch" runner="drmma">
+          <param id="tmp_dir">$(mktemp -d /mnt/scratch/fastest/gxyjobXXXXXXXXXXX)</param>
+        </destination>
         <destination id="real_user_cluster" runner="drmaa">
             <!-- Make sure to setup 3 real user parameters in galaxy.ini. -->
         </destination>

lib/galaxy/config.py

@@ -188,10 +188,12 @@ class Configuration(object):
 
         # Where dataset files are stored
         self.file_path = resolve_path(kwargs.get("file_path", "database/files"), self.root)
+        # new_file_path and legacy_home_dir can be overridden per destination in job_conf.
         self.new_file_path = resolve_path(kwargs.get("new_file_path", "database/tmp"), self.root)
         override_tempdir = string_as_bool(kwargs.get("override_tempdir", "True"))
         if override_tempdir:
             tempfile.tempdir = self.new_file_path
+        self.shared_home_dir = kwargs.get("shared_home_dir", None)
         self.openid_consumer_cache_path = resolve_path(kwargs.get("openid_consumer_cache_path", "database/openid_consumer_cache"), self.root)
         self.cookie_path = kwargs.get("cookie_path", "/")
         # Galaxy OpenID settings
@@ -481,7 +483,7 @@ class Configuration(object):
 
         involucro_path = kwargs.get('involucro_path', None)
         if involucro_path is None:
-            involucro_path = os.path.join(tool_dependency_dir, "involucro")
+            involucro_path = os.path.join(tool_dependency_dir or "database", "involucro")
         self.involucro_path = resolve_path(involucro_path, self.root)
         self.involucro_auto_init = string_as_bool(kwargs.get('involucro_auto_init', True))
 

lib/galaxy/jobs/__init__.py

@@ -1110,6 +1110,15 @@ class JobWrapper(object, HasResourceParameters):
         if flush:
             self.sa_session.flush()
 
+    @property
+    def home_target(self):
+        home_target = self.tool.home_target
+        return home_target
+
+    @property
+    def tmp_target(self):
+        return self.tool.tmp_target
+
     def get_destination_configuration(self, key, default=None):
         """ Get a destination parameter that can be defaulted back
         in app.config if it needs to be applied globally.
@@ -1603,6 +1612,39 @@ class JobWrapper(object, HasResourceParameters):
                 return dp.dataset_id
         return None
 
+    @property
+    def tmp_dir_creation_statement(self):
+        tmp_dir = self.get_destination_configuration("tmp_dir", None)
+        if not tmp_dir or tmp_dir.lower() == "true":
+            working_directory = self.working_directory
+            return '''$([ ! -e '{0}/tmp' ] || mv '{0}/tmp' '{0}'/tmp.$(date +%Y%m%d-%H%M%S) ; mkdir '{0}/tmp'; echo '{0}/tmp')'''.format(working_directory)
+        else:
+            return tmp_dir
+
+    def home_directory(self):
+        home_target = self.home_target
+        return self._target_to_directory(home_target)
+
+    def tmp_directory(self):
+        tmp_target = self.tmp_target
+        return self._target_to_directory(tmp_target)
+
+    def _target_to_directory(self, target):
+        working_directory = self.working_directory
+        tmp_dir = self.get_destination_configuration("tmp_dir", None)
+        if target is None or (target == "job_tmp_if_explicit" and tmp_dir is None):
+            return None
+        elif target in ["job_tmp", "job_tmp_if_explicit"]:
+            return "$_GALAXY_JOB_TMP_DIR"
+        elif target == "shared_home":
+            return self.get_destination_configuration("shared_home_dir", None)
+        elif target == "job_home":
+            return "$_GALAXY_JOB_HOME_DIR"
+        elif target == "pwd":
+            return os.path.join(working_directory, "working")
+        else:
+            raise Exception("Unknown target type [%s]" % target)
+
     def get_tool_provided_job_metadata(self):
         if self.tool_provided_job_metadata is not None:
             return self.tool_provided_job_metadata
@@ -2027,6 +2069,14 @@ class ComputeEnvironment(object):
         be rewritten.)
         """
 
+    @abstractmethod
+    def home_directory(self):
+        """Home directory of target job - none if HOME should not be set."""
+
+    @abstractmethod
+    def tmp_directory(self):
+        """Temp directory of target job - none if HOME should not be set."""
+
 
 class SimpleComputeEnvironment(object):
 
@@ -2072,6 +2122,12 @@ class SharedComputeEnvironment(SimpleComputeEnvironment):
     def tool_directory(self):
         return os.path.abspath(self.job_wrapper.tool.tool_dir)
 
+    def home_directory(self):
+        return self.job_wrapper.home_directory()
+
+    def tmp_directory(self):
+        return self.job_wrapper.tmp_directory()
+
 
 class NoopQueue(object):
     """

lib/galaxy/jobs/rule_helper.py

@@ -44,7 +44,7 @@ class RuleHelper(object):
             tool = self.app.toolbox.get_tool(job_or_tool.tool_id, tool_version=job_or_tool.tool_version)
         # Can't import at top because circular import between galaxy.tools and galaxy.jobs.
         import galaxy.tools.deps.containers
-        tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment)
+        tool_info = galaxy.tools.deps.containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
         container_description = self.app.container_finder.find_best_container_description(["docker"], tool_info)
         return container_description is not None
 

lib/galaxy/jobs/runners/__init__.py

@@ -327,7 +327,9 @@ class BaseJobRunner(object):
         for env in envs:
             env_setup_commands.append(env_to_statement(env))
         command_line = job_wrapper.runner_command_line
+        tmp_dir_creation_statement = job_wrapper.tmp_dir_creation_statement
         options = dict(
+            tmp_dir_creation_statement=tmp_dir_creation_statement,
             job_instrumenter=job_instrumenter,
             galaxy_lib=job_wrapper.galaxy_lib_dir,
             galaxy_virtual_env=job_wrapper.galaxy_virtual_env,
@@ -353,6 +355,7 @@ class BaseJobRunner(object):
         compute_working_directory=None,
         compute_tool_directory=None,
         compute_job_directory=None,
+        compute_tmp_directory=None,
     ):
         job_directory_type = "galaxy" if compute_working_directory is None else "pulsar"
         if not compute_working_directory:
@@ -364,13 +367,17 @@ class BaseJobRunner(object):
         if not compute_tool_directory:
             compute_tool_directory = job_wrapper.tool.tool_dir
 
+        if not compute_tmp_directory:
+            compute_tmp_directory = job_wrapper.tmp_directory()
+
         tool = job_wrapper.tool
         from galaxy.tools.deps import containers
-        tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment)
+        tool_info = containers.ToolInfo(tool.containers, tool.requirements, tool.requires_galaxy_python_environment, tool.docker_env_pass_through)
         job_info = containers.JobInfo(
             compute_working_directory,
             compute_tool_directory,
             compute_job_directory,
+            compute_tmp_directory,
             job_directory_type,
         )
 

lib/galaxy/jobs/runners/pulsar.py

@@ -871,6 +871,16 @@ class PulsarComputeEnvironment(ComputeEnvironment):
     def tool_directory(self):
         return self._tool_dir
 
+    def home_directory(self):
+        # TODO: revisit and implement this, won't break anything working in the
+        # meantime.
+        return None
+
+    def tmp_directory(self):
+        # TODO: revisit and implement this, won't break anything working in the
+        # meantime.
+        return None
+
 
 class UnsupportedPulsarException(Exception):
 

lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh

@@ -12,7 +12,12 @@ _galaxy_setup_environment() {
         fi
         export PYTHONPATH
     fi
+    _GALAXY_JOB_HOME_DIR="$working_directory/home"
+    _GALAXY_JOB_TMP_DIR=$tmp_dir_creation_statement
     $env_setup_commands
+    # These don't get cleaned on a re-run but may in the future.
+    [ -z "$_GALAXY_JOB_TMP_DIR" -a ! -f "$_GALAXY_JOB_TMP_DIR" ] || mkdir -p "$_GALAXY_JOB_TMP_DIR"
+    [ -z "$_GALAXY_JOB_HOME_DIR" -a ! -f "$_GALAXY_JOB_HOME_DIR" ] || mkdir -p "$_GALAXY_JOB_HOME_DIR"
     if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" \
          -a "`command -v python`" != "$GALAXY_VIRTUAL_ENV/bin/python" ]; then
         . "$GALAXY_VIRTUAL_ENV/bin/activate"

lib/galaxy/jobs/runners/util/job_script/__init__.py

@@ -52,6 +52,7 @@ OPTIONAL_TEMPLATE_PARAMS = {
     'integrity_injection': INTEGRITY_INJECTION,
     'shell': DEFAULT_SHELL,
     'preserve_python_environment': True,
+    'tmp_dir_creation_statement': '""',
 }
 
 

lib/galaxy/tools/__init__.py

@@ -623,6 +623,23 @@ class Tool(object, Dictifiable):
 
         self.parse_command(tool_source)
         self.environment_variables = self.parse_environment_variables(tool_source)
+        self.tmp_directory_vars = tool_source.parse_tmp_directory_vars()
+
+        home_target = tool_source.parse_home_target()
+        tmp_target = tool_source.parse_tmp_target()
+        # If a tool explicitly sets one of these variables just respect that and turn off
+        # explicit processing by Galaxy.
+        for environment_variable in self.environment_variables:
+            if environment_variable.get("name") == "HOME":
+                home_target = None
+                continue
+            for tmp_directory_var in self.tmp_directory_vars:
+                if environment_variable.get("name") == tmp_directory_var:
+                    tmp_target = None
+                    break
+        self.home_target = home_target
+        self.tmp_target = tmp_target
+        self.docker_env_pass_through = tool_source.parse_docker_env_pass_through()
 
         # Parameters used to build URL for redirection to external app
         redirect_url_params = tool_source.parse_redirect_url_params_elem()

lib/galaxy/tools/deps/containers.py

@@ -283,21 +283,24 @@ class ToolInfo(object):
     # variables they can consume (e.g. JVM options, license keys, etc..)
     # and add these to env_path_through
 
-    def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False):
+    def __init__(self, container_descriptions=[], requirements=[], requires_galaxy_python_environment=False, env_pass_through=["GALAXY_SLOTS"]):
         self.container_descriptions = container_descriptions
         self.requirements = requirements
         self.requires_galaxy_python_environment = requires_galaxy_python_environment
-        self.env_pass_through = ["GALAXY_SLOTS"]
+        self.env_pass_through = env_pass_through
 
 
 class JobInfo(object):
 
-    def __init__(self, working_directory, tool_directory, job_directory, job_directory_type):
+    def __init__(
+        self, working_directory, tool_directory, job_directory, tmp_directory, job_directory_type
+    ):
         self.working_directory = working_directory
         self.job_directory = job_directory
         # Tool files may be remote staged - so this is unintuitively a property
         # of the job not of the tool.
         self.tool_directory = tool_directory
+        self.tmp_directory = tmp_directory
         self.job_directory_type = job_directory_type  # "galaxy" or "pulsar"
 
 
@@ -394,6 +397,7 @@ class HasDockerLikeVolumes:
                 variables[name] = os.path.abspath(value)
 
         add_var("working_directory", self.job_info.working_directory)
+        add_var("tmp_directory", self.job_info.tmp_directory)
         add_var("job_directory", self.job_info.job_directory)
         add_var("tool_directory", self.job_info.tool_directory)
         add_var("galaxy_root", self.app_info.galaxy_root_dir)
@@ -408,6 +412,8 @@ class HasDockerLikeVolumes:
             defaults = "$galaxy_root:default_ro,$tool_directory:default_ro"
             if self.job_info.job_directory:
                 defaults += ",$job_directory:default_ro"
+            if self.job_info.tmp_directory is not None:
+                defaults += ",$tmp_directory:rw"
             if self.app_info.outputs_to_working_directory:
                 # Should need default_file_path (which is of course an estimate given
                 # object stores anyway).
@@ -454,6 +460,11 @@ class DockerContainer(Container, HasDockerLikeVolumes):
         preprocessed_volumes_str = preprocess_volumes(volumes_raw, self.container_type)
         # TODO: Remove redundant volumes...
         volumes = docker_util.DockerVolume.volumes_from_str(preprocessed_volumes_str)
+        # If a tool definitely has a temp directory available set it to /tmp in container for compat.
+        # with CWL. This is part of that spec and should make it easier to share containers between CWL
+        # and Galaxy.
+        if self.job_info.tmp_directory is not None:
+            volumes.append(docker_util.DockerVolume.volume_from_str("%s:/tmp:rw" % self.job_info.tmp_directory))
         volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM)
 
         docker_host_props = dict(

lib/galaxy/tools/deps/docker_util.py

@@ -155,7 +155,9 @@ def build_docker_run_command(
     if terminal:
         command_parts.append("-t")
     for env_directive in env_directives:
-        command_parts.extend(["-e", shlex_quote(env_directive)])
+        # e.g. -e "GALAXY_SLOTS=$GALAXY_SLOTS"
+        # These are environment variable expansions so we don't quote these.
+        command_parts.extend(["-e", env_directive])
     for volume in volumes:
         command_parts.extend(["-v", shlex_quote(str(volume))])
     if volumes_from:
@@ -175,8 +177,13 @@ def build_docker_run_command(
     if set_user:
         user = set_user
         if set_user == DEFAULT_SET_USER:
-            user = str(os.geteuid())
-        command_parts.extend(["-u", user])
+            # If future-us is ever in here and fixing this for docker-machine just
+            # use cwltool.docker_id - it takes care of this default nicely.
+            euid = os.geteuid()
+            egid = os.getgid()
+
+            user = "%d:%d" % (euid, egid)
+        command_parts.extend(["--user", user])
     full_image = image
     if tag:
         full_image = "%s:%s" % (full_image, tag)

lib/galaxy/tools/evaluation.py

@@ -521,6 +521,15 @@ class ToolEvaluator(object):
             environment_variable["raw"] = True
             environment_variables.append(environment_variable)
 
+        home_dir = self.compute_environment.home_directory()
+        tmp_dir = self.compute_environment.tmp_directory()
+        if home_dir:
+            environment_variable = dict(name="HOME", value='"%s"' % home_dir, raw=True)
+            environment_variables.append(environment_variable)
+        if tmp_dir:
+            for tmp_directory_var in self.tool.tmp_directory_vars:
+                environment_variable = dict(name=tmp_directory_var, value='"%s"' % tmp_dir, raw=True)
+                environment_variables.append(environment_variable)
         self.environment_variables = environment_variables
         return environment_variables
 

lib/galaxy/tools/parser/interface.py

@@ -91,6 +91,23 @@ class ToolSource(object):
         """ Return environment variable templates to expose.
         """
 
+    def parse_home_target(self):
+        """Should be "job_home", "shared_home", "job_tmp", "pwd", or None.
+        """
+        return "pwd"
+
+    def parse_tmp_target(self):
+        """Should be "pwd", "shared_home", "job_tmp", "job_tmp_if_explicit", or None.
+        """
+        return "job_tmp"
+
+    def parse_tmp_directory_vars(self):
+        """Directories to override if a tmp_target is not None."""
+        return ["TMPDIR", "TMP", "TEMP"]
+
+    def parse_docker_env_pass_through(self):
+        return ["GALAXY_SLOTS", "HOME"] + self.parse_tmp_directory_vars()
+
     @abstractmethod
     def parse_interpreter(self):
         """ Return string containing the interpreter to prepend to the command

lib/galaxy/tools/parser/xml.py

@@ -126,6 +126,27 @@ class XmlToolSource(ToolSource):
             )
         return environment_variables
 
+    def parse_home_target(self):
+        target = "job_home" if self.parse_profile() >= "18.01" else "shared_home"
+        command_el = self._command_el
+        command_legacy = (command_el is not None) and command_el.get("use_shared_home", None)
+        if command_legacy is not None:
+            target = "shared_home" if string_as_bool(command_legacy) else "job_home"
+        return target
+
+    def parse_tmp_target(self):
+        # Default to not touching TMPDIR et. al. but if job_tmp is set
+        # in job_conf then do. This is a very conservative approach that shouldn't
+        # break or modify any configurations by default.
+        return "job_tmp_if_explicit"
+
+    def parse_docker_env_pass_through(self):
+        if self.parse_profile() < "18.01":
+            return ["GALAXY_SLOTS"]
+        else:
+            # Pass home, etc...
+            return super(XmlToolSource, self).parse_docker_env_pass_through()
+
     def parse_interpreter(self):
         interpreter = None
         command_el = self._command_el

lib/galaxy/tools/xsd/galaxy.xsd

@@ -2706,6 +2706,11 @@ deprecated and using the ``$__tool_directory__`` variable is superior.
             <xs:documentation>Only used if ``detect_errors="exit_code", tells Galaxy the specified exit code indicates an out of memory error. Galaxy instances may be configured to retry such jobs on resources with more memory.</xs:documentation>
           </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="use_shared_home" type="xs:string">
+          <xs:annotation>
+            <xs:documentation>When running a job for this tool, do not isolate its $HOME directory within the job's directory - use either the shared_home_dir setting in Galaxy or the default $HOME specified in the job's default environment.</xs:documentation>
+          </xs:annotation>
+        </xs:attribute>
         <xs:attribute name="interpreter" type="xs:string" gxdocs:deprecated="true">
           <xs:annotation>
             <xs:documentation xml:lang="en">This attribute defines the programming language in which the tool's executable file is written. Any language can be used (tools can be written in Python, C, Perl, Java, etc.). The executable file must be in the same directory of the XML file. If instead this attribute is not specified, the tag content should be a Bash command calling executable(s) available in the $PATH. </xs:documentation>

test/base/driver_util.py

@@ -133,6 +133,8 @@ def setup_galaxy_config(
     conda_auto_install=False
 ):
     """Setup environment and build config for test Galaxy instance."""
+    # For certain docker operations this needs to be evaluated out - e.g. for cwltool.
+    tmpdir = os.path.realpath(tmpdir)
     if not os.path.exists(tmpdir):
         os.makedirs(tmpdir)
     file_path = os.path.join(tmpdir, 'files')

test/base/integration_util.py

@@ -7,6 +7,7 @@ tessting configuration.
 import os
 from unittest import skip, TestCase
 
+from galaxy.tools.deps.commands import which
 from .api import UsesApiTestCaseMixin
 from .driver_util import GalaxyTestDriver
 
@@ -21,6 +22,16 @@ def skip_if_jenkins(cls):
     return cls
 
 
+def skip_unless_executable(executable):
+    if which(executable):
+        return lambda func: func
+    return skip("PATH doesn't contain executable %s" % executable)
+
+
+def skip_unless_docker():
+    return skip_unless_executable("docker")
+
+
 class IntegrationTestCase(TestCase, UsesApiTestCaseMixin):
     """Unit test case with utilities for spinning up Galaxy."""
 

test/functional/tools/job_environment_default.xml

@@ -0,0 +1,23 @@
+<tool id="job_environment_default" name="job_environment_default" version="0.1.0" profile="18.01">
+    <requirements>
+      <container type="docker">busybox:ubuntu-14.04</container>
+    </requirements>
+    <command><![CDATA[
+      echo `id -u` > '$user_id';
+      echo `id -g` > '$group_id';
+      echo `pwd` > '$pwd';
+      echo "\$HOME" > '$home';
+      echo "\$TMP"  > '$tmp';
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+        <data name="user_id" format="txt" label="user_id" />
+        <data name="group_id" format="txt" label="group_id" />
+        <data name="pwd" format="txt" label="pwd" />
+        <data name="home" format="txt" label="home" />
+        <data name="tmp" format="txt" label="tmp" />
+    </outputs>
+    <help>
+    </help>
+</tool>

test/functional/tools/job_environment_default_legacy.xml

@@ -0,0 +1,23 @@
+<tool id="job_environment_default_legacy" name="job_environment_default_legacy" version="0.1.0">
+    <requirements>
+      <container type="docker">busybox:ubuntu-14.04</container>
+    </requirements>
+    <command><![CDATA[
+      echo `id -u` > '$user_id';
+      echo `id -g` > '$group_id';
+      echo `pwd` > '$pwd';
+      echo "\$HOME" > '$home';
+      echo "\$TMP"  > '$tmp';
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+        <data name="user_id" format="txt" label="user_id" />
+        <data name="group_id" format="txt" label="group_id" />
+        <data name="pwd" format="txt" label="pwd" />
+        <data name="home" format="txt" label="home" />
+        <data name="tmp" format="txt" label="tmp" />
+    </outputs>
+    <help>
+    </help>
+</tool>

test/functional/tools/job_environment_explicit_shared_home.xml

@@ -0,0 +1,23 @@
+<tool id="job_environment_explicit_shared_home" name="job_environment_explicit_shared_home" version="0.1.0" profile="18.01">
+    <requirements>
+      <container type="docker">busybox:ubuntu-14.04</container>
+    </requirements>
+    <command use_shared_home="true"><![CDATA[
+      echo `id -u` > '$user_id';
+      echo `id -g` > '$group_id';
+      echo `pwd` > '$pwd';
+      echo "\$HOME" > '$home';
+      echo "\$TMP"  > '$tmp';
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+        <data name="user_id" format="txt" label="user_id" />
+        <data name="group_id" format="txt" label="group_id" />
+        <data name="pwd" format="txt" label="pwd" />
+        <data name="home" format="txt" label="home" />
+        <data name="tmp" format="txt" label="tmp" />
+    </outputs>
+    <help>
+    </help>
+</tool>

test/functional/tools/mulled_example_explicit.xml

@@ -0,0 +1,22 @@
+<tool id="mulled_example_explicit" name="mulled_example_explicit" version="0.1.0">
+    <requirements>
+        <requirement type="package" version="0.7.15">bwa</requirement>
+        <container type="docker">quay.io/biocontainers/bwa:0.7.15--0</container>
+    </requirements>
+    <stdio>
+        <exit_code range="2:" />
+    </stdio>
+    <command><![CDATA[
+        bwa > $output_1 2>&1
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+      <data name="output_1" />
+    </outputs>
+    <help><![CDATA[
+        TODO: Fill in help.
+    ]]></help>
+    <tests>
+    </tests>
+</tool>

test/functional/tools/mulled_example_simple.xml

@@ -0,0 +1,21 @@
+<tool id="mulled_example_simple" name="mulled_example_simple" version="0.1.0">
+    <requirements>
+        <requirement type="package" version="0.7.15">bwa</requirement>
+    </requirements>
+    <stdio>
+        <exit_code range="2:" />
+    </stdio>
+    <command><![CDATA[
+        bwa > $output_1 2>&1
+    ]]></command>
+    <inputs>
+    </inputs>
+    <outputs>
+      <data name="output_1" />
+    </outputs>
+    <help><![CDATA[
+        TODO: Fill in help.
+    ]]></help>
+    <tests>
+    </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -52,6 +52,9 @@
   <tool file="maxseconds.xml" />
   -->
   <tool file="job_properties.xml" />
+  <tool file="job_environment_default.xml" />
+  <tool file="job_environment_default_legacy.xml" />
+  <tool file="job_environment_explicit_shared_home.xml" />
   <tool file="version_command_plain.xml" />
   <tool file="version_command_interpreter.xml" />
   <tool file="version_command_tool_dir.xml" />
@@ -161,6 +164,8 @@
   <tool file="mulled_example_multi_versionless.xml" />
   <tool file="mulled_example_multi_incomplete.xml" />
   <tool file="mulled_example_conflict.xml" />
+  <tool file="mulled_example_simple.xml" />
+  <tool file="mulled_example_explicit.xml" />
 
   <tool file="simple_constructs.yml" />
 

test/integration/dockerized_job_conf.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
+    </plugins>
+
+    <handlers>
+        <handler id="main"/>
+    </handlers>
+
+    <destinations default="local_docker">
+        <destination id="local_docker" runner="local">
+            <param id="docker_enabled">true</param>
+            <param id="docker_sudo">false</param>
+        </destination>
+
+        <destination id="local_upload" runner="local">
+        </destination>
+
+    </destinations>
+
+    <tools>
+        <tool id="upload1" destination="local_upload" />
+    </tools>
+
+</job_conf>

test/integration/sets_tmp_dir_expression_job_conf.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
+    </plugins>
+
+    <handlers>
+        <handler id="main"/>
+    </handlers>
+
+    <destinations>
+        <destination id="local_dest" runner="local">
+            <param id="tmp_dir">$(mktemp cooltmpXXXXXXXXXXXX)</param>
+        </destination>
+    </destinations>
+
+</job_conf>

test/integration/sets_tmp_dir_to_true_job_conf.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
+    </plugins>
+
+    <handlers>
+        <handler id="main"/>
+    </handlers>
+
+    <destinations>
+        <destination id="local_dest" runner="local">
+            <param id="tmp_dir">True</param>
+        </destination>
+    </destinations>
+
+</job_conf>

test/integration/simple_job_conf.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
+    </plugins>
+
+    <handlers>
+        <handler id="main"/>
+    </handlers>
+
+    <destinations>
+        <destination id="local_dest" runner="local">
+        </destination>
+    </destinations>
+
+</job_conf>

test/integration/test_dockerized_jobs.py

@@ -0,0 +1,73 @@
+"""Integration tests for running tools in Docker containers."""
+
+import os
+import tempfile
+
+from base import integration_util
+from base.populators import (
+    DatasetPopulator,
+)
+
+from .test_job_environments import RunsEnvironmentJobs
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+DOCKERIZED_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "dockerized_job_conf.xml")
+
+
+@integration_util.skip_unless_docker()
+class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs):
+
+    framework_tool_and_types = True
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = tempfile.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = DOCKERIZED_JOB_CONFIG_FILE
+        # Disable tool dependency resolution.
+        config["tool_dependency_dir"] = "none"
+        config["enable_beta_mulled_containers"] = "true"
+
+    def setUp(self):
+        super(DockerizedJobsIntegrationTestCase, self).setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+        self.history_id = self.dataset_populator.new_history()
+
+    def test_explicit(self):
+        self.dataset_populator.run_tool("mulled_example_explicit", {}, self.history_id)
+        self.dataset_populator.wait_for_history(self.history_id, assert_ok=True)
+        output = self.dataset_populator.get_history_dataset_content(self.history_id)
+        assert "0.7.15-r1140" in output
+
+    def test_mulled_simple(self):
+        self.dataset_populator.run_tool("mulled_example_simple", {}, self.history_id)
+        self.dataset_populator.wait_for_history(self.history_id, assert_ok=True)
+        output = self.dataset_populator.get_history_dataset_content(self.history_id)
+        assert "0.7.15-r1140" in output
+
+    def test_docker_job_enviornment(self):
+        job_env = self._run_and_get_environment_properties("job_environment_default")
+
+        euid = os.geteuid()
+        egid = os.getgid()
+
+        assert job_env.user_id == str(euid), job_env.user_id
+        assert job_env.group_id == str(egid), job_env.group_id
+        assert job_env.pwd.startswith(self.jobs_directory)
+        assert job_env.pwd.endswith("/working")
+        assert job_env.home.startswith(self.jobs_directory)
+        assert job_env.home.endswith("/home")
+
+    def test_docker_job_environment_legacy(self):
+        job_env = self._run_and_get_environment_properties("job_environment_default_legacy")
+
+        euid = os.geteuid()
+        egid = os.getgid()
+
+        assert job_env.user_id == str(euid), job_env.user_id
+        assert job_env.group_id == str(egid), job_env.group_id
+        assert job_env.pwd.startswith(self.jobs_directory)
+        assert job_env.pwd.endswith("/working")
+        # Should we change env_pass_through to just always include TMP and HOME for docker?
+        # I'm not sure, if yes this would change.
+        assert job_env.home == "/", job_env.home

test/integration/test_job_environments.py

@@ -0,0 +1,165 @@
+"""Integration tests for the Pulsar embedded runner."""
+
+import collections
+import os
+import tempfile
+
+from base import integration_util
+from base.populators import (
+    DatasetPopulator,
+    skip_without_tool,
+)
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+SIMPLE_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "simple_job_conf.xml")
+SETS_TMP_DIR_TO_TRUE_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "sets_tmp_dir_to_true_job_conf.xml")
+SETS_TMP_DIR_AS_EXPRESSION_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "sets_tmp_dir_expression_job_conf.xml")
+
+JobEnviromentProperties = collections.namedtuple("JobEnvironmentProperties", [
+    "user_id",
+    "group_id",
+    "pwd",
+    "home",
+    "tmp",
+])
+
+
+class RunsEnvironmentJobs:
+
+    def _run_and_get_environment_properties(self, tool_id="job_environment_default"):
+        with self.dataset_populator.test_history() as history_id:
+            self.dataset_populator.run_tool(tool_id, {}, history_id)
+            self.dataset_populator.wait_for_history(history_id, assert_ok=True)
+            return self._environment_properties(history_id)
+
+    def _environment_properties(self, history_id):
+        user_id = self.dataset_populator.get_history_dataset_content(history_id, hid=1).strip()
+        group_id = self.dataset_populator.get_history_dataset_content(history_id, hid=2).strip()
+        pwd = self.dataset_populator.get_history_dataset_content(history_id, hid=3).strip()
+        home = self.dataset_populator.get_history_dataset_content(history_id, hid=4).strip()
+        tmp = self.dataset_populator.get_history_dataset_content(history_id, hid=5).strip()
+
+        return JobEnviromentProperties(user_id, group_id, pwd, home, tmp)
+
+
+class BaseJobEnvironmentIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs):
+
+    framework_tool_and_types = True
+
+    def setUp(self):
+        super(BaseJobEnvironmentIntegrationTestCase, self).setUp()
+        self.dataset_populator = DatasetPopulator(self.galaxy_interactor)
+
+
+class DefaultJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = tempfile.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = SIMPLE_JOB_CONFIG_FILE  # Ensure no Docker for these tests
+
+    @skip_without_tool("job_environment_default")
+    def test_default_environment_1801(self):
+        job_env = self._run_and_get_environment_properties()
+
+        euid = os.geteuid()
+        egid = os.getgid()
+
+        assert job_env.user_id == str(euid), job_env.user_id
+        assert job_env.group_id == str(egid), job_env.group_id
+        assert job_env.pwd.startswith(self.jobs_directory)
+        assert job_env.pwd.endswith("/working")
+
+        # Newer tools have isolated home directories in job_directory/home
+        job_directory = os.path.dirname(job_env.pwd)
+        assert job_env.home == os.path.join(job_directory, "home"), job_env.home
+
+        # Since job_conf doesn't set tmp_dir parameter - temp isn't in job_directory
+        assert not job_env.tmp.startswith(job_directory)
+
+    @skip_without_tool("job_environment_default_legacy")
+    def test_default_environment_legacy(self):
+        job_env = self._run_and_get_environment_properties("job_environment_default_legacy")
+
+        euid = os.geteuid()
+        egid = os.getgid()
+        home = os.getenv("HOME")
+
+        assert job_env.user_id == str(euid), job_env.user_id
+        assert job_env.group_id == str(egid), job_env.group_id
+        assert job_env.home == home, job_env.home
+
+    @skip_without_tool("job_environment_explicit_shared_home")
+    def test_default_environment_force_legacy_home(self):
+        # Home should not overridden because we haven't set legacy_home_dir in job_conf
+        # or app, so it should just HOME.
+        job_env = self._run_and_get_environment_properties("job_environment_explicit_shared_home")
+        home = os.getenv("HOME")
+        assert job_env.home == home, job_env.home
+
+
+class TmpDirToTrueJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = tempfile.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = SETS_TMP_DIR_TO_TRUE_JOB_CONFIG
+
+    @skip_without_tool("job_environment_default")
+    def test_default_environment_1801(self):
+        job_env = self._run_and_get_environment_properties()
+
+        job_directory = os.path.dirname(job_env.pwd)
+
+        # Since job_conf sets tmp_dir parameter to True - temp is in job_directory
+        assert job_env.tmp.startswith(job_directory), job_env
+
+
+class TmpDirAsShellCommandJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = tempfile.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = SETS_TMP_DIR_AS_EXPRESSION_JOB_CONFIG
+
+    @skip_without_tool("job_environment_default")
+    def test_default_environment_1801(self):
+        job_env = self._run_and_get_environment_properties()
+
+        # Since job_conf sets tmp_dir parameter to $(mktemp cooltmpXXXXXXXXXXXX) should
+        # start with cooltmp.
+        basename = os.path.basename(job_env.tmp)
+        assert basename.startswith("cooltmp"), job_env.tmp
+
+
+class SharedHomeJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = tempfile.mkdtemp()
+        cls.shared_home_directory = tempfile.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = SIMPLE_JOB_CONFIG_FILE  # Ensure no Docker for these tests
+        config["shared_home_dir"] = cls.shared_home_directory
+
+    @skip_without_tool("job_environment_default")
+    def test_default_environment(self):
+        # Test shared_home_dir ignored for newer tools by default
+        job_env = self._run_and_get_environment_properties()
+        job_directory = os.path.dirname(job_env.pwd)
+        assert job_env.home == os.path.join(job_directory, "home"), job_env.home
+
+    @skip_without_tool("job_environment_default_legacy")
+    def test_default_environment_legacy(self):
+        # shared_home_dir used by default for older tools
+        job_env = self._run_and_get_environment_properties("job_environment_default_legacy")
+        assert job_env.home == self.shared_home_directory, job_env.home
+
+    @skip_without_tool("job_environment_explicit_shared_home")
+    def test_default_environment_force_legacy_home(self):
+        # shared_home_dir used for newer tools if forced in tool XML
+        job_env = self._run_and_get_environment_properties("job_environment_explicit_shared_home")
+        assert job_env.home == self.shared_home_directory, job_env.home

test/unit/jobs/test_runner_local.py

@@ -147,6 +147,7 @@ class MockJobWrapper(object):
         self.galaxy_virtual_env = None
         self.shell = "/bin/bash"
         self.cleanup_job = "never"
+        self.tmp_dir_creation_statement = ""
 
         # Cruft for setting metadata externally, axe at some point.
         self.external_output_metadata = bunch.Bunch(
@@ -210,3 +211,9 @@ class MockJobWrapper(object):
         self.stdout = stdout
         self.stderr = stderr
         self.exit_code = exit_code
+
+    def tmp_directory(self):
+        return None
+
+    def home_directory(self):
+        return None

test/unit/tools/test_evaluation.py

@@ -243,6 +243,12 @@ class TestComputeEnviornment(SimpleComputeEnvironment):
     def working_directory(self):
         return self._working_directory
 
+    def home_directory(self):
+        return self._working_directory
+
+    def tmp_directory(self):
+        return self._working_directory
+
     def new_file_path(self):
         return self._new_file_path
 
@@ -293,6 +299,14 @@ class MockTool(object):
             output1=ToolOutput("output1"),
         )
 
+    @property
+    def config_file(self):
+        return self._config_files[0]
+
+    @property
+    def tmp_directory_vars(self):
+        return ["TMP"]
+
     @property
     def config_files(self):
         return self._config_files