Merge branch 'release_21.05' into dev

client/src/components/Libraries/LibrariesList.vue

@@ -89,7 +89,7 @@
                     <font-awesome-icon icon="unlock" />
                     Undelete
                 </b-button>
-                <div v-else>
+                <div v-else-if="isAdmin">
                     <b-button
                         v-if="row.item.can_user_modify && row.item.editMode"
                         size="sm"

lib/galaxy/dependencies/dev-requirements.txt

@@ -144,7 +144,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.5
+pulsar-galaxy-lib==0.14.6
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -127,7 +127,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.5
+pulsar-galaxy-lib==0.14.6
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"

lib/galaxy/jobs/runners/pulsar.py

@@ -1003,6 +1003,8 @@ class PulsarComputeEnvironment(ComputeEnvironment):
         self._working_directory = remote_job_config["working_directory"]
         self._sep = remote_job_config["system_properties"]["separator"]
         self._tool_dir = remote_job_config["tools_directory"]
+        self._tmp_dir = remote_job_config.get('tmp_dir')
+        self._shared_home_dir = remote_job_config.get('shared_home_dir')
         version_path = self.local_path_config.version_path()
         new_version_path = self.path_mapper.remote_version_path_rewrite(version_path)
         if new_version_path:
@@ -1092,14 +1094,23 @@ class PulsarComputeEnvironment(ComputeEnvironment):
         return self._tool_dir
 
     def home_directory(self):
-        # TODO: revisit and implement this, won't break anything working in the
-        # meantime.
-        return None
+        return self._target_to_directory(self.job_wrapper.home_target)
 
     def tmp_directory(self):
-        # TODO: revisit and implement this, won't break anything working in the
-        # meantime.
-        return None
+        return self._target_to_directory(self.job_wrapper.tmp_target)
+
+    def _target_to_directory(self, target):
+        tmp_dir = self._tmp_dir
+        if target is None or (target == "job_tmp_if_explicit" and tmp_dir is None):
+            return None
+        elif target in ["job_tmp", "job_tmp_if_explicit"]:
+            return "$_GALAXY_JOB_TMP_DIR"
+        elif target == "shared_home":
+            return self._shared_home_dir
+        elif target == "job_home":
+            return "$_GALAXY_JOB_HOME_DIR"
+        else:
+            raise Exception(f"Unknown target type [{target}]")
 
     def galaxy_url(self):
         return self.job_wrapper.get_destination_configuration("galaxy_infrastructure_url")

lib/galaxy/tool_util/deps/dependencies.py

@@ -42,7 +42,7 @@ class ToolInfo:
 
     def __init__(self, container_descriptions=None, requirements=None, requires_galaxy_python_environment=False, env_pass_through=None, guest_ports=None, tool_id=None, tool_version=None, profile=-1):
         if env_pass_through is None:
-            env_pass_through = ["GALAXY_SLOTS"]
+            env_pass_through = ["GALAXY_SLOTS", "GALAXY_MEMORY_MB", "GALAXY_MEMORY_MB_PER_SLOT"]
         if container_descriptions is None:
             container_descriptions = []
         if requirements is None:

lib/galaxy/tool_util/parser/interface.py

@@ -110,7 +110,8 @@ class ToolSource(metaclass=ABCMeta):
         return ["TMPDIR", "TMP", "TEMP"]
 
     def parse_docker_env_pass_through(self):
-        return ["GALAXY_SLOTS", "HOME", "_GALAXY_JOB_HOME_DIR", "_GALAXY_JOB_TMP_DIR"] + self.parse_tmp_directory_vars()
+        return ["GALAXY_SLOTS", "GALAXY_MEMORY_MB", "GALAXY_MEMORY_MB_PER_SLOT", "HOME",
+                "_GALAXY_JOB_HOME_DIR", "_GALAXY_JOB_TMP_DIR"] + self.parse_tmp_directory_vars()
 
     @abstractmethod
     def parse_interpreter(self):

packages/tool_util/HISTORY.rst

@@ -6,7 +6,13 @@ History
 .. to_doc
 
 ---------------------
-21.1.1.dev0
+21.1.2.dev0
+---------------------
+
+
+
+---------------------
+21.1.1 (2021-05-21)
 ---------------------
 
 

test/integration/test_job_environments.py

@@ -15,6 +15,7 @@ SIMPLE_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "simple_job_conf.xml")
 IO_INJECTION_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "io_injection_job_conf.yml")
 SETS_TMP_DIR_TO_TRUE_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "sets_tmp_dir_to_true_job_conf.xml")
 SETS_TMP_DIR_AS_EXPRESSION_JOB_CONFIG = os.path.join(SCRIPT_DIRECTORY, "sets_tmp_dir_expression_job_conf.xml")
+EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_job_conf.xml")
 
 JobEnvironmentProperties = collections.namedtuple("JobEnvironmentProperties", [
     "user_id",
@@ -115,6 +116,35 @@ class DefaultJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTest
         assert job_env.home == os.path.join(job_directory, "home"), job_env.home
 
 
+class EmbeddedPulsarDefaultJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        cls.jobs_directory = tempfile.mkdtemp()
+        config["jobs_directory"] = cls.jobs_directory
+        config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
+
+    @skip_without_tool("job_environment_default")
+    def test_default_environment_1801(self):
+        job_env = self._run_and_get_environment_properties()
+
+        euid = os.geteuid()
+        egid = os.getgid()
+
+        assert job_env.user_id == str(euid), job_env.user_id
+        assert job_env.group_id == str(egid), job_env.group_id
+        assert 'pulsar_staging' in job_env.pwd
+        assert job_env.pwd.endswith("/working")
+        # job ran in embedded pulsar, not local job dir
+        assert self.jobs_directory not in job_env.pwd
+
+        # Newer tools get isolated home directories in job_directory/home
+        job_directory = os.path.dirname(job_env.pwd)
+        assert os.path.abspath(job_env.home) == os.path.join(job_directory, "home"), job_env.home
+
+        # Since job_conf doesn't set tmp_dir parameter - temp isn't in job_directory
+        assert not job_env.tmp.startswith(job_directory)
+
+
 class TmpDirToTrueJobEnvironmentIntegrationTestCase(BaseJobEnvironmentIntegrationTestCase):
 
     @classmethod