diff --git a/lib/galaxy/jobs/runners/condor.py b/lib/galaxy/jobs/runners/condor.py index 23518538882..0aa81a8a65f 100644 --- a/lib/galaxy/jobs/runners/condor.py +++ b/lib/galaxy/jobs/runners/condor.py @@ -83,9 +83,9 @@ class CondorJobRunner(AsynchronousJobRunner): galaxy_slots = query_params.get('request_cpus', None) if galaxy_slots: - galaxy_slots_statement = 'GALAXY_SLOTS="%s"; export GALAXY_SLOTS_CONFIGURED="1"' % galaxy_slots + galaxy_slots_statement = 'GALAXY_SLOTS="%s"; export GALAXY_SLOTS; GALAXY_SLOTS_CONFIGURED="1"; export GALAXY_SLOTS_CONFIGURED;' % galaxy_slots else: - galaxy_slots_statement = 'GALAXY_SLOTS="1"' + galaxy_slots_statement = 'GALAXY_SLOTS="1"; export GALAXY_SLOTS;' # define job attributes cjs = CondorJobState( diff --git a/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh index 01dccf22c5d..06e0cbc0adb 100644 --- a/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh @@ -30,3 +30,4 @@ else GALAXY_SLOTS="1" unset GALAXY_SLOTS_CONFIGURED fi +export GALAXY_SLOTS diff --git a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh index 471682827c5..065b1cb7ba1 100644 --- a/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh +++ b/lib/galaxy/jobs/runners/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh @@ -27,13 +27,14 @@ _galaxy_setup_environment() { $integrity_injection $slots_statement -export GALAXY_SLOTS export PYTHONWARNINGS="ignore" GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" _GALAXY_VIRTUAL_ENV="$galaxy_virtual_env" PRESERVE_GALAXY_ENVIRONMENT="$preserve_python_environment" GALAXY_LIB="$galaxy_lib" _galaxy_setup_environment "$PRESERVE_GALAXY_ENVIRONMENT" +export _GALAXY_JOB_HOME_DIR +export _GALAXY_JOB_TMP_DIR GALAXY_PYTHON=`command -v python` cd $working_directory $memory_statement diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index f4cf6a11764..303f936e816 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -80,7 +80,7 @@ def job_script(template=DEFAULT_JOB_FILE_TEMPLATE, **kwds): >>> script.startswith('#!/bin/bash\\n\\n#PBS -test\\n') True >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', slots_statement='GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"') - >>> script.find('GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"\\nexport GALAXY_SLOTS\\n') > 0 + >>> script.find('GALAXY_SLOTS="$SLURM_JOB_NUM_NODES"\\n') > 0 True >>> script = job_script(working_directory='wd', command='uptime', exit_code_path='ec', memory_statement='GALAXY_MEMORY_MB="32768"') >>> script.find('GALAXY_MEMORY_MB="32768"\\n') > 0 diff --git a/lib/galaxy/tool_util/deps/container_classes.py b/lib/galaxy/tool_util/deps/container_classes.py index e2da1e211eb..4702ccecfa5 100644 --- a/lib/galaxy/tool_util/deps/container_classes.py +++ b/lib/galaxy/tool_util/deps/container_classes.py @@ -200,7 +200,6 @@ class HasDockerLikeVolumes: add_var("library_import_dir", self.app_info.library_import_dir) add_var('tool_data_path', self.app_info.tool_data_path) add_var('shed_tool_data_path', self.app_info.shed_tool_data_path) - if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar": # We have a Pulsar job directory, so everything needed (excluding index # files) should be available in job_directory... @@ -248,7 +247,6 @@ class HasDockerLikeVolumes: if end_index < 0: end_index = len(volumes_str) volumes_str = volumes_str[0:tool_directory_index] + volumes_str[end_index:len(volumes_str)] - return volumes_str @@ -299,8 +297,9 @@ class DockerContainer(Container, HasDockerLikeVolumes): # and Galaxy. if self.job_info.tmp_directory is not None: volumes.append(DockerVolume.from_str("%s:/tmp:rw" % self.job_info.tmp_directory)) + else: + volumes.append(DockerVolume.from_str("$_GALAXY_JOB_TMP_DIR:$_GALAXY_JOB_TMP_DIR:rw")) volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM) - docker_host_props = self.docker_host_props cached_image_file = self.__get_cached_image_file() diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 68a33e352c9..3654bd1c26a 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -3162,6 +3162,7 @@ Name | Description ``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime). ``\$GALAXY_MEMORY_MB`` | Total amount of memory in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. ``\$GALAXY_MEMORY_MB_PER_SLOT`` | Amount of memory per slot in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. +``\$_GALAXY_JOB_TMP_DIR`` | Path to an empty directory in the job's working directory that can be used as a temporary directory. See the [Planemo docs](https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage) on the topic of ``GALAXY_SLOTS`` for more information and examples. diff --git a/packages/objectstore/HISTORY.rst b/packages/objectstore/HISTORY.rst index 08c9654f4a5..569ebced7b2 100644 --- a/packages/objectstore/HISTORY.rst +++ b/packages/objectstore/HISTORY.rst @@ -6,7 +6,13 @@ History .. to_doc --------------------- -20.9.1.dev0 +20.9.2.dev0 +--------------------- + + + +--------------------- +20.9.1 (2021-03-01) --------------------- diff --git a/packages/objectstore/galaxy/project_galaxy_objectstore.py b/packages/objectstore/galaxy/project_galaxy_objectstore.py index 0ba6a66904f..02e5318ca69 100644 --- a/packages/objectstore/galaxy/project_galaxy_objectstore.py +++ b/packages/objectstore/galaxy/project_galaxy_objectstore.py @@ -1,4 +1,4 @@ -__version__ = '20.9.1.dev0' +__version__ = '20.9.2.dev0' PROJECT_NAME = "galaxy-objectstore" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/test/functional/tools/job_environment_default.xml b/test/functional/tools/job_environment_default.xml index 46d87b26b57..f733d2623b6 100644 --- a/test/functional/tools/job_environment_default.xml +++ b/test/functional/tools/job_environment_default.xml @@ -10,7 +10,9 @@ echo \$(id -g) > '$group_id' && echo \$(pwd) > '$pwd' && echo "\$HOME" > '$home' && echo "\$TMP" > '$tmp' && -echo "\$SOME_ENV_VAR" > '$some_env_var' +echo "\$SOME_ENV_VAR" > '$some_env_var' && +touch "\$_GALAXY_JOB_TMP_DIR/tmp_test" && +touch "\$HOME/home_test" ]]> diff --git a/test/integration/chained_dyndest_rules/module3/rules.py b/test/integration/chained_dyndest_rules/module3/rules.py index 72b3f9b68f5..61fd78aba41 100644 --- a/test/integration/chained_dyndest_rules/module3/rules.py +++ b/test/integration/chained_dyndest_rules/module3/rules.py @@ -14,6 +14,6 @@ def dyndest_chain_2(): def dyndest_chain_3(tmp_dir_prefix_two): - tmp_dir = '$(mktemp %sand3XXXXXXXXXXXX)' % tmp_dir_prefix_two + tmp_dir = '$(mktemp -d -p $(pwd) %sand3XXXXXXXXXXXX)' % tmp_dir_prefix_two return JobDestination(runner="local", params={'tmp_dir': tmp_dir}) diff --git a/test/integration/sets_tmp_dir_expression_job_conf.xml b/test/integration/sets_tmp_dir_expression_job_conf.xml index b4be01bda77..ca5de21f526 100644 --- a/test/integration/sets_tmp_dir_expression_job_conf.xml +++ b/test/integration/sets_tmp_dir_expression_job_conf.xml @@ -6,7 +6,7 @@ - $(mktemp cooltmpXXXXXXXXXXXX) + $(mktemp -d -p $(pwd) cooltmpXXXXXXXXXXXX)