From ee76c06da486e47c87fb4b8c236fa2191a403980 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 1 Mar 2018 13:14:56 +0100 Subject: [PATCH 1/6] added GALAXY_MEMORY_MB_PERSLOT --- .../jobs/runners/util/job_script/MEMORY_STATEMENT.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh index fef76e5fff9..f4f383521c6 100644 --- a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh @@ -1,4 +1,11 @@ if [ -n "$SLURM_JOB_ID" ]; then GALAXY_MEMORY_MB=`scontrol -do show job "$SLURM_JOB_ID" | sed 's/.*\( \|^\)Mem=\([0-9][0-9]*\)\( \|$\).*/\2/p;d'` 2>memory_statement.log fi + +if [ -z "$GALAXY_MEMORY_MB_PERSLOT" -a -n "$GALAXY_MEMORY_MB" ]; then + GALAXY_MEMORY_MB_PERSLOT=$(($GALAXY_MEMORY_MB / $GALAXY_SLOTS)) +elif [ -z "$GALAXY_MEMORY_MB" -a -n "$GALAXY_MEMORY_MB_PERSLOT" ]; then + GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB_PERSLOT * $GALAXY_SLOTS)) +fi [ "${GALAXY_MEMORY_MB--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB || unset GALAXY_MEMORY_MB +[ "${GALAXY_MEMORY_MB_PERSLOT--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB_PERSLOT || unset GALAXY_MEMORY_MB_PERSLOT From c6c47be0cbb7d29cefe3e1c0657f4b188286abb6 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 1 Mar 2018 13:14:56 +0100 Subject: [PATCH 2/6] added GALAXY_MEMORY_MB_PERSLOT --- .../jobs/runners/util/job_script/MEMORY_STATEMENT.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh index fef76e5fff9..f4f383521c6 100644 --- a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh @@ -1,4 +1,11 @@ if [ -n "$SLURM_JOB_ID" ]; then GALAXY_MEMORY_MB=`scontrol -do show job "$SLURM_JOB_ID" | sed 's/.*\( \|^\)Mem=\([0-9][0-9]*\)\( \|$\).*/\2/p;d'` 2>memory_statement.log fi + +if [ -z "$GALAXY_MEMORY_MB_PERSLOT" -a -n "$GALAXY_MEMORY_MB" ]; then + GALAXY_MEMORY_MB_PERSLOT=$(($GALAXY_MEMORY_MB / $GALAXY_SLOTS)) +elif [ -z "$GALAXY_MEMORY_MB" -a -n "$GALAXY_MEMORY_MB_PERSLOT" ]; then + GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB_PERSLOT * $GALAXY_SLOTS)) +fi [ "${GALAXY_MEMORY_MB--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB || unset GALAXY_MEMORY_MB +[ "${GALAXY_MEMORY_MB_PERSLOT--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB_PERSLOT || unset GALAXY_MEMORY_MB_PERSLOT From 7316302ba385c329f458aa333994e6eeb6d5b97d Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 5 Mar 2018 22:51:20 +0100 Subject: [PATCH 3/6] renamed variable to GALAXY_MEMORY_MB_PER_SLOT --- .../jobs/runners/util/job_script/MEMORY_STATEMENT.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh index f4f383521c6..5d58124365c 100644 --- a/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh +++ b/lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh @@ -2,10 +2,10 @@ if [ -n "$SLURM_JOB_ID" ]; then GALAXY_MEMORY_MB=`scontrol -do show job "$SLURM_JOB_ID" | sed 's/.*\( \|^\)Mem=\([0-9][0-9]*\)\( \|$\).*/\2/p;d'` 2>memory_statement.log fi -if [ -z "$GALAXY_MEMORY_MB_PERSLOT" -a -n "$GALAXY_MEMORY_MB" ]; then - GALAXY_MEMORY_MB_PERSLOT=$(($GALAXY_MEMORY_MB / $GALAXY_SLOTS)) -elif [ -z "$GALAXY_MEMORY_MB" -a -n "$GALAXY_MEMORY_MB_PERSLOT" ]; then - GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB_PERSLOT * $GALAXY_SLOTS)) +if [ -z "$GALAXY_MEMORY_MB_PER_SLOT" -a -n "$GALAXY_MEMORY_MB" ]; then + GALAXY_MEMORY_MB_PER_SLOT=$(($GALAXY_MEMORY_MB / $GALAXY_SLOTS)) +elif [ -z "$GALAXY_MEMORY_MB" -a -n "$GALAXY_MEMORY_MB_PER_SLOT" ]; then + GALAXY_MEMORY_MB=$(($GALAXY_MEMORY_MB_PER_SLOT * $GALAXY_SLOTS)) fi [ "${GALAXY_MEMORY_MB--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB || unset GALAXY_MEMORY_MB -[ "${GALAXY_MEMORY_MB_PERSLOT--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB_PERSLOT || unset GALAXY_MEMORY_MB_PERSLOT +[ "${GALAXY_MEMORY_MB_PER_SLOT--1}" -gt 0 ] 2>>memory_statement.log && export GALAXY_MEMORY_MB_PER_SLOT || unset GALAXY_MEMORY_MB_PER_SLOT From f54f3aa9e47141fa4b564ec56d37a97a129bea12 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 6 Mar 2018 11:30:50 +0100 Subject: [PATCH 4/6] added a bit of documentation --- lib/galaxy/tools/xsd/galaxy.xsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/xsd/galaxy.xsd b/lib/galaxy/tools/xsd/galaxy.xsd index 47ba3be03de..e056c6ed3a3 100644 --- a/lib/galaxy/tools/xsd/galaxy.xsd +++ b/lib/galaxy/tools/xsd/galaxy.xsd @@ -2656,7 +2656,8 @@ be escaped with a backslash (``\``) when appearing in ``command`` or ``configfil Name | Description ---- | ----------- ``\${GALAXY_SLOTS:-4}`` | Number of cores/threads allocated by the job runner or resource manager to the tool for the given job (here 4 is the default number of threads to use if running via custom runner that does not configure GALAXY_SLOTS or in an older Galaxy runtime). -``\$GALAXY_MEMORY_MB`` | Amount of memory in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. +``\$GALAXY_MEMORY_MB`` | Total amount of memory in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. +``\$GALAXY_MEMORY_MB_PER_SLOT`` | Amount of memory per slot in megabytes (1024^2 bytes) allocated by the administrator (via the resource manager) to the tool for the given job. If unset, tools should not attempt to limit memory usage. See the [Planemo docs](https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage) on the topic of ``GALAXY_SLOTS`` for more information and examples. From 4a59f30209c9a9bee4e2ae70b56ff02f5248716c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 13 Mar 2018 17:22:05 +0100 Subject: [PATCH 5/6] added cluster admin docs for the new variables --- doc/source/admin/cluster.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/source/admin/cluster.md b/doc/source/admin/cluster.md index 5f40c91440b..6332005ba79 100644 --- a/doc/source/admin/cluster.md +++ b/doc/source/admin/cluster.md @@ -195,7 +195,7 @@ Most options available to `qsub(1b)` and `pbs_submit(3b)` are supported. Except ``` -The value of *ppn=* is used by PBS to define the environment variable $PBS_NCPUS which in turn is used by galaxy for [GALAXY_SLOTS](https://galaxyproject.org/admin/config/galaxy_slots/). +The value of *ppn=* is used by PBS to define the environment variable `$PBS_NCPUS` which in turn is used by galaxy for [GALAXY_SLOTS](https://galaxyproject.org/admin/config/galaxy_slots/). ### Condor @@ -355,6 +355,31 @@ It is also a good idea to make sure that only trusted users, e.g. root, have wri Some maintenance and support of this code will be provided via the usual [Support](https://galaxyproject.org/support/) channels, but improvements and fixes would be greatly welcomed, as this is a complex feature which is not used by the Galaxy Development Team. +## Special environment variables for parallel jobs + +Galaxy *tries* to define special enironment variables for each job that contain +the information on the number of available slots and the amount of available +memory: + +* `GALAXY_SLOTS`: number of available slots +* `GALAXY_MEMORY_MB`: total amount of available memory in MB +* `GALAXY_MEMORY_MB_PER_SLOT`: amount of memory that is available for each slot in MB + +More precisely Galaxy inserts bash code in the job submit script that +tries to determine these values. This bash code is defined here: + +* lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh +* lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh + +If this code is unable to determine the variables then they will not be set. +Therefore in the tool xml files the variables should be used with a default, +e.g. `\${GALAXY_SLOTS:-DEFAULT}` (see also https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage). + +In particular `GALAXY_MEMORY_MB` and `GALAXY_MEMORY_MB_PER_SLOT` are currently +defined only for a few cluster types. Contributions are very welcome, e.g. let +the Galaxy development mailing list know how to modify that file to support +it. + ## Contributors * **Oleksandr Moskalenko**, debugged a number of problems related to running jobs as the real user and using DRMAA with TORQUE. From b92a5e1b470cbe3848e251c8c844cc652782d43d Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 13 Mar 2018 16:51:17 +0000 Subject: [PATCH 6/6] Small doc fixes --- doc/source/admin/cluster.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/source/admin/cluster.md b/doc/source/admin/cluster.md index 6332005ba79..3ff9eaff053 100644 --- a/doc/source/admin/cluster.md +++ b/doc/source/admin/cluster.md @@ -355,9 +355,9 @@ It is also a good idea to make sure that only trusted users, e.g. root, have wri Some maintenance and support of this code will be provided via the usual [Support](https://galaxyproject.org/support/) channels, but improvements and fixes would be greatly welcomed, as this is a complex feature which is not used by the Galaxy Development Team. -## Special environment variables for parallel jobs +## Special environment variables for job resources -Galaxy *tries* to define special enironment variables for each job that contain +Galaxy *tries* to define special environment variables for each job that contain the information on the number of available slots and the amount of available memory: @@ -371,14 +371,13 @@ tries to determine these values. This bash code is defined here: * lib/galaxy/jobs/runners/util/job_script/CLUSTER_SLOTS_STATEMENT.sh * lib/galaxy/jobs/runners/util/job_script/MEMORY_STATEMENT.sh -If this code is unable to determine the variables then they will not be set. -Therefore in the tool xml files the variables should be used with a default, -e.g. `\${GALAXY_SLOTS:-DEFAULT}` (see also https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage). +If this code is unable to determine the variables, then they will not be set. +Therefore in the tool XML files the variables should be used with a default, +e.g. `\${GALAXY_SLOTS:-1}` (see also https://planemo.readthedocs.io/en/latest/writing_advanced.html#cluster-usage). In particular `GALAXY_MEMORY_MB` and `GALAXY_MEMORY_MB_PER_SLOT` are currently defined only for a few cluster types. Contributions are very welcome, e.g. let -the Galaxy development mailing list know how to modify that file to support -it. +the Galaxy developers know how to modify that file to support your cluster. ## Contributors