opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-24 02:04:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add some documentation for aws batch runner

Browse Source
This commit is contained in:
qiagu
2022-08-18 12:42:38 -07:00
parent d3a2b0f2d0
commit 667e6247f6
2 changed files with 26 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/config/sample/job_conf.xml.sample_advanced
+14 -4
View File
@@ -374,6 +374,11 @@
<!-- <param id="aws_access_key_id">xxxxxxxxx</param>
<param id="aws_secret_access_key">xxxxxxxxxxxxxxxxxxx</param>
<param id="region">us-west-1</param> -->
<!--
This runner requires that an AWS EFS is mounted as a local drive and all Galaxy
job-related paths, such as objects, job_directory, tool_directory and so on, are placed
in the EFS drive.
-->
</plugin>
<!-- Additionally any plugin or destination (below) may define an "enabled" parameter that should
evaluate to True or False. When setup using
@@ -1133,27 +1138,32 @@
<!-- Number of retries to attempt if a command returns a non-zero status -->
</destination>
<destination id="aws_batch_auto" runner="aws_batch">
<!-- Fargete and non-GPU EC2 -->
<param id="docker_enabled">true</param>
<!-- `docker_enabled = true` is always required -->
<param id="job_queue">arn_for_Fargate_job_queue, arn_for_EC2_job_queue</param>
<!-- Fargete and non-GPU EC2 -->
<param id="job_role_arn">arn:aws:iam::xxxxxxxxxxxxxxxxxx</param>
<param id="vcpu">1</param>
<param id="memory">2048</param>
<param id="efs_filesystem_id">fs-xxxxxxxxxxxxxx</param>
<param id="efs_mount_point">/mnt/efs</param>
<param id="efs_mount_point">/mnt/efs/fs1</param>
<!-- This is the location where the EFS is mounted -->
<param id="fargate_version">1.4.0</param>
<!-- `fargate_version` is required to use Fargate compute resources -->
<param id="auto_platform">true</param>
</destination>
<destination id="aws_batch_gpu" runner="aws_batch">
<param id="docker_enabled">true</param>
<!-- Job queue must be built on GPU-specific compute environment -->
<!-- `docker_enabled = true` is always required -->
<param id="job_queue">arn_for_gpu_job_queue</param>
<!-- Job queue must be built on GPU-specific compute environment -->
<param id="job_role_arn">arn:aws:iam::xxxxxxxxxxxxxxxxxx</param>
<param id="vcpu">4</param>
<param id="memory">20000</param>
<param id="gpu">1</param>
<param id="efs_filesystem_id">fs-xxxxxxxxxxxxxx</param>
<param id="efs_mount_point">/mnt/efs</param>
<param id="efs_mount_point">/mnt/efs/fs1</param>
<!-- This is the location where the EFS is mounted -->
</destination>
<!-- Templatized destinations - macros can be used to create templated
destinations with reduced XML duplication. Here we are creating 4 destinations in 4 lines instead of 28 using the macros defined below.
lib/galaxy/jobs/runners/aws.py
+12 -1
View File
@@ -1,4 +1,5 @@
import functools
""" Galaxy job runners to use Amazon AWS native compute resources, such as AWS Batch.
"""
import hashlib
import json
import logging
@@ -77,6 +78,16 @@ def _add_resource_requirements(destination_params):
class AWSBatchJobRunner(AsynchronousJobRunner):
"""
This runner uses container only. It requires that an AWS EFS is mounted as a local drive
and all Galaxy job-related paths, such as objects, job_directory, tool_directory and so
on, are placed in the EFS drive. Via this runner, Galaxy jobs are dispatched to AWS Batch
for compute using the docker image specified by a Galaxy tool. As AWS EFS is designed to
be able to mount at multiple places with read and write capabilities, Galaxy and Batch
containers share the same EFS drive as a local device. Sample configurations can be found
in `config/job_conf.xml.sample_advanced`.
"""
runner_name = "AWSBatchRunner"
RUNNER_PARAM_SPEC_KEY = "runner_param_specs"
JOB_NAME_PREFIX = "galaxy-"