mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #15018 from natefoo/job-conf-yaml
Replace XML job conf sample with YAML
This commit is contained in:
Martin Cech
Symlink
+1
@@ -0,0 +1 @@
|
||||
../lib/galaxy/config/sample/job_conf.sample.yml
|
||||
@@ -1,35 +0,0 @@
|
||||
<?xml version="1.0"?>
|
||||
<!-- A sample job config for InteractiveTools using local runner. -->
|
||||
<job_conf>
|
||||
<plugins>
|
||||
<plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
|
||||
</plugins>
|
||||
<destinations default="docker_dispatch">
|
||||
<destination id="local" runner="local"/>
|
||||
<destination id="docker_local" runner="local">
|
||||
<param id="docker_enabled">true</param>
|
||||
<!-- If you have not set 'outputs_to_working_directory: true' in galaxy.yml you can remove the docker_volumes setting. -->
|
||||
<param id="docker_volumes">$galaxy_root:ro,$tool_directory:ro,$job_directory:rw,$working_directory:rw,$default_file_path:ro</param>
|
||||
<param id="docker_sudo">false</param>
|
||||
<param id="docker_net">bridge</param>
|
||||
<param id="docker_auto_rm">true</param>
|
||||
<param id="require_container">true</param>
|
||||
<param id="container_monitor">true</param>
|
||||
<param id="docker_set_user"></param>
|
||||
<!-- InteractiveTools do need real hostnames or URLs to work - simply specifying IPs will not work.
|
||||
If you develop interactive tools on your 'localhost' and don't have a proper domain name
|
||||
you need to tell all Docker containers a hostname where Galaxy is running.
|
||||
This can be done via the add-host parameter during the `docker run` command.
|
||||
'localhost' here is an arbitrary hostname that matches the IP address of your
|
||||
Galaxy host. Make sure this hostname ('localhost') is also set in your galaxy.yml file, e.g.
|
||||
`galaxy_infrastructure_url: http://localhost:8080`.
|
||||
-->
|
||||
<param id="docker_run_extra_arguments">--add-host localhost:host-gateway</param>
|
||||
</destination>
|
||||
<destination id="docker_dispatch" runner="dynamic">
|
||||
<param id="type">docker_dispatch</param>
|
||||
<param id="docker_destination_id">docker_local</param>
|
||||
<param id="default_destination_id">local</param>
|
||||
</destination>
|
||||
</destinations>
|
||||
</job_conf>
|
||||
@@ -1 +0,0 @@
|
||||
../lib/galaxy/config/sample/job_conf.xml.sample_advanced
|
||||
@@ -1 +0,0 @@
|
||||
../lib/galaxy/config/sample/job_conf.xml.sample_basic
|
||||
@@ -2001,8 +2001,7 @@
|
||||
file staging and Interactive Tool containers for communicating
|
||||
back with Galaxy via the API.
|
||||
If you plan to run Interactive Tools make sure the docker
|
||||
container can reach this URL. For more details see
|
||||
`job_conf.xml.interactivetools`.
|
||||
container can reach this URL.
|
||||
:Default: ``http://localhost:8080``
|
||||
:Type: str
|
||||
|
||||
@@ -4113,10 +4112,10 @@
|
||||
process and notifies itself of new jobs via in-memory queues.
|
||||
Jobs are run locally on the system on which Galaxy is started.
|
||||
Advanced job running capabilities can be configured through the
|
||||
job configuration file.
|
||||
job configuration file or the <job_config> option.
|
||||
The value of this option will be resolved with respect to
|
||||
<config_dir>.
|
||||
:Default: ``job_conf.xml``
|
||||
:Default: ``job_conf.yml``
|
||||
:Type: str
|
||||
|
||||
|
||||
@@ -4170,15 +4169,15 @@
|
||||
When jobs fail due to job runner problems, Galaxy can be
|
||||
configured to retry these or reroute the jobs to new destinations.
|
||||
Very fine control of this is available with resubmit declarations
|
||||
in job_conf.xml. For simple deployments of Galaxy though, the
|
||||
in the job config. For simple deployments of Galaxy though, the
|
||||
following attribute can define resubmission conditions for all job
|
||||
destinations. If any job destination defines even one resubmission
|
||||
condition explicitly in job_conf.xml - the condition described by
|
||||
this option will not apply to that destination. For instance, the
|
||||
condition: 'attempt < 3 and unknown_error and (time_running < 300
|
||||
or time_since_queued < 300)' would retry up to two times jobs that
|
||||
didn't fail due to detected memory or walltime limits but did fail
|
||||
quickly (either while queueing or running). The commented out
|
||||
condition explicitly in the job config - the condition described
|
||||
by this option will not apply to that destination. For instance,
|
||||
the condition: 'attempt < 3 and unknown_error and (time_running <
|
||||
300 or time_since_queued < 300)' would retry up to two times jobs
|
||||
that didn't fail due to detected memory or walltime limits but did
|
||||
fail quickly (either while queueing or running). The commented out
|
||||
default below results in no default job resubmission condition,
|
||||
failing jobs are just failed outright.
|
||||
:Default: ``None``
|
||||
|
||||
@@ -1120,8 +1120,7 @@ galaxy:
|
||||
# file staging and Interactive Tool containers for communicating back
|
||||
# with Galaxy via the API.
|
||||
# If you plan to run Interactive Tools make sure the docker container
|
||||
# can reach this URL. For more details see
|
||||
# `job_conf.xml.interactivetools`.
|
||||
# can reach this URL.
|
||||
#galaxy_infrastructure_url: http://localhost:8080
|
||||
|
||||
# If the above URL cannot be determined ahead of time in dynamic
|
||||
@@ -2066,10 +2065,10 @@ galaxy:
|
||||
# process and notifies itself of new jobs via in-memory queues. Jobs
|
||||
# are run locally on the system on which Galaxy is started. Advanced
|
||||
# job running capabilities can be configured through the job
|
||||
# configuration file.
|
||||
# configuration file or the <job_config> option.
|
||||
# The value of this option will be resolved with respect to
|
||||
# <config_dir>.
|
||||
#job_config_file: job_conf.xml
|
||||
#job_config_file: job_conf.yml
|
||||
|
||||
# Description of job running configuration, can be embedded into
|
||||
# Galaxy configuration or loaded from an additional file with the
|
||||
@@ -2094,11 +2093,11 @@ galaxy:
|
||||
|
||||
# When jobs fail due to job runner problems, Galaxy can be configured
|
||||
# to retry these or reroute the jobs to new destinations. Very fine
|
||||
# control of this is available with resubmit declarations in
|
||||
# job_conf.xml. For simple deployments of Galaxy though, the following
|
||||
# control of this is available with resubmit declarations in the job
|
||||
# config. For simple deployments of Galaxy though, the following
|
||||
# attribute can define resubmission conditions for all job
|
||||
# destinations. If any job destination defines even one resubmission
|
||||
# condition explicitly in job_conf.xml - the condition described by
|
||||
# condition explicitly in the job config - the condition described by
|
||||
# this option will not apply to that destination. For instance, the
|
||||
# condition: 'attempt < 3 and unknown_error and (time_running < 300 or
|
||||
# time_since_queued < 300)' would retry up to two times jobs that
|
||||
|
||||
@@ -1445,7 +1445,7 @@ mapping:
|
||||
Galaxy via the API.
|
||||
|
||||
If you plan to run Interactive Tools make sure the docker container
|
||||
can reach this URL. For more details see `job_conf.xml.interactivetools`.
|
||||
can reach this URL.
|
||||
|
||||
galaxy_infrastructure_web_port:
|
||||
type: int
|
||||
@@ -2982,7 +2982,7 @@ mapping:
|
||||
|
||||
job_config_file:
|
||||
type: str
|
||||
default: job_conf.xml
|
||||
default: job_conf.yml
|
||||
path_resolves_to: config_dir
|
||||
required: false
|
||||
desc: |
|
||||
@@ -2995,7 +2995,7 @@ mapping:
|
||||
By default, Galaxy manages and executes jobs from within a single process and
|
||||
notifies itself of new jobs via in-memory queues. Jobs are run locally on
|
||||
the system on which Galaxy is started. Advanced job running capabilities can
|
||||
be configured through the job configuration file.
|
||||
be configured through the job configuration file or the <job_config> option.
|
||||
|
||||
job_config:
|
||||
!include job_config_schema.yml
|
||||
@@ -3047,10 +3047,10 @@ mapping:
|
||||
desc: |
|
||||
When jobs fail due to job runner problems, Galaxy can be configured to retry
|
||||
these or reroute the jobs to new destinations. Very fine control of this is
|
||||
available with resubmit declarations in job_conf.xml. For simple deployments
|
||||
available with resubmit declarations in the job config. For simple deployments
|
||||
of Galaxy though, the following attribute can define resubmission conditions
|
||||
for all job destinations. If any job destination defines even one
|
||||
resubmission condition explicitly in job_conf.xml - the condition described
|
||||
resubmission condition explicitly in the job config - the condition described
|
||||
by this option will not apply to that destination. For instance, the condition:
|
||||
'attempt < 3 and unknown_error and (time_running < 300 or time_since_queued < 300)'
|
||||
would retry up to two times jobs that didn't fail due to detected memory or
|
||||
|
||||
@@ -58,7 +58,11 @@ class ConditionalDependencies:
|
||||
if "job_config" in self.config:
|
||||
load_job_config_dict(self.config.get("job_config"))
|
||||
else:
|
||||
job_conf_path = self.config.get("job_config_file", join(dirname(self.config_file), "job_conf.xml"))
|
||||
job_conf_path = self.config.get("job_config_file")
|
||||
if not job_conf_path:
|
||||
job_conf_path = join(dirname(self.config_file), "job_conf.yml")
|
||||
if not exists(job_conf_path):
|
||||
job_conf_path = join(dirname(self.config_file), "job_conf.xml")
|
||||
if ".xml" in job_conf_path:
|
||||
try:
|
||||
try:
|
||||
|
||||
@@ -288,7 +288,7 @@ def job_config_xml_to_dict(config, root):
|
||||
class JobConfiguration(ConfiguresHandlers):
|
||||
"""A parser and interface to advanced job management features.
|
||||
|
||||
These features are configured in the job configuration, by default, ``job_conf.xml``
|
||||
These features are configured in the job configuration, by default, ``job_conf.yml``
|
||||
"""
|
||||
|
||||
runner_plugins: List[dict]
|
||||
@@ -362,14 +362,24 @@ class JobConfiguration(ConfiguresHandlers):
|
||||
try:
|
||||
if "job_config" in self.app.config.config_dict:
|
||||
job_config_dict = self.app.config.config_dict["job_config"]
|
||||
log.debug("Read job configuration inline from Galaxy config")
|
||||
else:
|
||||
job_config_file = self.app.config.job_config_file
|
||||
if not self.app.config.is_set("job_config_file") and not os.path.exists(job_config_file):
|
||||
job_config_file = os.path.join(os.path.dirname(self.app.config.config_file), "job_conf.xml")
|
||||
if os.path.exists(job_config_file):
|
||||
log.warning(
|
||||
"Implicit loading of job_conf.xml has been deprecated and will be removed in a future"
|
||||
f" release of Galaxy. Please convert to YAML at {self.app.config.job_config_file} or"
|
||||
f" explicitly set `job_config_file` to {job_config_file} to remove this message"
|
||||
)
|
||||
if ".xml" in job_config_file:
|
||||
tree = load(job_config_file)
|
||||
job_config_dict = self.__parse_job_conf_xml(tree)
|
||||
else:
|
||||
with open(job_config_file) as f:
|
||||
job_config_dict = yaml.safe_load(f)
|
||||
log.debug(f"Read job configuration from file: {job_config_file}")
|
||||
|
||||
# Load tasks if configured
|
||||
if self.app.config.use_tasked_jobs:
|
||||
|
||||
@@ -173,7 +173,7 @@ def __externalize_commands(
|
||||
local_container_script = join(job_wrapper.working_directory, script_name)
|
||||
tool_commands = commands_builder.build()
|
||||
integrity_injection = ""
|
||||
# Setting shell to none in job_conf.xml disables creating a tool command script,
|
||||
# Setting shell to none in the job config disables creating a tool command script,
|
||||
# set -e doesn't work for composite commands but this is necessary for Windows jobs
|
||||
# for instance.
|
||||
if shell and shell.lower() == "none":
|
||||
|
||||
@@ -338,7 +338,7 @@ class JobHandlerQueue(Monitors):
|
||||
# Already dispatched and running
|
||||
job_wrapper = self.job_wrapper(job)
|
||||
# Use the persisted destination as its params may differ from
|
||||
# what's in the job_conf xml
|
||||
# what's in the job config
|
||||
job_destination = JobDestination(
|
||||
id=job.destination_id, runner=job.job_runner_name, params=job.destination_params
|
||||
)
|
||||
|
||||
@@ -101,7 +101,7 @@ class JobRunnerMapper:
|
||||
actual_args = {}
|
||||
|
||||
for arg in function_arg_names:
|
||||
# Send through any job_conf.xml defined args to function
|
||||
# Send through any job config defined args to function
|
||||
if arg in destination.params:
|
||||
actual_args[arg] = destination.params[arg]
|
||||
# Populate needed args
|
||||
|
||||
@@ -92,7 +92,7 @@ class AWSBatchJobRunner(AsynchronousJobRunner):
|
||||
for compute using the docker image specified by a Galaxy tool. As AWS EFS is designed to
|
||||
be able to mount at multiple places with read and write capabilities, Galaxy and Batch
|
||||
containers share the same EFS drive as a local device. Sample configurations can be found
|
||||
in `config/job_conf.xml.sample_advanced`.
|
||||
in `config/job_conf.sample.yml`.
|
||||
"""
|
||||
|
||||
runner_name = "AWSBatchRunner"
|
||||
|
||||
@@ -124,7 +124,7 @@ class GodockerJobRunner(AsynchronousJobRunner):
|
||||
runner_name = "GodockerJobRunner"
|
||||
|
||||
def __init__(self, app, nworkers, **kwargs):
|
||||
"""1: Get runner_param_specs from job_conf.xml
|
||||
"""1: Get runner_param_specs from the job config
|
||||
2: Initialise job runner parent object
|
||||
3: Login to godocker and store the token
|
||||
4: Start the worker and monitor threads
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
""" Stock job 'dynamic' rules for use in job_conf.xml - these may cover some
|
||||
""" Stock job 'dynamic' rules for use in the job config file - these may cover some
|
||||
simple use cases but will just proxy into functions in rule_helper so similar
|
||||
functionality - but more tailored and composable can be utilized in custom
|
||||
rules.
|
||||
|
||||
@@ -21,7 +21,7 @@ from galaxy.web_stack.handlers import HANDLER_ASSIGNMENT_METHODS
|
||||
# there are advantages to testing the documentation/examples.
|
||||
SIMPLE_JOB_CONF = os.path.join(galaxy_samples_directory(), "job_conf.xml.sample_basic")
|
||||
ADVANCED_JOB_CONF = os.path.join(galaxy_samples_directory(), "job_conf.xml.sample_advanced")
|
||||
ADVANCED_JOB_CONF_YAML = os.path.join(os.path.dirname(__file__), "job_conf.sample_advanced.yml")
|
||||
ADVANCED_JOB_CONF_YAML = os.path.join(galaxy_samples_directory(), "job_conf.sample.yml")
|
||||
CONDITIONAL_RUNNER_JOB_CONF = os.path.join(os.path.dirname(__file__), "conditional_runners_job_conf.xml")
|
||||
HANDLER_TEMPLATE_JOB_CONF = os.path.join(os.path.dirname(__file__), "handler_template_job_conf.xml")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user