diff --git a/config/job_conf.sample.yml b/config/job_conf.sample.yml new file mode 120000 index 00000000000..f3eeab8bce4 --- /dev/null +++ b/config/job_conf.sample.yml @@ -0,0 +1 @@ +../lib/galaxy/config/sample/job_conf.sample.yml \ No newline at end of file diff --git a/config/job_conf.xml.interactivetools b/config/job_conf.xml.interactivetools deleted file mode 100644 index f422ca36799..00000000000 --- a/config/job_conf.xml.interactivetools +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - true - - $galaxy_root:ro,$tool_directory:ro,$job_directory:rw,$working_directory:rw,$default_file_path:ro - false - bridge - true - true - true - - - --add-host localhost:host-gateway - - - docker_dispatch - docker_local - local - - - diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced deleted file mode 120000 index 19297cf6cfe..00000000000 --- a/config/job_conf.xml.sample_advanced +++ /dev/null @@ -1 +0,0 @@ -../lib/galaxy/config/sample/job_conf.xml.sample_advanced \ No newline at end of file diff --git a/config/job_conf.xml.sample_basic b/config/job_conf.xml.sample_basic deleted file mode 120000 index 9a38a78a624..00000000000 --- a/config/job_conf.xml.sample_basic +++ /dev/null @@ -1 +0,0 @@ -../lib/galaxy/config/sample/job_conf.xml.sample_basic \ No newline at end of file diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 4ea0b31ffe3..77fec76138f 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -2001,8 +2001,7 @@ file staging and Interactive Tool containers for communicating back with Galaxy via the API. If you plan to run Interactive Tools make sure the docker - container can reach this URL. For more details see - `job_conf.xml.interactivetools`. + container can reach this URL. :Default: ``http://localhost:8080`` :Type: str @@ -4113,10 +4112,10 @@ process and notifies itself of new jobs via in-memory queues. Jobs are run locally on the system on which Galaxy is started. Advanced job running capabilities can be configured through the - job configuration file. + job configuration file or the option. The value of this option will be resolved with respect to . -:Default: ``job_conf.xml`` +:Default: ``job_conf.yml`` :Type: str @@ -4170,15 +4169,15 @@ When jobs fail due to job runner problems, Galaxy can be configured to retry these or reroute the jobs to new destinations. Very fine control of this is available with resubmit declarations - in job_conf.xml. For simple deployments of Galaxy though, the + in the job config. For simple deployments of Galaxy though, the following attribute can define resubmission conditions for all job destinations. If any job destination defines even one resubmission - condition explicitly in job_conf.xml - the condition described by - this option will not apply to that destination. For instance, the - condition: 'attempt < 3 and unknown_error and (time_running < 300 - or time_since_queued < 300)' would retry up to two times jobs that - didn't fail due to detected memory or walltime limits but did fail - quickly (either while queueing or running). The commented out + condition explicitly in the job config - the condition described + by this option will not apply to that destination. For instance, + the condition: 'attempt < 3 and unknown_error and (time_running < + 300 or time_since_queued < 300)' would retry up to two times jobs + that didn't fail due to detected memory or walltime limits but did + fail quickly (either while queueing or running). The commented out default below results in no default job resubmission condition, failing jobs are just failed outright. :Default: ``None`` diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index f1800bda700..c2db49af9e9 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1120,8 +1120,7 @@ galaxy: # file staging and Interactive Tool containers for communicating back # with Galaxy via the API. # If you plan to run Interactive Tools make sure the docker container - # can reach this URL. For more details see - # `job_conf.xml.interactivetools`. + # can reach this URL. #galaxy_infrastructure_url: http://localhost:8080 # If the above URL cannot be determined ahead of time in dynamic @@ -2066,10 +2065,10 @@ galaxy: # process and notifies itself of new jobs via in-memory queues. Jobs # are run locally on the system on which Galaxy is started. Advanced # job running capabilities can be configured through the job - # configuration file. + # configuration file or the option. # The value of this option will be resolved with respect to # . - #job_config_file: job_conf.xml + #job_config_file: job_conf.yml # Description of job running configuration, can be embedded into # Galaxy configuration or loaded from an additional file with the @@ -2094,11 +2093,11 @@ galaxy: # When jobs fail due to job runner problems, Galaxy can be configured # to retry these or reroute the jobs to new destinations. Very fine - # control of this is available with resubmit declarations in - # job_conf.xml. For simple deployments of Galaxy though, the following + # control of this is available with resubmit declarations in the job + # config. For simple deployments of Galaxy though, the following # attribute can define resubmission conditions for all job # destinations. If any job destination defines even one resubmission - # condition explicitly in job_conf.xml - the condition described by + # condition explicitly in the job config - the condition described by # this option will not apply to that destination. For instance, the # condition: 'attempt < 3 and unknown_error and (time_running < 300 or # time_since_queued < 300)' would retry up to two times jobs that diff --git a/test/unit/app/jobs/job_conf.sample_advanced.yml b/lib/galaxy/config/sample/job_conf.sample.yml similarity index 100% rename from test/unit/app/jobs/job_conf.sample_advanced.yml rename to lib/galaxy/config/sample/job_conf.sample.yml diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index b33c10a8403..bee8c8e5d86 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -1445,7 +1445,7 @@ mapping: Galaxy via the API. If you plan to run Interactive Tools make sure the docker container - can reach this URL. For more details see `job_conf.xml.interactivetools`. + can reach this URL. galaxy_infrastructure_web_port: type: int @@ -2982,7 +2982,7 @@ mapping: job_config_file: type: str - default: job_conf.xml + default: job_conf.yml path_resolves_to: config_dir required: false desc: | @@ -2995,7 +2995,7 @@ mapping: By default, Galaxy manages and executes jobs from within a single process and notifies itself of new jobs via in-memory queues. Jobs are run locally on the system on which Galaxy is started. Advanced job running capabilities can - be configured through the job configuration file. + be configured through the job configuration file or the option. job_config: !include job_config_schema.yml @@ -3047,10 +3047,10 @@ mapping: desc: | When jobs fail due to job runner problems, Galaxy can be configured to retry these or reroute the jobs to new destinations. Very fine control of this is - available with resubmit declarations in job_conf.xml. For simple deployments + available with resubmit declarations in the job config. For simple deployments of Galaxy though, the following attribute can define resubmission conditions for all job destinations. If any job destination defines even one - resubmission condition explicitly in job_conf.xml - the condition described + resubmission condition explicitly in the job config - the condition described by this option will not apply to that destination. For instance, the condition: 'attempt < 3 and unknown_error and (time_running < 300 or time_since_queued < 300)' would retry up to two times jobs that didn't fail due to detected memory or diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 67803280216..0581bbd151e 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -58,7 +58,11 @@ class ConditionalDependencies: if "job_config" in self.config: load_job_config_dict(self.config.get("job_config")) else: - job_conf_path = self.config.get("job_config_file", join(dirname(self.config_file), "job_conf.xml")) + job_conf_path = self.config.get("job_config_file") + if not job_conf_path: + job_conf_path = join(dirname(self.config_file), "job_conf.yml") + if not exists(job_conf_path): + job_conf_path = join(dirname(self.config_file), "job_conf.xml") if ".xml" in job_conf_path: try: try: diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 58960e61f22..722af5fc6bb 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -288,7 +288,7 @@ def job_config_xml_to_dict(config, root): class JobConfiguration(ConfiguresHandlers): """A parser and interface to advanced job management features. - These features are configured in the job configuration, by default, ``job_conf.xml`` + These features are configured in the job configuration, by default, ``job_conf.yml`` """ runner_plugins: List[dict] @@ -362,14 +362,24 @@ class JobConfiguration(ConfiguresHandlers): try: if "job_config" in self.app.config.config_dict: job_config_dict = self.app.config.config_dict["job_config"] + log.debug("Read job configuration inline from Galaxy config") else: job_config_file = self.app.config.job_config_file + if not self.app.config.is_set("job_config_file") and not os.path.exists(job_config_file): + job_config_file = os.path.join(os.path.dirname(self.app.config.config_file), "job_conf.xml") + if os.path.exists(job_config_file): + log.warning( + "Implicit loading of job_conf.xml has been deprecated and will be removed in a future" + f" release of Galaxy. Please convert to YAML at {self.app.config.job_config_file} or" + f" explicitly set `job_config_file` to {job_config_file} to remove this message" + ) if ".xml" in job_config_file: tree = load(job_config_file) job_config_dict = self.__parse_job_conf_xml(tree) else: with open(job_config_file) as f: job_config_dict = yaml.safe_load(f) + log.debug(f"Read job configuration from file: {job_config_file}") # Load tasks if configured if self.app.config.use_tasked_jobs: diff --git a/lib/galaxy/jobs/command_factory.py b/lib/galaxy/jobs/command_factory.py index f24aa3596ba..5a4155a64a8 100644 --- a/lib/galaxy/jobs/command_factory.py +++ b/lib/galaxy/jobs/command_factory.py @@ -173,7 +173,7 @@ def __externalize_commands( local_container_script = join(job_wrapper.working_directory, script_name) tool_commands = commands_builder.build() integrity_injection = "" - # Setting shell to none in job_conf.xml disables creating a tool command script, + # Setting shell to none in the job config disables creating a tool command script, # set -e doesn't work for composite commands but this is necessary for Windows jobs # for instance. if shell and shell.lower() == "none": diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py index 3697bbc9b04..d77c71d2259 100644 --- a/lib/galaxy/jobs/handler.py +++ b/lib/galaxy/jobs/handler.py @@ -338,7 +338,7 @@ class JobHandlerQueue(Monitors): # Already dispatched and running job_wrapper = self.job_wrapper(job) # Use the persisted destination as its params may differ from - # what's in the job_conf xml + # what's in the job config job_destination = JobDestination( id=job.destination_id, runner=job.job_runner_name, params=job.destination_params ) diff --git a/lib/galaxy/jobs/mapper.py b/lib/galaxy/jobs/mapper.py index 294c0115493..d435f15cdda 100644 --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -101,7 +101,7 @@ class JobRunnerMapper: actual_args = {} for arg in function_arg_names: - # Send through any job_conf.xml defined args to function + # Send through any job config defined args to function if arg in destination.params: actual_args[arg] = destination.params[arg] # Populate needed args diff --git a/lib/galaxy/jobs/runners/aws.py b/lib/galaxy/jobs/runners/aws.py index 720df6cb79b..9a7278ea668 100644 --- a/lib/galaxy/jobs/runners/aws.py +++ b/lib/galaxy/jobs/runners/aws.py @@ -92,7 +92,7 @@ class AWSBatchJobRunner(AsynchronousJobRunner): for compute using the docker image specified by a Galaxy tool. As AWS EFS is designed to be able to mount at multiple places with read and write capabilities, Galaxy and Batch containers share the same EFS drive as a local device. Sample configurations can be found - in `config/job_conf.xml.sample_advanced`. + in `config/job_conf.sample.yml`. """ runner_name = "AWSBatchRunner" diff --git a/lib/galaxy/jobs/runners/godocker.py b/lib/galaxy/jobs/runners/godocker.py index 295248663f0..869994a8838 100644 --- a/lib/galaxy/jobs/runners/godocker.py +++ b/lib/galaxy/jobs/runners/godocker.py @@ -124,7 +124,7 @@ class GodockerJobRunner(AsynchronousJobRunner): runner_name = "GodockerJobRunner" def __init__(self, app, nworkers, **kwargs): - """1: Get runner_param_specs from job_conf.xml + """1: Get runner_param_specs from the job config 2: Initialise job runner parent object 3: Login to godocker and store the token 4: Start the worker and monitor threads diff --git a/lib/galaxy/jobs/stock_rules.py b/lib/galaxy/jobs/stock_rules.py index 0fb09c440a3..25601b26644 100644 --- a/lib/galaxy/jobs/stock_rules.py +++ b/lib/galaxy/jobs/stock_rules.py @@ -1,4 +1,4 @@ -""" Stock job 'dynamic' rules for use in job_conf.xml - these may cover some +""" Stock job 'dynamic' rules for use in the job config file - these may cover some simple use cases but will just proxy into functions in rule_helper so similar functionality - but more tailored and composable can be utilized in custom rules. diff --git a/test/unit/app/jobs/test_job_configuration.py b/test/unit/app/jobs/test_job_configuration.py index 1b3d478d068..56ba267fb11 100644 --- a/test/unit/app/jobs/test_job_configuration.py +++ b/test/unit/app/jobs/test_job_configuration.py @@ -21,7 +21,7 @@ from galaxy.web_stack.handlers import HANDLER_ASSIGNMENT_METHODS # there are advantages to testing the documentation/examples. SIMPLE_JOB_CONF = os.path.join(galaxy_samples_directory(), "job_conf.xml.sample_basic") ADVANCED_JOB_CONF = os.path.join(galaxy_samples_directory(), "job_conf.xml.sample_advanced") -ADVANCED_JOB_CONF_YAML = os.path.join(os.path.dirname(__file__), "job_conf.sample_advanced.yml") +ADVANCED_JOB_CONF_YAML = os.path.join(galaxy_samples_directory(), "job_conf.sample.yml") CONDITIONAL_RUNNER_JOB_CONF = os.path.join(os.path.dirname(__file__), "conditional_runners_job_conf.xml") HANDLER_TEMPLATE_JOB_CONF = os.path.join(os.path.dirname(__file__), "handler_template_job_conf.xml")