From 52cbaba160099a7e43f40a598f703260abef8976 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 14 Jul 2021 12:42:23 -0400 Subject: [PATCH] Allow specifying useful classes of tools for mapping in job conf. --- lib/galaxy/jobs/__init__.py | 45 ++++++++++++++----- lib/galaxy/tools/__init__.py | 14 +++++- .../webapps/galaxy/job_config_schema.yml | 2 + test/integration/delay_job_conf.yml | 2 +- test/integration/embedded_pulsar_job_conf.xml | 2 +- ...dded_pulsar_metadata_extended_job_conf.yml | 2 +- .../embedded_pulsar_metadata_job_conf.yml | 2 +- .../embedded_pulsar_singularity_job_conf.yml | 2 +- .../resubmission_default_job_conf.yml | 2 +- .../resubmission_dynamic_job_conf.xml | 2 +- test/integration/resubmission_job_conf.yml | 2 +- .../resubmission_pulsar_job_conf.xml | 2 +- test/integration/singularity_job_conf.xml | 2 +- test/unit/jobs/job_conf.sample_advanced.yml | 8 ++++ test/unit/tools_support.py | 2 +- 15 files changed, 68 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 7a2ff427e33..01d2d1a1105 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -82,6 +82,7 @@ DEFAULT_JOB_SHELL = '/bin/bash' DEFAULT_LOCAL_WORKERS = 4 DEFAULT_CLEANUP_JOB = "always" +VALID_TOOL_CLASSES = ["local", "requires_galaxy"] class JobDestination(Bunch): @@ -244,10 +245,10 @@ def job_config_xml_to_dict(config, root): tools = root.find('tools') config_dict['tools'] = [] if tools is not None: - for tool in ConfiguresHandlers._findall_with_required(tools, 'tool'): + for tool in tools.findall('tool'): # There can be multiple definitions with identical ids, but different params tool_mapping_conf = {} - for key in ['handler', 'destination', 'id', 'resources']: + for key in ['handler', 'destination', 'id', 'resources', 'class']: value = tool.get(key) if value: if key == "destination": @@ -283,6 +284,7 @@ class JobConfiguration(ConfiguresHandlers): handlers: dict handler_runner_plugins: Dict[str, str] tools: Dict[str, list] + tool_classes: Dict[str, list] resource_groups: Dict[str, list] destinations: Dict[str, tuple] resource_parameters: Dict[str, Any] @@ -317,6 +319,7 @@ class JobConfiguration(ConfiguresHandlers): self.destinations = {} self.default_destination_id = None self.tools = {} + self.tool_classes = {} self.resource_groups = {} self.default_resource_group = None self.resource_parameters = {} @@ -473,9 +476,18 @@ class JobConfiguration(ConfiguresHandlers): tools = job_config_dict.get('tools', []) for tool in tools: - tool_id = tool.get('id').lower().rstrip('/') - if tool_id not in self.tools: - self.tools[tool_id] = list() + raw_tool_id = tool.get('id') + tool_class = tool.get('class') + if raw_tool_id is not None: + assert tool_class is None + tool_id = raw_tool_id.lower().rstrip('/') + if tool_id not in self.tools: + self.tools[tool_id] = list() + else: + assert tool_class in VALID_TOOL_CLASSES, tool_class + if tool_class not in self.tool_classes: + self.tool_classes[tool_class] = list() + params = tool.get("params") if params is None: params = {} @@ -486,7 +498,12 @@ class JobConfiguration(ConfiguresHandlers): tool["params"] = params if "environment" in tool: tool["destination"] = tool.pop("environment") - self.tools[tool_id].append(JobToolConfiguration(**dict(tool.items()))) + + jtc = JobToolConfiguration(**dict(tool.items())) + if raw_tool_id: + self.tools[tool_id].append(jtc) + else: + self.tool_classes[tool_class].append(jtc) types = dict(registered_user_concurrent_jobs=int, anonymous_user_concurrent_jobs=int, @@ -693,7 +710,7 @@ class JobConfiguration(ConfiguresHandlers): return JobToolConfiguration(id='_default_', handler=self.default_handler_id, destination=self.default_destination_id) # Called upon instantiation of a Tool object - def get_job_tool_configurations(self, ids): + def get_job_tool_configurations(self, ids, tool_classes): """ Get all configured JobToolConfigurations for a tool ID, or, if given a list of IDs, the JobToolConfigurations for the first id in ``ids`` @@ -713,6 +730,7 @@ class JobConfiguration(ConfiguresHandlers): * Tool config tool id: ``filter_tool`` """ rval = [] + match_found = False # listify if ids is a single (string) id ids = util.listify(ids) for id in ids: @@ -723,11 +741,16 @@ class JobConfiguration(ConfiguresHandlers): for job_tool_configuration in self.tools[id]: if not job_tool_configuration.params: break - else: - rval.append(self.default_job_tool_configuration) rval.extend(self.tools[id]) - break - else: + match_found = True + + if not match_found: + for tool_class in tool_classes: + if tool_class in self.tool_classes: + rval.extend(self.tool_classes[tool_class]) + match_found = True + break + if not match_found: rval.append(self.default_job_tool_configuration) return rval diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 8f8eed52b6a..662ce439517 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -490,6 +490,7 @@ class Tool(Dictifiable): requires_setting_metadata = True produces_entry_points = False default_tool_action = DefaultToolAction + tool_type_local = False dict_collection_visible_keys = ['id', 'name', 'version', 'description', 'labels'] def __init__(self, config_file, tool_source, app, guid=None, repository_id=None, tool_shed_repository=None, allow_code_files=True, dynamic=False): @@ -897,7 +898,16 @@ class Tool(Dictifiable): # In the toolshed context, there is no job config. if hasattr(self.app, 'job_config'): - self.job_tool_configurations = self.app.job_config.get_job_tool_configurations(self_ids) + # Order of this list must match documentation in job_conf.sample_advanced.yml + tool_classes = [] + if self.tool_type_local: + tool_classes.append("local") + elif self.old_id in ['upload1', '__DATA_FETCH__']: + tool_classes.append("local") + if self.requires_galaxy_python_environment: + tool_classes.append("requires_galaxy") + + self.job_tool_configurations = self.app.job_config.get_job_tool_configurations(self_ids, tool_classes) # Is this a 'hidden' tool (hidden in tool menu) self.hidden = tool_source.parse_hidden() @@ -2431,6 +2441,7 @@ class OutputParameterJSONTool(Tool): class ExpressionTool(Tool): requires_js_runtime = True tool_type = 'expression' + tool_type_local = True EXPRESSION_INPUTS_NAME = "_expression_inputs_.json" def parse_command(self, tool_source): @@ -2772,6 +2783,7 @@ class DataManagerTool(OutputParameterJSONTool): class DatabaseOperationTool(Tool): default_tool_action = ModelOperationToolAction require_dataset_ok = True + tool_type_local = True @property def valid_input_states(self): diff --git a/lib/galaxy/webapps/galaxy/job_config_schema.yml b/lib/galaxy/webapps/galaxy/job_config_schema.yml index f025bc85be1..63b5532a7af 100644 --- a/lib/galaxy/webapps/galaxy/job_config_schema.yml +++ b/lib/galaxy/webapps/galaxy/job_config_schema.yml @@ -81,6 +81,8 @@ mapping: mapping: id: type: str + class: + type: str handler: type: str environment: diff --git a/test/integration/delay_job_conf.yml b/test/integration/delay_job_conf.yml index 61021c1fe5e..b2d5781f6ec 100644 --- a/test/integration/delay_job_conf.yml +++ b/test/integration/delay_job_conf.yml @@ -14,5 +14,5 @@ execution: upload_dest: runner: local tools: - - id: upload1 + - class: local destination: upload_dest diff --git a/test/integration/embedded_pulsar_job_conf.xml b/test/integration/embedded_pulsar_job_conf.xml index f7d08cb3347..26c1f158189 100644 --- a/test/integration/embedded_pulsar_job_conf.xml +++ b/test/integration/embedded_pulsar_job_conf.xml @@ -13,6 +13,6 @@ - + diff --git a/test/integration/embedded_pulsar_metadata_extended_job_conf.yml b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml index 76c7b8b5d22..aeab99c5454 100644 --- a/test/integration/embedded_pulsar_metadata_extended_job_conf.yml +++ b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml @@ -20,5 +20,5 @@ execution: action: none tools: -- id: upload1 +- class: local environment: local diff --git a/test/integration/embedded_pulsar_metadata_job_conf.yml b/test/integration/embedded_pulsar_metadata_job_conf.yml index 76520bf4ed5..62e9ddcee98 100644 --- a/test/integration/embedded_pulsar_metadata_job_conf.yml +++ b/test/integration/embedded_pulsar_metadata_job_conf.yml @@ -15,5 +15,5 @@ execution: default_file_action: copy tools: -- id: upload1 +- class: local environment: local diff --git a/test/integration/embedded_pulsar_singularity_job_conf.yml b/test/integration/embedded_pulsar_singularity_job_conf.yml index cdcdedc19ff..d224519c951 100644 --- a/test/integration/embedded_pulsar_singularity_job_conf.yml +++ b/test/integration/embedded_pulsar_singularity_job_conf.yml @@ -15,5 +15,5 @@ execution: singularity_required: true tools: -- id: upload1 +- class: local environment: local diff --git a/test/integration/resubmission_default_job_conf.yml b/test/integration/resubmission_default_job_conf.yml index accb17eabd3..2f0b2321694 100644 --- a/test/integration/resubmission_default_job_conf.yml +++ b/test/integration/resubmission_default_job_conf.yml @@ -29,6 +29,6 @@ resources: test: [test_name,failure_state,initial_target_environment,run_for] tools: - - id: upload1 + - class: local environment: local resources: upload diff --git a/test/integration/resubmission_dynamic_job_conf.xml b/test/integration/resubmission_dynamic_job_conf.xml index 6221974fdc4..cf139722b83 100644 --- a/test/integration/resubmission_dynamic_job_conf.xml +++ b/test/integration/resubmission_dynamic_job_conf.xml @@ -26,7 +26,7 @@ - + diff --git a/test/integration/resubmission_job_conf.yml b/test/integration/resubmission_job_conf.yml index e55c236c718..cc2cb9e8998 100644 --- a/test/integration/resubmission_job_conf.yml +++ b/test/integration/resubmission_job_conf.yml @@ -109,6 +109,6 @@ resources: test: [test_name,failure_state,initial_target_environment,run_for] tools: - - id: upload1 + - class: local environment: local resources: upload diff --git a/test/integration/resubmission_pulsar_job_conf.xml b/test/integration/resubmission_pulsar_job_conf.xml index 9ee1c867ae2..da6a2cd6559 100644 --- a/test/integration/resubmission_pulsar_job_conf.xml +++ b/test/integration/resubmission_pulsar_job_conf.xml @@ -22,6 +22,6 @@ - + diff --git a/test/integration/singularity_job_conf.xml b/test/integration/singularity_job_conf.xml index ccdc921727e..1a500dcab3f 100644 --- a/test/integration/singularity_job_conf.xml +++ b/test/integration/singularity_job_conf.xml @@ -15,7 +15,7 @@ - + diff --git a/test/unit/jobs/job_conf.sample_advanced.yml b/test/unit/jobs/job_conf.sample_advanced.yml index c0edf199626..89ae458a310 100644 --- a/test/unit/jobs/job_conf.sample_advanced.yml +++ b/test/unit/jobs/job_conf.sample_advanced.yml @@ -1016,6 +1016,14 @@ tools: id: foo handler: handler0 source: trackster +- + # Classes can be used to map groups of tools - the current classes include + # - local (these special tools that aren't parameterized for remote execution - expression tools, upload, etc..) + # - requires_galaxy (these special tools require Galaxy's Python environment during execution) + # If a tool matches multiple classes, it will match the first class according to the order listed + # above in this document (local, requires_galaxy). + class: local + environment: local resources: default: default diff --git a/test/unit/tools_support.py b/test/unit/tools_support.py index 5c86cd64dbd..b3f5326b519 100644 --- a/test/unit/tools_support.py +++ b/test/unit/tools_support.py @@ -94,7 +94,7 @@ class UsesTools: self.app.config.drmaa_external_runjob_script = "" self.app.config.tool_secret = "testsecret" self.app.config.track_jobs_in_database = False - self.app.job_config["get_job_tool_configurations"] = lambda ids: [Bunch(handler=Bunch())] + self.app.job_config["get_job_tool_configurations"] = lambda ids, tool_classes: [Bunch(handler=Bunch())] def __setup_tool(self): tool_source = get_tool_source(self.tool_file)