Allow specifying useful classes of tools for mapping in job conf.

lib/galaxy/jobs/__init__.py

@@ -82,6 +82,7 @@ DEFAULT_JOB_SHELL = '/bin/bash'
 DEFAULT_LOCAL_WORKERS = 4
 
 DEFAULT_CLEANUP_JOB = "always"
+VALID_TOOL_CLASSES = ["local", "requires_galaxy"]
 
 
 class JobDestination(Bunch):
@@ -244,10 +245,10 @@ def job_config_xml_to_dict(config, root):
     tools = root.find('tools')
     config_dict['tools'] = []
     if tools is not None:
-        for tool in ConfiguresHandlers._findall_with_required(tools, 'tool'):
+        for tool in tools.findall('tool'):
             # There can be multiple definitions with identical ids, but different params
             tool_mapping_conf = {}
-            for key in ['handler', 'destination', 'id', 'resources']:
+            for key in ['handler', 'destination', 'id', 'resources', 'class']:
                 value = tool.get(key)
                 if value:
                     if key == "destination":
@@ -283,6 +284,7 @@ class JobConfiguration(ConfiguresHandlers):
     handlers: dict
     handler_runner_plugins: Dict[str, str]
     tools: Dict[str, list]
+    tool_classes: Dict[str, list]
     resource_groups: Dict[str, list]
     destinations: Dict[str, tuple]
     resource_parameters: Dict[str, Any]
@@ -317,6 +319,7 @@ class JobConfiguration(ConfiguresHandlers):
         self.destinations = {}
         self.default_destination_id = None
         self.tools = {}
+        self.tool_classes = {}
         self.resource_groups = {}
         self.default_resource_group = None
         self.resource_parameters = {}
@@ -473,9 +476,18 @@ class JobConfiguration(ConfiguresHandlers):
 
         tools = job_config_dict.get('tools', [])
         for tool in tools:
-            tool_id = tool.get('id').lower().rstrip('/')
-            if tool_id not in self.tools:
-                self.tools[tool_id] = list()
+            raw_tool_id = tool.get('id')
+            tool_class = tool.get('class')
+            if raw_tool_id is not None:
+                assert tool_class is None
+                tool_id = raw_tool_id.lower().rstrip('/')
+                if tool_id not in self.tools:
+                    self.tools[tool_id] = list()
+            else:
+                assert tool_class in VALID_TOOL_CLASSES, tool_class
+                if tool_class not in self.tool_classes:
+                    self.tool_classes[tool_class] = list()
+
             params = tool.get("params")
             if params is None:
                 params = {}
@@ -486,7 +498,12 @@ class JobConfiguration(ConfiguresHandlers):
                 tool["params"] = params
             if "environment" in tool:
                 tool["destination"] = tool.pop("environment")
-            self.tools[tool_id].append(JobToolConfiguration(**dict(tool.items())))
+
+            jtc = JobToolConfiguration(**dict(tool.items()))
+            if raw_tool_id:
+                self.tools[tool_id].append(jtc)
+            else:
+                self.tool_classes[tool_class].append(jtc)
 
         types = dict(registered_user_concurrent_jobs=int,
                      anonymous_user_concurrent_jobs=int,
@@ -693,7 +710,7 @@ class JobConfiguration(ConfiguresHandlers):
         return JobToolConfiguration(id='_default_', handler=self.default_handler_id, destination=self.default_destination_id)
 
     # Called upon instantiation of a Tool object
-    def get_job_tool_configurations(self, ids):
+    def get_job_tool_configurations(self, ids, tool_classes):
         """
         Get all configured JobToolConfigurations for a tool ID, or, if given
         a list of IDs, the JobToolConfigurations for the first id in ``ids``
@@ -713,6 +730,7 @@ class JobConfiguration(ConfiguresHandlers):
         * Tool config tool id: ``filter_tool``
         """
         rval = []
+        match_found = False
         # listify if ids is a single (string) id
         ids = util.listify(ids)
         for id in ids:
@@ -723,11 +741,16 @@ class JobConfiguration(ConfiguresHandlers):
                 for job_tool_configuration in self.tools[id]:
                     if not job_tool_configuration.params:
                         break
-                else:
-                    rval.append(self.default_job_tool_configuration)
                 rval.extend(self.tools[id])
-                break
-        else:
+                match_found = True
+
+        if not match_found:
+            for tool_class in tool_classes:
+                if tool_class in self.tool_classes:
+                    rval.extend(self.tool_classes[tool_class])
+                    match_found = True
+                    break
+        if not match_found:
             rval.append(self.default_job_tool_configuration)
         return rval
 

lib/galaxy/tools/__init__.py

@@ -490,6 +490,7 @@ class Tool(Dictifiable):
     requires_setting_metadata = True
     produces_entry_points = False
     default_tool_action = DefaultToolAction
+    tool_type_local = False
     dict_collection_visible_keys = ['id', 'name', 'version', 'description', 'labels']
 
     def __init__(self, config_file, tool_source, app, guid=None, repository_id=None, tool_shed_repository=None, allow_code_files=True, dynamic=False):
@@ -897,7 +898,16 @@ class Tool(Dictifiable):
 
         # In the toolshed context, there is no job config.
         if hasattr(self.app, 'job_config'):
-            self.job_tool_configurations = self.app.job_config.get_job_tool_configurations(self_ids)
+            # Order of this list must match documentation in job_conf.sample_advanced.yml
+            tool_classes = []
+            if self.tool_type_local:
+                tool_classes.append("local")
+            elif self.old_id in ['upload1', '__DATA_FETCH__']:
+                tool_classes.append("local")
+            if self.requires_galaxy_python_environment:
+                tool_classes.append("requires_galaxy")
+
+            self.job_tool_configurations = self.app.job_config.get_job_tool_configurations(self_ids, tool_classes)
 
         # Is this a 'hidden' tool (hidden in tool menu)
         self.hidden = tool_source.parse_hidden()
@@ -2431,6 +2441,7 @@ class OutputParameterJSONTool(Tool):
 class ExpressionTool(Tool):
     requires_js_runtime = True
     tool_type = 'expression'
+    tool_type_local = True
     EXPRESSION_INPUTS_NAME = "_expression_inputs_.json"
 
     def parse_command(self, tool_source):
@@ -2772,6 +2783,7 @@ class DataManagerTool(OutputParameterJSONTool):
 class DatabaseOperationTool(Tool):
     default_tool_action = ModelOperationToolAction
     require_dataset_ok = True
+    tool_type_local = True
 
     @property
     def valid_input_states(self):

lib/galaxy/webapps/galaxy/job_config_schema.yml

@@ -81,6 +81,8 @@ mapping:
         mapping:
           id:
             type: str
+          class:
+            type: str
           handler:
             type: str
           environment:

test/integration/delay_job_conf.yml

@@ -14,5 +14,5 @@ execution:
     upload_dest:
       runner: local
 tools:
-  - id: upload1
+  - class: local
     destination: upload_dest

test/integration/embedded_pulsar_job_conf.xml

@@ -13,6 +13,6 @@
         </destination>
     </destinations>
     <tools>
-        <tool id="upload1" destination="local" />
+        <tool class="local" destination="local" />
     </tools>
 </job_conf>

test/integration/embedded_pulsar_metadata_extended_job_conf.yml

@@ -20,5 +20,5 @@ execution:
             action: none
 
 tools:
-- id: upload1
+- class: local
   environment: local

test/integration/embedded_pulsar_metadata_job_conf.yml

@@ -15,5 +15,5 @@ execution:
       default_file_action: copy
 
 tools:
-- id: upload1
+- class: local
   environment: local

test/integration/embedded_pulsar_singularity_job_conf.yml

@@ -15,5 +15,5 @@ execution:
       singularity_required: true
 
 tools:
-- id: upload1
+- class: local
   environment: local

test/integration/resubmission_default_job_conf.yml

@@ -29,6 +29,6 @@ resources:
     test: [test_name,failure_state,initial_target_environment,run_for]
 
 tools:
-  - id: upload1
+  - class: local
     environment: local
     resources: upload

test/integration/resubmission_dynamic_job_conf.xml

@@ -26,7 +26,7 @@
     </destinations>
 
     <tools>
-        <tool id="upload1" destination="local" />
+        <tool class="local" destination="local" />
     </tools>
 
 </job_conf>

test/integration/resubmission_job_conf.yml

@@ -109,6 +109,6 @@ resources:
     test: [test_name,failure_state,initial_target_environment,run_for]
 
 tools:
-  - id: upload1
+  - class: local
     environment: local
     resources: upload

test/integration/resubmission_pulsar_job_conf.xml

@@ -22,6 +22,6 @@
     </destinations>
 
     <tools>
-        <tool id="upload1" destination="local" resources="upload" />
+        <tool class="local" destination="local" resources="upload" />
     </tools>
 </job_conf>

test/integration/singularity_job_conf.xml

@@ -15,7 +15,7 @@
     </destinations>
 
     <tools>
-        <tool id="upload1" destination="local_upload" />
+        <tool class="local" destination="local_upload" />
     </tools>
 
 </job_conf>

test/unit/jobs/job_conf.sample_advanced.yml

@@ -1016,6 +1016,14 @@ tools:
   id: foo
   handler: handler0
   source: trackster
+- 
+  # Classes can be used to map groups of tools - the current classes include
+  # - local (these special tools that aren't parameterized for remote execution - expression tools, upload, etc..)
+  # - requires_galaxy (these special tools require Galaxy's Python environment during execution)
+  # If a tool matches multiple classes, it will match the first class according to the order listed
+  # above in this document (local, requires_galaxy).
+  class: local
+  environment: local
 
 resources:
   default: default

test/unit/tools_support.py

@@ -94,7 +94,7 @@ class UsesTools:
         self.app.config.drmaa_external_runjob_script = ""
         self.app.config.tool_secret = "testsecret"
         self.app.config.track_jobs_in_database = False
-        self.app.job_config["get_job_tool_configurations"] = lambda ids: [Bunch(handler=Bunch())]
+        self.app.job_config["get_job_tool_configurations"] = lambda ids, tool_classes: [Bunch(handler=Bunch())]
 
     def __setup_tool(self):
         tool_source = get_tool_source(self.tool_file)