Merge pull request #17515 from wm75/data_source_fixes2

Fixing bugs with data_source tools - part 2

lib/galaxy/tool_util/linters/_util.py

@@ -3,7 +3,7 @@ import re
 
 def is_datasource(tool_xml):
     """Returns true if the tool is a datasource tool"""
-    return tool_xml.getroot().attrib.get("tool_type", "") == "data_source"
+    return tool_xml.getroot().attrib.get("tool_type", "") in ["data_source", "data_source_async"]
 
 
 def is_valid_cheetah_placeholder(name):

lib/galaxy/tool_util/linters/xml_order.py

@@ -4,16 +4,18 @@ For more information on the IUC standard for XML block order see -
 https://github.com/galaxy-iuc/standards.
 """
 
-from typing import TYPE_CHECKING
+from typing import (
+    Optional,
+    TYPE_CHECKING,
+)
 
 from galaxy.tool_util.lint import Linter
+from ._util import is_datasource
 
 if TYPE_CHECKING:
     from galaxy.tool_util.lint import LintContext
     from galaxy.tool_util.parser.interface import ToolSource
 
-from typing import Optional
-
 # https://github.com/galaxy-iuc/standards
 # https://github.com/galaxy-iuc/standards/pull/7/files
 TAG_ORDER = [
@@ -42,6 +44,7 @@ TAG_ORDER = [
 DATASOURCE_TAG_ORDER = [
     "description",
     "macros",
+    "requirements",
     "command",
     "configfiles",
     "inputs",
@@ -62,7 +65,7 @@ class XMLOrder(Linter):
             return
         tool_root = tool_xml.getroot()
 
-        if tool_root.attrib.get("tool_type", "") == "data_source":
+        if is_datasource(tool_xml):
             tag_ordering = DATASOURCE_TAG_ORDER
         else:
             tag_ordering = TAG_ORDER

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -68,6 +68,7 @@ List of behavior changes associated with profile versions:
 ### 24.0
 
 - Do not use Galaxy python environment for `data_source_async` tools.
+- Drop request parameters received by data source tools that are not declared in `<request_param_translation>` section.
 
 ### Examples
 
@@ -6690,6 +6691,7 @@ Examples are included in the test tools directory including:
       <xs:enumeration value="dbkey" />
       <xs:enumeration value="organism" />
       <xs:enumeration value="table" />
+      <xs:enumeration value="position" />
       <xs:enumeration value="description" />
       <xs:enumeration value="name" />
       <xs:enumeration value="info" />

lib/galaxy/tools/__init__.py

@@ -2897,6 +2897,13 @@ class DataSourceTool(OutputParameterJSONTool):
     tool_type = "data_source"
     default_tool_action = DataSourceToolAction
 
+    @property
+    def wants_params_cleaned(self):
+        """Indicates whether received, but undeclared request params should be cleaned."""
+        if self.profile < 24.0:
+            return False
+        return True
+
     def _build_GALAXY_URL_parameter(self):
         return ToolParameter.build(
             self, XML(f'<param name="GALAXY_URL" type="baseurl" value="/tool_runner?tool_id={self.id}" />')
@@ -2906,6 +2913,8 @@ class DataSourceTool(OutputParameterJSONTool):
         super().parse_inputs(tool_source)
         # Open all data_source tools in _top.
         self.target = "_top"
+        # data_source tools cannot check param values
+        self.check_values = False
         if "GALAXY_URL" not in self.inputs:
             self.inputs["GALAXY_URL"] = self._build_GALAXY_URL_parameter()
             self.inputs_by_page[0]["GALAXY_URL"] = self.inputs["GALAXY_URL"]

lib/galaxy/tools/parameters/input_translation.py

@@ -62,6 +62,8 @@ class ToolInputTranslator:
             value_trans = {}
             append_param = None
 
+            rval.vocabulary.add(remote_name)
+
             value_trans_elem = req_param.find("value_translation")
             if value_trans_elem is not None:
                 for value_elem in value_trans_elem.findall("value"):
@@ -81,6 +83,7 @@ class ToolInputTranslator:
                     value_missing = value_elem.get("missing")
                     if None not in [value_name, value_missing]:
                         append_dict[value_name] = value_missing
+                        rval.vocabulary.add(value_name)
                 append_param = Bunch(
                     separator=separator, first_separator=first_separator, join_str=join_str, append_dict=append_dict
                 )
@@ -93,6 +96,7 @@ class ToolInputTranslator:
 
     def __init__(self):
         self.param_trans_dict = {}
+        self.vocabulary = set()
 
     def translate(self, params):
         """

lib/galaxy/webapps/galaxy/controllers/async.py

@@ -86,7 +86,7 @@ class ASync(BaseUIController):
                     translator.galaxy_name for translator in tool.input_translator.param_trans_dict.values()
                 }
                 for param in params:
-                    if param in tool_declared_params:
+                    if param in tool_declared_params or not tool.wants_params_cleaned:
                         params_dict[param] = params.get(param, None)
             params = params_dict
 

lib/galaxy/webapps/galaxy/controllers/tool_runner.py

@@ -78,20 +78,36 @@ class ToolRunner(BaseUIController):
         # execute tool without displaying form
         # (used for datasource tools, but note that data_source_async tools
         # are handled separately by the async controller)
-        params = galaxy.util.Params(kwd, sanitize=False)
-        if tool.tool_type == "data_source":
-            # preserve original params sent by the remote server as extra dict
-            params.update({"incoming_request_params": params.__dict__.copy()})
-        # do param translation here, used by datasource tools
+        params = galaxy.util.Params(kwd, sanitize=False).__dict__
         if tool.input_translator:
-            tool.input_translator.translate(params)
-        if "runtool_btn" not in params.__dict__ and "URL" not in params.__dict__:
-            error("Tool execution through the `tool_runner` requires a `runtool_btn` flag or `URL` parameter.")
+            # perform test translation of the incoming params without affecting originals
+            # the actual translation will happen later
+            # this is only for checking if we end up with required parameters
+            test_params = params.copy()
+            tool.input_translator.translate(test_params)
+        else:
+            test_params = params
+        if tool.tool_type == "data_source":
+            if "URL" not in test_params:
+                error("Execution of `data_source` tools requires a `URL` parameter")
+            # preserve original params sent by the remote server as extra dict
+            # before in-place translation happens, then clean the incoming params
+            params.update({"incoming_request_params": params.copy()})
+            if tool.input_translator and tool.wants_params_cleaned:
+                for k in list(params.keys()):
+                    if k not in tool.input_translator.vocabulary and k not in ("URL", "incoming_request_params"):
+                        # the remote server has sent a param
+                        # that the tool is not expecting -> drop it
+                        del params[k]
+        else:
+            if "runtool_btn" not in test_params:
+                error("Tool execution through the `tool_runner` requires a `runtool_btn` flag")
+
         # We may be visiting Galaxy for the first time ( e.g., sending data from UCSC ),
         # so make sure to create a new history if we've never had one before.
         history = tool.get_default_history_by_trans(trans, create=True)
         try:
-            vars = tool.handle_input(trans, params.__dict__, history=history)
+            vars = tool.handle_input(trans, params, history=history)
         except Exception as e:
             error(galaxy.util.unicodify(e))
         if len(params) > 0:

tools/data_source/ucsc_tablebrowser.xml

@@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz
         <request_param galaxy_name="organism" remote_name="org" missing="unknown species" />
         <request_param galaxy_name="table" remote_name="hgta_table" missing="unknown table" />
         <request_param galaxy_name="description" remote_name="hgta_regionType" missing="no description" />
+        <request_param galaxy_name="position" remote_name="position" missing="unknown position" />
         <request_param galaxy_name="data_type" remote_name="hgta_outputType" missing="auto" >
             <value_translation>
                 <value galaxy_value="auto" remote_value="primaryTable" />
@@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz
     </request_param_translation>
     <uihints minwidth="800"/>
     <outputs>
-        <data name="output" format="tabular" label="${tool.name} on ${organism}: ${table} (#if $description == 'range' then $getVar( 'position', 'unknown position' ) else $description#)"/>
+        <data name="output" format="tabular" label="${tool.name} on ${organism}: ${table} (#if $description == 'range' then $position else $description#)"/>
     </outputs>
     <options sanitize="False" refresh="True"/>
     <citations>

tools/data_source/ucsc_tablebrowser_archaea.xml

@@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz
         <request_param galaxy_name="organism" remote_name="org" missing="unknown species" />
         <request_param galaxy_name="table" remote_name="hgta_table" missing="unknown table" />
         <request_param galaxy_name="description" remote_name="hgta_regionType" missing="no description" />
+        <request_param galaxy_name="position" remote_name="position" missing="unknown position" />
         <request_param galaxy_name="data_type" remote_name="hgta_outputType" missing="auto" >
             <value_translation>
                 <value galaxy_value="auto" remote_value="primaryTable" />
@@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz
     </request_param_translation>
     <uihints minwidth="800"/>
     <outputs>
-        <data name="output" format="tabular" label="${tool.name} on ${organism}: ${table} (#if $description == 'range' then $getVar( 'position', 'unknown position' ) else $description#)"/>
+        <data name="output" format="tabular" label="${tool.name} on ${organism}: ${table} (#if $description == 'range' then $position else $description#)"/>
     </outputs>
     <options sanitize="False" refresh="True"/>
     <citations>

tools/data_source/ucsc_tablebrowser_test.xml

@@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz
         <request_param galaxy_name="organism" remote_name="org" missing="unknown species" />
         <request_param galaxy_name="table" remote_name="hgta_table" missing="unknown table" />
         <request_param galaxy_name="description" remote_name="hgta_regionType" missing="no description" />
+        <request_param galaxy_name="position" remote_name="position" missing="unknown position" />
         <request_param galaxy_name="data_type" remote_name="hgta_outputType" missing="auto" >
             <value_translation>
                 <value galaxy_value="auto" remote_value="primaryTable" />
@@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz
     </request_param_translation>
     <uihints minwidth="800"/>
     <outputs>
-        <data name="output" format="tabular" label="${tool.name} on ${organism}: ${table} (#if $description == 'range' then $getVar( 'position', 'unknown position' ) else $description#)"/>
+        <data name="output" format="tabular" label="${tool.name} on ${organism}: ${table} (#if $description == 'range' then $position else $description#)"/>
     </outputs>
     <options sanitize="False" refresh="True"/>
     <citations>