From a0b27e49a318d660a79b69353aca4a932d85f9ec Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Tue, 20 Feb 2024 18:04:14 +0100 Subject: [PATCH 1/8] Always disable check_values for data_source tools --- lib/galaxy/tools/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 311987a399e..a8e926bd5f7 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2904,6 +2904,8 @@ class DataSourceTool(OutputParameterJSONTool): super().parse_inputs(tool_source) # Open all data_source tools in _top. self.target = "_top" + # data_source tools cannot check param values + self.check_values = False if "GALAXY_URL" not in self.inputs: self.inputs["GALAXY_URL"] = self._build_GALAXY_URL_parameter() self.inputs_by_page[0]["GALAXY_URL"] = self.inputs["GALAXY_URL"] From 55409ad50b10e93fe7ae728e080b68ea9b6c0420 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Wed, 21 Feb 2024 12:07:10 +0100 Subject: [PATCH 2/8] Update linting of data source tools --- lib/galaxy/tool_util/linters/_util.py | 2 +- lib/galaxy/tool_util/linters/xml_order.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/linters/_util.py b/lib/galaxy/tool_util/linters/_util.py index 025926259ae..84ee89d1d98 100644 --- a/lib/galaxy/tool_util/linters/_util.py +++ b/lib/galaxy/tool_util/linters/_util.py @@ -3,7 +3,7 @@ import re def is_datasource(tool_xml): """Returns true if the tool is a datasource tool""" - return tool_xml.getroot().attrib.get("tool_type", "") == "data_source" + return tool_xml.getroot().attrib.get("tool_type", "") in ["data_source", "data_source_async"] def is_valid_cheetah_placeholder(name): diff --git a/lib/galaxy/tool_util/linters/xml_order.py b/lib/galaxy/tool_util/linters/xml_order.py index af3f1f2cdd6..6ead8908c2b 100644 --- a/lib/galaxy/tool_util/linters/xml_order.py +++ b/lib/galaxy/tool_util/linters/xml_order.py @@ -42,6 +42,7 @@ TAG_ORDER = [ DATASOURCE_TAG_ORDER = [ "description", "macros", + "requirements", "command", "configfiles", "inputs", @@ -62,7 +63,7 @@ class XMLOrder(Linter): return tool_root = tool_xml.getroot() - if tool_root.attrib.get("tool_type", "") == "data_source": + if tool_root.attrib.get("tool_type", "") in ["data_source", "data_source_async"]: tag_ordering = DATASOURCE_TAG_ORDER else: tag_ordering = TAG_ORDER From a5eeb951c555f73e4eb3a792451f3ef26741e342 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 14:40:45 +0100 Subject: [PATCH 3/8] Eliminate duplicate params translation and make params check more stringent --- .../webapps/galaxy/controllers/tool_runner.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index c4ccf28b2a1..c40de6dedcc 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -78,20 +78,30 @@ class ToolRunner(BaseUIController): # execute tool without displaying form # (used for datasource tools, but note that data_source_async tools # are handled separately by the async controller) - params = galaxy.util.Params(kwd, sanitize=False) - if tool.tool_type == "data_source": - # preserve original params sent by the remote server as extra dict - params.update({"incoming_request_params": params.__dict__.copy()}) - # do param translation here, used by datasource tools + params = galaxy.util.Params(kwd, sanitize=False).__dict__ if tool.input_translator: - tool.input_translator.translate(params) - if "runtool_btn" not in params.__dict__ and "URL" not in params.__dict__: - error("Tool execution through the `tool_runner` requires a `runtool_btn` flag or `URL` parameter.") + # perform test translation of the incoming params without affecting originals + # the actual translation will happen later + # this is only for checking if we end up with required parameters + test_params = params.copy() + tool.input_translator.translate(test_params) + else: + test_params = params + if tool.tool_type == "data_source": + if "URL" not in test_params: + error("Execution of `data_source` tools requires a `URL` parameter") + # preserve original params sent by the remote server as extra dict + # before in-place translation happens + params.update({"incoming_request_params": params.copy()}) + else: + if "runtool_btn" not in test_params: + error("Tool execution through the `tool_runner` requires a `runtool_btn` flag") + # We may be visiting Galaxy for the first time ( e.g., sending data from UCSC ), # so make sure to create a new history if we've never had one before. history = tool.get_default_history_by_trans(trans, create=True) try: - vars = tool.handle_input(trans, params.__dict__, history=history) + vars = tool.handle_input(trans, params, history=history) except Exception as e: error(galaxy.util.unicodify(e)) if len(params) > 0: From fb07082bc1083df7ab0f57311df85cdd64c4c9c1 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 15:48:55 +0100 Subject: [PATCH 4/8] Drop unexpected incoming params in data_source tools like in data_source_async tools --- lib/galaxy/tools/parameters/input_translation.py | 4 ++++ lib/galaxy/webapps/galaxy/controllers/tool_runner.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/parameters/input_translation.py b/lib/galaxy/tools/parameters/input_translation.py index 035a2c30ceb..267d91dd72a 100644 --- a/lib/galaxy/tools/parameters/input_translation.py +++ b/lib/galaxy/tools/parameters/input_translation.py @@ -62,6 +62,8 @@ class ToolInputTranslator: value_trans = {} append_param = None + rval.vocabulary.add(remote_name) + value_trans_elem = req_param.find("value_translation") if value_trans_elem is not None: for value_elem in value_trans_elem.findall("value"): @@ -81,6 +83,7 @@ class ToolInputTranslator: value_missing = value_elem.get("missing") if None not in [value_name, value_missing]: append_dict[value_name] = value_missing + rval.vocabulary.add(value_name) append_param = Bunch( separator=separator, first_separator=first_separator, join_str=join_str, append_dict=append_dict ) @@ -93,6 +96,7 @@ class ToolInputTranslator: def __init__(self): self.param_trans_dict = {} + self.vocabulary = set() def translate(self, params): """ diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index c40de6dedcc..23fa8f44fa3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -91,8 +91,14 @@ class ToolRunner(BaseUIController): if "URL" not in test_params: error("Execution of `data_source` tools requires a `URL` parameter") # preserve original params sent by the remote server as extra dict - # before in-place translation happens + # before in-place translation happens, then clean the incoming params params.update({"incoming_request_params": params.copy()}) + if tool.input_translator: + for k in list(params.keys()): + if k not in tool.input_translator.vocabulary and k not in ("URL", "incoming_request_params"): + # the remote server has sent a param + # that the tool is not expecting -> drop it + del params[k] else: if "runtool_btn" not in test_params: error("Tool execution through the `tool_runner` requires a `runtool_btn` flag") From 76968a68c77008504b13b8d9fde78605870f26e9 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 17:29:33 +0100 Subject: [PATCH 5/8] Adjust UCSC Table browser tools to code changes Since the code now drops undeclared incoming parameters it enforces cleaner wrapper style. --- tools/data_source/ucsc_tablebrowser.xml | 3 ++- tools/data_source/ucsc_tablebrowser_archaea.xml | 3 ++- tools/data_source/ucsc_tablebrowser_test.xml | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/data_source/ucsc_tablebrowser.xml b/tools/data_source/ucsc_tablebrowser.xml index 19852ecf6fc..5e7f7744196 100644 --- a/tools/data_source/ucsc_tablebrowser.xml +++ b/tools/data_source/ucsc_tablebrowser.xml @@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz + @@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz - + diff --git a/tools/data_source/ucsc_tablebrowser_archaea.xml b/tools/data_source/ucsc_tablebrowser_archaea.xml index a1707e441a1..e67a30fc953 100644 --- a/tools/data_source/ucsc_tablebrowser_archaea.xml +++ b/tools/data_source/ucsc_tablebrowser_archaea.xml @@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz + @@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz - + diff --git a/tools/data_source/ucsc_tablebrowser_test.xml b/tools/data_source/ucsc_tablebrowser_test.xml index 9782f8d2b41..0e04e039bea 100644 --- a/tools/data_source/ucsc_tablebrowser_test.xml +++ b/tools/data_source/ucsc_tablebrowser_test.xml @@ -27,6 +27,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz + @@ -41,7 +42,7 @@ python '$__tool_directory__/data_source.py' '$output' $__app__.config.output_siz - + From 82ca92ac536259df5cafc9a6a56ff5d6e2f10be7 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Thu, 22 Feb 2024 17:56:03 +0100 Subject: [PATCH 6/8] Fix tool validation --- lib/galaxy/tool_util/xsd/galaxy.xsd | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 79dec3a85fc..3ad702ce749 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -6688,6 +6688,7 @@ Examples are included in the test tools directory including: + From 2554a1b6b95870ac169b451e4a37bc0d7863c4e6 Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Fri, 23 Feb 2024 10:59:07 +0100 Subject: [PATCH 7/8] Reuse is_datasource instead of duplicating code --- lib/galaxy/tool_util/linters/xml_order.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/linters/xml_order.py b/lib/galaxy/tool_util/linters/xml_order.py index 6ead8908c2b..13132e909c4 100644 --- a/lib/galaxy/tool_util/linters/xml_order.py +++ b/lib/galaxy/tool_util/linters/xml_order.py @@ -4,16 +4,18 @@ For more information on the IUC standard for XML block order see - https://github.com/galaxy-iuc/standards. """ -from typing import TYPE_CHECKING +from typing import ( + Optional, + TYPE_CHECKING, +) from galaxy.tool_util.lint import Linter +from ._util import is_datasource if TYPE_CHECKING: from galaxy.tool_util.lint import LintContext from galaxy.tool_util.parser.interface import ToolSource -from typing import Optional - # https://github.com/galaxy-iuc/standards # https://github.com/galaxy-iuc/standards/pull/7/files TAG_ORDER = [ @@ -63,7 +65,7 @@ class XMLOrder(Linter): return tool_root = tool_xml.getroot() - if tool_root.attrib.get("tool_type", "") in ["data_source", "data_source_async"]: + if is_datasource(tool_xml): tag_ordering = DATASOURCE_TAG_ORDER else: tag_ordering = TAG_ORDER From 41defa80ac33ba37cde28401a99470029607374f Mon Sep 17 00:00:00 2001 From: Wolfgang Maier Date: Sat, 24 Feb 2024 09:48:33 +0100 Subject: [PATCH 8/8] Make cleaning of request params profile-dependent as suggested by @martenson --- lib/galaxy/tool_util/xsd/galaxy.xsd | 1 + lib/galaxy/tools/__init__.py | 7 +++++++ lib/galaxy/webapps/galaxy/controllers/async.py | 2 +- lib/galaxy/webapps/galaxy/controllers/tool_runner.py | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 3ad702ce749..812e8019819 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -68,6 +68,7 @@ List of behavior changes associated with profile versions: ### 24.0 - Do not use Galaxy python environment for `data_source_async` tools. +- Drop request parameters received by data source tools that are not declared in `` section. ### Examples diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a8e926bd5f7..d75aed1b09f 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2895,6 +2895,13 @@ class DataSourceTool(OutputParameterJSONTool): tool_type = "data_source" default_tool_action = DataSourceToolAction + @property + def wants_params_cleaned(self): + """Indicates whether received, but undeclared request params should be cleaned.""" + if self.profile < 24.0: + return False + return True + def _build_GALAXY_URL_parameter(self): return ToolParameter.build( self, XML(f'') diff --git a/lib/galaxy/webapps/galaxy/controllers/async.py b/lib/galaxy/webapps/galaxy/controllers/async.py index d9ecae7f230..4a9615fc700 100644 --- a/lib/galaxy/webapps/galaxy/controllers/async.py +++ b/lib/galaxy/webapps/galaxy/controllers/async.py @@ -86,7 +86,7 @@ class ASync(BaseUIController): translator.galaxy_name for translator in tool.input_translator.param_trans_dict.values() } for param in params: - if param in tool_declared_params: + if param in tool_declared_params or not tool.wants_params_cleaned: params_dict[param] = params.get(param, None) params = params_dict diff --git a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py index 23fa8f44fa3..f85e9b2d9a3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -93,7 +93,7 @@ class ToolRunner(BaseUIController): # preserve original params sent by the remote server as extra dict # before in-place translation happens, then clean the incoming params params.update({"incoming_request_params": params.copy()}) - if tool.input_translator: + if tool.input_translator and tool.wants_params_cleaned: for k in list(params.keys()): if k not in tool.input_translator.vocabulary and k not in ("URL", "incoming_request_params"): # the remote server has sent a param