From 32a1bfddc2049b5302bdbae89172060b261107f2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 26 Jan 2022 12:40:03 +0100 Subject: [PATCH] Bind tool_xml to node_props, so we don't have to pass it around --- lib/galaxy/tool_util/linters/_util.py | 17 ++-- lib/galaxy/tool_util/linters/citations.py | 17 ++-- lib/galaxy/tool_util/linters/command.py | 17 ++-- lib/galaxy/tool_util/linters/help.py | 17 ++-- lib/galaxy/tool_util/linters/inputs.py | 100 +++++++++++----------- lib/galaxy/tool_util/linters/outputs.py | 27 +++--- lib/galaxy/tool_util/linters/stdio.py | 29 ++++--- lib/galaxy/tool_util/linters/tests.py | 43 +++++----- lib/galaxy/tool_util/linters/xml_order.py | 7 +- 9 files changed, 146 insertions(+), 128 deletions(-) diff --git a/lib/galaxy/tool_util/linters/_util.py b/lib/galaxy/tool_util/linters/_util.py index 03e728b38ef..56ab8ef8e76 100644 --- a/lib/galaxy/tool_util/linters/_util.py +++ b/lib/galaxy/tool_util/linters/_util.py @@ -1,14 +1,17 @@ import re -def node_props(node, tool_xml): - if node is None: - return {"line": 0, "fname": None, "xpath": None} - else: - return {"line": node.sourceline, - "fname": node.base, - "xpath": tool_xml.getpath(node)} +def node_props_factory(tool_xml): + def node_props(node): + if node is None: + return {"line": 0, "fname": None, "xpath": None} + else: + return {"line": node.sourceline, + "fname": node.base, + "xpath": tool_xml.getpath(node)} + + return node_props def is_datasource(tool_xml): """Returns true if the tool is a datasource tool""" diff --git a/lib/galaxy/tool_util/linters/citations.py b/lib/galaxy/tool_util/linters/citations.py index 6bab16702f3..ae55b3993a4 100644 --- a/lib/galaxy/tool_util/linters/citations.py +++ b/lib/galaxy/tool_util/linters/citations.py @@ -3,36 +3,37 @@ Citations describe references that should be used when consumers of the tool publish results. """ -from ._util import node_props +from ._util import node_props_factory def lint_citations(tool_xml, lint_ctx): """Ensure tool contains at least one valid citation.""" root = tool_xml.getroot() + node_props = node_props_factory(tool_xml) citations = root.findall("citations") if len(citations) > 1: - lint_ctx.error("More than one citation section found, behavior undefined.", **node_props(citations[1], tool_xml)) + lint_ctx.error("More than one citation section found, behavior undefined.", **node_props(citations[1])) return if len(citations) == 0: - lint_ctx.warn("No citations found, consider adding citations to your tool.", **node_props(root, tool_xml)) + lint_ctx.warn("No citations found, consider adding citations to your tool.", **node_props(root)) return valid_citations = 0 for citation in citations[0]: if citation.tag != "citation": - lint_ctx.warn(f"Unknown tag discovered in citations block [{citation.tag}], will be ignored.", **node_props(citation, tool_xml)) + lint_ctx.warn(f"Unknown tag discovered in citations block [{citation.tag}], will be ignored.", **node_props(citation)) continue citation_type = citation.attrib.get("type") if citation_type not in ('bibtex', 'doi'): - lint_ctx.warn(f"Unknown citation type discovered [{citation_type}], will be ignored.", **node_props(citation, tool_xml)) + lint_ctx.warn(f"Unknown citation type discovered [{citation_type}], will be ignored.", **node_props(citation)) continue if citation.text is None or not citation.text.strip(): - lint_ctx.error(f'Empty {citation_type} citation.', **node_props(citation, tool_xml)) + lint_ctx.error(f'Empty {citation_type} citation.', **node_props(citation)) continue valid_citations += 1 if valid_citations > 0: - lint_ctx.valid(f"Found {valid_citations} likely valid citations.", **node_props(root, tool_xml)) + lint_ctx.valid(f"Found {valid_citations} likely valid citations.", **node_props(root)) else: - lint_ctx.warn("Found no valid citations.", **node_props(root, tool_xml)) + lint_ctx.warn("Found no valid citations.", **node_props(root)) diff --git a/lib/galaxy/tool_util/linters/command.py b/lib/galaxy/tool_util/linters/command.py index fa76d487e3a..d5f428bc3e3 100644 --- a/lib/galaxy/tool_util/linters/command.py +++ b/lib/galaxy/tool_util/linters/command.py @@ -3,26 +3,27 @@ A command description describes how to build the command-line to execute from supplied inputs. """ -from ._util import node_props +from ._util import node_props_factory def lint_command(tool_xml, lint_ctx): """Ensure tool contains exactly one command and check attributes.""" root = tool_xml.getroot() + node_props = node_props_factory(tool_xml) commands = root.findall("command") if len(commands) > 1: - lint_ctx.error("More than one command tag found, behavior undefined.", **node_props(commands[1], tool_xml)) + lint_ctx.error("More than one command tag found, behavior undefined.", **node_props(commands[1])) return if len(commands) == 0: - lint_ctx.error("No command tag found, must specify a command template to execute.", **node_props(root, tool_xml)) + lint_ctx.error("No command tag found, must specify a command template to execute.", **node_props(root)) return command = get_command(tool_xml) if command.text is None: - lint_ctx.error("Command is empty.", **node_props(root, tool_xml)) + lint_ctx.error("Command is empty.", **node_props(root)) elif "TODO" in command.text: - lint_ctx.warn("Command template contains TODO text.", **node_props(command, tool_xml)) + lint_ctx.warn("Command template contains TODO text.", **node_props(command)) command_attrib = command.attrib interpreter_type = None @@ -32,14 +33,14 @@ def lint_command(tool_xml, lint_ctx): elif key == "detect_errors": detect_errors = value if detect_errors not in ["default", "exit_code", "aggressive"]: - lint_ctx.warn(f"Unknown detect_errors attribute [{detect_errors}]", **node_props(command, tool_xml)) + lint_ctx.warn(f"Unknown detect_errors attribute [{detect_errors}]", **node_props(command)) interpreter_info = "" if interpreter_type: interpreter_info = f" with interpreter of type [{interpreter_type}]" if interpreter_type: - lint_ctx.warn("Command uses deprecated 'interpreter' attribute.", **node_props(command, tool_xml)) - lint_ctx.info(f"Tool contains a command{interpreter_info}.", **node_props(command, tool_xml)) + lint_ctx.warn("Command uses deprecated 'interpreter' attribute.", **node_props(command)) + lint_ctx.info(f"Tool contains a command{interpreter_info}.", **node_props(command)) def get_command(tool_xml): diff --git a/lib/galaxy/tool_util/linters/help.py b/lib/galaxy/tool_util/linters/help.py index b62392d8bc3..4e10fed384b 100644 --- a/lib/galaxy/tool_util/linters/help.py +++ b/lib/galaxy/tool_util/linters/help.py @@ -3,37 +3,38 @@ from galaxy.util import ( rst_to_html, unicodify, ) -from ._util import node_props +from ._util import node_props_factory def lint_help(tool_xml, lint_ctx): """Ensure tool contains exactly one valid RST help block.""" # determine node to report for general problems with help root = tool_xml.getroot() + node_props = node_props_factory(tool_xml) helps = root.findall("help") if len(helps) > 1: - lint_ctx.error("More than one help section found, behavior undefined.", **node_props(helps[1], tool_xml)) + lint_ctx.error("More than one help section found, behavior undefined.", **node_props(helps[1])) return if len(helps) == 0: - lint_ctx.warn("No help section found, consider adding a help section to your tool.", **node_props(root, tool_xml)) + lint_ctx.warn("No help section found, consider adding a help section to your tool.", **node_props(root)) return help = helps[0].text or '' if not help.strip(): - lint_ctx.warn("Help section appears to be empty.", **node_props(helps[0], tool_xml)) + lint_ctx.warn("Help section appears to be empty.", **node_props(helps[0])) return - lint_ctx.valid("Tool contains help section.", **node_props(helps[0], tool_xml)) + lint_ctx.valid("Tool contains help section.", **node_props(helps[0])) invalid_rst = rst_invalid(help) if "TODO" in help: - lint_ctx.warn("Help contains TODO text.", **node_props(helps[0], tool_xml)) + lint_ctx.warn("Help contains TODO text.", **node_props(helps[0])) if invalid_rst: - lint_ctx.warn(f"Invalid reStructuredText found in help - [{invalid_rst}].", **node_props(helps[0], tool_xml)) + lint_ctx.warn(f"Invalid reStructuredText found in help - [{invalid_rst}].", **node_props(helps[0])) else: - lint_ctx.valid("Help contains valid reStructuredText.", **node_props(helps[0], tool_xml)) + lint_ctx.valid("Help contains valid reStructuredText.", **node_props(helps[0])) def rst_invalid(text): diff --git a/lib/galaxy/tool_util/linters/inputs.py b/lib/galaxy/tool_util/linters/inputs.py index a800dd58a0d..5277b9507c5 100644 --- a/lib/galaxy/tool_util/linters/inputs.py +++ b/lib/galaxy/tool_util/linters/inputs.py @@ -5,7 +5,7 @@ from galaxy.util import string_as_bool from ._util import ( is_datasource, is_valid_cheetah_placeholder, - node_props + node_props_factory ) from ..parser.util import _parse_name @@ -53,6 +53,7 @@ PARAMETER_VALIDATOR_TYPE_COMPATIBILITY = { def lint_inputs(tool_xml, lint_ctx): """Lint parameters in a tool's inputs block.""" # determine node to report for general problems with outputs + node_props = node_props_factory(tool_xml) tool_node = tool_xml.find("./tool") datasource = is_datasource(tool_xml) inputs = tool_xml.findall("./inputs//param") @@ -61,24 +62,24 @@ def lint_inputs(tool_xml, lint_ctx): num_inputs += 1 param_attrib = param.attrib if "name" not in param_attrib and "argument" not in param_attrib: - lint_ctx.error("Found param input with no name specified.", **node_props(param, tool_xml)) + lint_ctx.error("Found param input with no name specified.", **node_props(param)) continue param_name = _parse_name(param_attrib.get("name"), param_attrib.get("argument")) if "name" in param_attrib and "argument" in param_attrib: if param_attrib.get("name") == _parse_name(None, param_attrib.get("argument")): - lint_ctx.warn(f"Param input [{param_name}] 'name' attribute is redundant if argument implies the same name.", **node_props(param, tool_xml)) + lint_ctx.warn(f"Param input [{param_name}] 'name' attribute is redundant if argument implies the same name.", **node_props(param)) if param_name.strip() == "": - lint_ctx.error("Param input with empty name.", **node_props(param, tool_xml)) + lint_ctx.error("Param input with empty name.", **node_props(param)) elif not is_valid_cheetah_placeholder(param_name): - lint_ctx.warn(f"Param input [{param_name}] is not a valid Cheetah placeholder.", **node_props(param, tool_xml)) + lint_ctx.warn(f"Param input [{param_name}] is not a valid Cheetah placeholder.", **node_props(param)) # TODO lint for params with duplicated name (in inputs & outputs) if "type" not in param_attrib: - lint_ctx.error(f"Param input [{param_name}] input with no type specified.", **node_props(param, tool_xml)) + lint_ctx.error(f"Param input [{param_name}] input with no type specified.", **node_props(param)) continue elif param_attrib["type"].strip() == "": - lint_ctx.error(f"Param input [{param_name}] with empty type specified.", **node_props(param, tool_xml)) + lint_ctx.error(f"Param input [{param_name}] with empty type specified.", **node_props(param)) continue param_type = param_attrib["type"] @@ -86,7 +87,7 @@ def lint_inputs(tool_xml, lint_ctx): if param_type == "data": if "format" not in param_attrib: - lint_ctx.warn(f"Param input [{param_name}] with no format specified - 'data' format will be assumed.", **node_props(param, tool_xml)) + lint_ctx.warn(f"Param input [{param_name}] with no format specified - 'data' format will be assumed.", **node_props(param)) elif param_type == "select": # get dynamic/statically defined options dynamic_options = param.get("dynamic_options", None) @@ -95,15 +96,15 @@ def lint_inputs(tool_xml, lint_ctx): select_options = param.findall('./option') if dynamic_options is not None: - lint_ctx.warn(f"Select parameter [{param_name}] uses deprecated 'dynamic_options' attribute.", **node_props(param, tool_xml)) + lint_ctx.warn(f"Select parameter [{param_name}] uses deprecated 'dynamic_options' attribute.", **node_props(param)) # check if options are defined by exactly one possibility if param.getparent().tag != "conditional": if (dynamic_options is not None) + (len(options) > 0) + (len(select_options) > 0) != 1: - lint_ctx.error(f"Select parameter [{param_name}] options have to be defined by either 'option' children elements, a 'options' element or the 'dynamic_options' attribute.", **node_props(param, tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] options have to be defined by either 'option' children elements, a 'options' element or the 'dynamic_options' attribute.", **node_props(param)) else: if len(select_options) == 0: - lint_ctx.error(f"Select parameter of a conditional [{param_name}] options have to be defined by 'option' children elements.", **node_props(param, tool_xml)) + lint_ctx.error(f"Select parameter of a conditional [{param_name}] options have to be defined by 'option' children elements.", **node_props(param)) # lint dynamic options if len(options) == 1: @@ -114,10 +115,10 @@ def lint_inputs(tool_xml, lint_ctx): for f in filters: ftype = f.get("type", None) if ftype is None: - lint_ctx.error(f"Select parameter [{param_name}] contains filter without type.", **node_props(f, tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] contains filter without type.", **node_props(f)) continue if ftype not in FILTER_TYPES: - lint_ctx.error(f"Select parameter [{param_name}] contains filter with unknown type '{ftype}'.", **node_props(f, tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] contains filter with unknown type '{ftype}'.", **node_props(f)) continue if ftype in ['add_value', 'data_meta']: filter_adds_options = True @@ -132,26 +133,26 @@ def lint_inputs(tool_xml, lint_ctx): if (from_file is None and from_parameter is None and from_dataset is None and from_data_table is None and not filter_adds_options): - lint_ctx.error(f"Select parameter [{param_name}] options tag defines no options. Use 'from_dataset', 'from_data_table', or a filter that adds values.", **node_props(options[0], tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] options tag defines no options. Use 'from_dataset', 'from_data_table', or a filter that adds values.", **node_props(options[0])) for deprecated_attr in ["from_file", "from_parameter", "options_filter_attribute", "transform_lines"]: if options[0].get(deprecated_attr) is not None: - lint_ctx.warn(f"Select parameter [{param_name}] options uses deprecated '{deprecated_attr}' attribute.", **node_props(options[0], tool_xml)) + lint_ctx.warn(f"Select parameter [{param_name}] options uses deprecated '{deprecated_attr}' attribute.", **node_props(options[0])) if from_dataset is not None and from_data_table is not None: - lint_ctx.error(f"Select parameter [{param_name}] options uses 'from_dataset' and 'from_data_table' attribute.", **node_props(options[0], tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] options uses 'from_dataset' and 'from_data_table' attribute.", **node_props(options[0])) if options[0].get("meta_file_key", None) is not None and from_dataset is None: - lint_ctx.error(f"Select parameter [{param_name}] 'meta_file_key' is only compatible with 'from_dataset'.", **node_props(options[0], tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] 'meta_file_key' is only compatible with 'from_dataset'.", **node_props(options[0])) elif len(options) > 1: - lint_ctx.error(f"Select parameter [{param_name}] contains multiple options elements.", **node_props(options[1], tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] contains multiple options elements.", **node_props(options[1])) # lint statically defined options if any('value' not in option.attrib for option in select_options): - lint_ctx.error(f"Select parameter [{param_name}] has option without value", **node_props(param, tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] has option without value", **node_props(param)) if any(option.text is None for option in select_options): - lint_ctx.warn(f"Select parameter [{param_name}] has option without text", **node_props(param, tool_xml)) + lint_ctx.warn(f"Select parameter [{param_name}] has option without text", **node_props(param)) select_options_texts = list() select_options_values = list() @@ -164,22 +165,22 @@ def lint_inputs(tool_xml, lint_ctx): select_options_texts.append((text, option.attrib.get("selected", "false"))) select_options_values.append((value, option.attrib.get("selected", "false"))) if len(set(select_options_texts)) != len(select_options_texts): - lint_ctx.error(f"Select parameter [{param_name}] has multiple options with the same text content", **node_props(param, tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] has multiple options with the same text content", **node_props(param)) if len(set(select_options_values)) != len(select_options_values): - lint_ctx.error(f"Select parameter [{param_name}] has multiple options with the same value", **node_props(param, tool_xml)) + lint_ctx.error(f"Select parameter [{param_name}] has multiple options with the same value", **node_props(param)) multiple = string_as_bool(param_attrib.get("multiple", "false")) optional = string_as_bool(param_attrib.get("optional", multiple)) if param_attrib.get("display") == "checkboxes": if not multiple: - lint_ctx.error(f'Select [{param_name}] `display="checkboxes"` is incompatible with `multiple="false"`, remove the `display` attribute', **node_props(param, tool_xml)) + lint_ctx.error(f'Select [{param_name}] `display="checkboxes"` is incompatible with `multiple="false"`, remove the `display` attribute', **node_props(param)) if not optional: - lint_ctx.error(f'Select [{param_name}] `display="checkboxes"` is incompatible with `optional="false"`, remove the `display` attribute', **node_props(param, tool_xml)) + lint_ctx.error(f'Select [{param_name}] `display="checkboxes"` is incompatible with `optional="false"`, remove the `display` attribute', **node_props(param)) if param_attrib.get("display") == "radio": if multiple: - lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with multiple="true"', **node_props(param, tool_xml)) + lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with multiple="true"', **node_props(param)) if optional: - lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with optional="true"', **node_props(param, tool_xml)) + lint_ctx.error(f'Select [{param_name}] display="radio" is incompatible with optional="true"', **node_props(param)) # TODO: Validate type, much more... # lint validators @@ -189,45 +190,45 @@ def lint_inputs(tool_xml, lint_ctx): vtype = validator.attrib['type'] if param_type in PARAMETER_VALIDATOR_TYPE_COMPATIBILITY: if vtype not in PARAMETER_VALIDATOR_TYPE_COMPATIBILITY[param_type]: - lint_ctx.error(f"Parameter [{param_name}]: validator with an incompatible type '{vtype}'", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: validator with an incompatible type '{vtype}'", **node_props(validator)) for attrib in ATTRIB_VALIDATOR_COMPATIBILITY: if attrib in validator.attrib and vtype not in ATTRIB_VALIDATOR_COMPATIBILITY[attrib]: - lint_ctx.error(f"Parameter [{param_name}]: attribute '{attrib}' is incompatible with validator of type '{vtype}'", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: attribute '{attrib}' is incompatible with validator of type '{vtype}'", **node_props(validator)) if vtype == "expression": if validator.text is None: - lint_ctx.error(f"Parameter [{param_name}]: expression validators are expected to contain text", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: expression validators are expected to contain text", **node_props(validator)) else: try: re.compile(validator.text) except Exception as e: - lint_ctx.error(f"Parameter [{param_name}]: '{validator.text}' is no valid regular expression: {str(e)}", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: '{validator.text}' is no valid regular expression: {str(e)}", **node_props(validator)) if vtype not in ["expression", "regex"] and validator.text is not None: - lint_ctx.warn(f"Parameter [{param_name}]: '{vtype}' validators are not expected to contain text (found '{validator.text}')", **node_props(validator, tool_xml)) + lint_ctx.warn(f"Parameter [{param_name}]: '{vtype}' validators are not expected to contain text (found '{validator.text}')", **node_props(validator)) if vtype in ["in_range", "length", "dataset_metadata_in_range"] and ("min" not in validator.attrib and "max" not in validator.attrib): - lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'min' or 'max' attribute(s)", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'min' or 'max' attribute(s)", **node_props(validator)) if vtype in ["metadata"] and ("check" not in validator.attrib and "skip" not in validator.attrib): - lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'check' or 'skip' attribute(s)", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'check' or 'skip' attribute(s)", **node_props(validator)) if vtype in ["value_in_data_table", "value_not_in_data_table", "dataset_metadata_in_data_table", "dataset_metadata_not_in_data_table"] and "table_name" not in validator.attrib: - lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'table_name' attribute", **node_props(validator, tool_xml)) + lint_ctx.error(f"Parameter [{param_name}]: '{vtype}' validators need to define the 'table_name' attribute", **node_props(validator)) conditional_selects = tool_xml.findall("./inputs//conditional") for conditional in conditional_selects: conditional_name = conditional.get('name') if not conditional_name: - lint_ctx.error("Conditional without a name", **node_props(conditional, tool_xml)) + lint_ctx.error("Conditional without a name", **node_props(conditional)) if conditional.get("value_from"): # Probably only the upload tool use this, no children elements continue first_param = conditional.findall("param") if len(first_param) != 1: - lint_ctx.error(f"Conditional [{conditional_name}] needs exactly one child found {len(first_param)}", **node_props(conditional, tool_xml)) + lint_ctx.error(f"Conditional [{conditional_name}] needs exactly one child found {len(first_param)}", **node_props(conditional)) continue first_param = first_param[0] first_param_type = first_param.get('type') if first_param_type == 'boolean': - lint_ctx.warn(f'Conditional [{conditional_name}] first param of type="boolean" is discouraged, use a select', **node_props(first_param, tool_xml)) + lint_ctx.warn(f'Conditional [{conditional_name}] first param of type="boolean" is discouraged, use a select', **node_props(first_param)) elif first_param_type != 'select': - lint_ctx.error(f'Conditional [{conditional_name}] first param should have type="select"', **node_props(first_param, tool_xml)) + lint_ctx.error(f'Conditional [{conditional_name}] first param should have type="select"', **node_props(first_param)) continue if first_param_type == 'select': @@ -241,48 +242,49 @@ def lint_inputs(tool_xml, lint_ctx): for incomp in ["optional", "multiple"]: if string_as_bool(first_param.get(incomp, False)): - lint_ctx.warn(f'Conditional [{conditional_name}] test parameter cannot be {incomp}="true"', **node_props(first_param, tool_xml)) + lint_ctx.warn(f'Conditional [{conditional_name}] test parameter cannot be {incomp}="true"', **node_props(first_param)) whens = conditional.findall('./when') if any('value' not in when.attrib for when in whens): - lint_ctx.error(f"Conditional [{conditional_name}] when without value", **node_props(conditional, tool_xml)) + lint_ctx.error(f"Conditional [{conditional_name}] when without value", **node_props(conditional)) when_ids = [w.get('value') for w in whens if w.get('value') is not None] for option_id in option_ids: if option_id not in when_ids: - lint_ctx.warn(f"Conditional [{conditional_name}] no block found for {first_param_type} option '{option_id}'", **node_props(conditional, tool_xml)) + lint_ctx.warn(f"Conditional [{conditional_name}] no block found for {first_param_type} option '{option_id}'", **node_props(conditional)) for when_id in when_ids: if when_id not in option_ids: if first_param_type == 'select': - lint_ctx.warn(f"Conditional [{conditional_name}] no