Merge remote-tracking branch 'upstream/release_22.05' into release_23.0

lib/galaxy/tool_util/linters/help.py

@@ -1,4 +1,7 @@
 """This module contains a linting function for a tool's help."""
+
+from typing import Union
+
 from galaxy.util import (
     rst_to_html,
     unicodify,
@@ -30,10 +33,22 @@ def lint_help(tool_xml, lint_ctx):
     if "TODO" in help_text:
         lint_ctx.warn("Help contains TODO text.", node=helps[0])
 
-    try:
-        rst_to_html(help_text, error=True)
-    except Exception as e:
-        lint_ctx.warn(f"Invalid reStructuredText found in help - [{unicodify(e)}].", node=helps[0])
-        return
+    invalid_rst = rst_invalid(help_text)
+    if invalid_rst:
+        lint_ctx.warn(f"Invalid reStructuredText found in help - [{invalid_rst}].", node=helps[0])
+    else:
+        lint_ctx.valid("Help contains valid reStructuredText.", node=helps[0])
 
-    lint_ctx.valid("Help contains valid reStructuredText.", node=helps[0])
+
+def rst_invalid(text: str) -> Union[bool, str]:
+    """
+    Predicate to determine if text is invalid reStructuredText.
+    Return False if the supplied text is valid reStructuredText or
+    a string indicating the problem.
+    """
+    invalid_rst: Union[bool, str] = False
+    try:
+        rst_to_html(text, error=True)
+    except Exception as e:
+        invalid_rst = unicodify(e)
+    return invalid_rst

lib/galaxy/tool_util/linters/inputs.py

@@ -109,7 +109,7 @@ PARAMETER_VALIDATOR_TYPE_COMPATIBILITY = {
 }
 
 PARAM_TYPE_CHILD_COMBINATIONS = [
-    ("./options", ["select", "drill_down"]),
+    ("./options", ["data", "select", "drill_down"]),
     ("./options/option", ["drill_down"]),
     ("./column", ["data_column"]),
 ]
@@ -186,6 +186,37 @@ def lint_inputs(tool_xml, lint_ctx):
                 lint_ctx.warn(
                     f"Param input [{param_name}] with no format specified - 'data' format will be assumed.", node=param
                 )
+            options = param.findall("./options")
+            has_options_filter_attribute = False
+            if len(options) == 1:
+                for oa in options[0].attrib:
+                    if oa == "options_filter_attribute":
+                        has_options_filter_attribute = True
+                    else:
+                        lint_ctx.error(f"Data parameter [{param_name}] uses invalid attribute: {oa}", node=param)
+            elif len(options) > 1:
+                lint_ctx.error(f"Data parameter [{param_name}] contains multiple options elements.", node=options[1])
+            # for data params only filters with key='build' of type='data_meta' are allowed
+            filters = param.findall("./options/filter")
+            for f in filters:
+                if not f.get("ref"):
+                    lint_ctx.error(
+                        f"Data parameter [{param_name}] filter needs to define a ref attribute",
+                        node=f,
+                    )
+                if has_options_filter_attribute:
+                    if f.get("type") != "data_meta":
+                        lint_ctx.error(
+                            f'Data parameter [{param_name}] for filters only type="data_meta" is allowed, found type="{f.get("type")}"',
+                            node=f,
+                        )
+                else:
+                    if f.get("key") != "dbkey" or f.get("type") != "data_meta":
+                        lint_ctx.error(
+                            f'Data parameter [{param_name}] for filters only type="data_meta" and key="dbkey" are allowed, found type="{f.get("type")}" and key="{f.get("key")}"',
+                            node=f,
+                        )
+
         elif param_type == "select":
             # get dynamic/statically defined options
             dynamic_options = param.get("dynamic_options", None)

lib/galaxy/tool_util/loader_directory.py

@@ -190,7 +190,7 @@ def looks_like_a_data_manager_xml(path):
 
 def as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes):
     """
-    get a dict from yaml file if it contains `class: CLASS`, where CLASS is
+    get a dict from yaml file if it contains a line `class: CLASS`, where CLASS is
     any string given in CLASSES. must appear in the first 5k and also load
     properly in total.
     """
@@ -199,7 +199,7 @@ def as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes):
             start_contents = f.read(5 * 1024)
         except UnicodeDecodeError:
             return False, None
-        if re.search(rf"\nclass:\s+({'|'.join(classes)})\s*\n", start_contents) is None:
+        if re.search(rf"^class:\s+{'|'.join(classes)}\s*$", start_contents, re.MULTILINE) is None:
             return False, None
 
     with open(path) as f:

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -3999,14 +3999,17 @@ dataset for the contained input of the type specified using the ``type`` tag.
   <xs:complexType name="ParamOptions">
     <xs:annotation>
       <xs:documentation xml:lang="en"><![CDATA[
-
 See [/tools/extract/liftOver_wrapper.xml](https://github.com/galaxyproject/galaxy/blob/master/tools/extract/liftOver_wrapper.xml)
 for an example of how to use this tag set. This tag set is optionally contained
 within the ``<param>`` tag when the ``type`` attribute value is ``select`` or
-``data`` and used to dynamically generated lists of options. This tag set
-dynamically creates a list of options whose values can be
-obtained from a predefined file stored locally or a dataset selected from the
-current history.
+``data`` and used to dynamically generated lists of options.
+
+For data parameters this tag can be used to restrict possible input datasets to datasets that match the ``dbkey`` of another data input by including a ``data_meta`` filter. See for
+instance here: [/tools/maf/interval2maf.xml](https://github.com/galaxyproject/galaxy/blob/master/tools/maf/interval2maf.xml)
+
+For select parameters this tag set dynamically creates a list of options whose
+values can be obtained from a predefined file stored locally or a dataset
+selected from the current history.
 
 There are at least five basic ways to use this tag - four of these correspond to
 a ``from_XXX`` attribute on the ``options`` directive and the other is to

test/unit/tool_util/test_loader_directory.py

@@ -0,0 +1,24 @@
+import tempfile
+
+from galaxy.tool_util.loader_directory import is_a_yaml_with_class
+
+
+def test_is_a_yaml_with_class():
+    with tempfile.NamedTemporaryFile("w", suffix=".yaml") as tf:
+        fname = tf.name
+        tf.write(
+            """class: GalaxyWorkflow
+name: "Test Workflow"
+inputs:
+  - id: input1
+outputs:
+  - id: wf_output_1
+    outputSource: first_cat/out_file1
+steps:
+  - tool_id: cat
+    label: first_cat
+    in:
+      input1: input1"""
+        )
+        tf.flush()
+        assert is_a_yaml_with_class(fname, ["GalaxyWorkflow"])

test/unit/tool_util/test_tool_linters.py

@@ -192,6 +192,43 @@ INPUTS_DATA_PARAM = """
 </tool>
 """
 
+INPUTS_DATA_PARAM_OPTIONS = """
+<tool>
+    <inputs>
+        <param name="valid_name" type="data" format="txt">
+            <options>
+                <filter type="data_meta" key="dbkey" ref="input"/>
+            </options>
+        </param>
+    </inputs>
+</tool>
+"""
+
+INPUTS_DATA_PARAM_OPTIONS_FILTER_ATTRIBUTE = """
+<tool>
+    <inputs>
+        <param name="valid_name" type="data" format="txt">
+            <options options_filter_attribute="metadata.foo">
+                <filter type="data_meta" key="foo" ref="input"/>
+            </options>
+        </param>
+    </inputs>
+</tool>
+"""
+
+INPUTS_DATA_PARAM_INVALIDOPTIONS = """
+<tool>
+    <inputs>
+        <param name="valid_name" type="data" format="txt">
+            <options/>
+            <options from_file="blah">
+                <filter type="expression"/>
+            </options>
+        </param>
+    </inputs>
+</tool>
+"""
+
 INPUTS_CONDITIONAL = """
 <tool>
     <inputs>
@@ -1099,6 +1136,42 @@ def test_inputs_data_param(lint_ctx):
     assert not lint_ctx.error_messages
 
 
+def test_inputs_data_param_options(lint_ctx):
+    tool_source = get_xml_tool_source(INPUTS_DATA_PARAM_OPTIONS)
+    run_lint(lint_ctx, inputs.lint_inputs, tool_source)
+    assert not lint_ctx.valid_messages
+    assert "Found 1 input parameters." in lint_ctx.info_messages
+    assert len(lint_ctx.info_messages) == 1
+    assert not lint_ctx.warn_messages
+    assert not lint_ctx.error_messages
+
+
+def test_inputs_data_param_options_filter_attribute(lint_ctx):
+    tool_source = get_xml_tool_source(INPUTS_DATA_PARAM_OPTIONS_FILTER_ATTRIBUTE)
+    run_lint(lint_ctx, inputs.lint_inputs, tool_source)
+    assert not lint_ctx.valid_messages
+    assert "Found 1 input parameters." in lint_ctx.info_messages
+    assert len(lint_ctx.info_messages) == 1
+    assert not lint_ctx.warn_messages
+    assert not lint_ctx.error_messages
+
+
+def test_inputs_data_param_invalid_options(lint_ctx):
+    tool_source = get_xml_tool_source(INPUTS_DATA_PARAM_INVALIDOPTIONS)
+    run_lint(lint_ctx, inputs.lint_inputs, tool_source)
+    assert not lint_ctx.valid_messages
+    assert "Found 1 input parameters." in lint_ctx.info_messages
+    assert len(lint_ctx.info_messages) == 1
+    assert not lint_ctx.warn_messages
+    assert "Data parameter [valid_name] contains multiple options elements." in lint_ctx.error_messages
+    assert "Data parameter [valid_name] filter needs to define a ref attribute" in lint_ctx.error_messages
+    assert (
+        'Data parameter [valid_name] for filters only type="data_meta" and key="dbkey" are allowed, found type="expression" and key="None"'
+        in lint_ctx.error_messages
+    )
+    assert len(lint_ctx.error_messages) == 3
+
+
 def test_inputs_conditional(lint_ctx):
     tool_source = get_xml_tool_source(INPUTS_CONDITIONAL)
     run_lint(lint_ctx, inputs.lint_inputs, tool_source)
@@ -1284,7 +1357,7 @@ def test_inputs_type_child_combinations(lint_ctx):
     assert not lint_ctx.valid_messages
     assert not lint_ctx.warn_messages
     assert (
-        "Parameter [text_param] './options' tags are only allowed for parameters of type ['select', 'drill_down']"
+        "Parameter [text_param] './options' tags are only allowed for parameters of type ['data', 'select', 'drill_down']"
         in lint_ctx.error_messages
     )
     assert (