Merge pull request #14247 from bernt-matthias/topic/add-doc-lint-compare

add doc and linting for test output compare attribute

lib/galaxy/tool_util/linters/tests.py

@@ -10,6 +10,22 @@ from ._util import is_datasource
 from ..verify import asserts
 
 
+def check_compare_attribs(element, lint_ctx, test_idx):
+    COMPARE_COMPATIBILITY = {
+        "sort": ["diff", "re_match", "re_match_multiline"],
+        "lines_diff": ["diff", "re_match", "contains"],
+        "decompress": ["diff"],
+        "delta": ["sim_size"],
+        "delta_frac": ["sim_size"],
+    }
+    compare = element.get("compare", "diff")
+    for attrib in COMPARE_COMPATIBILITY:
+        if attrib in element.attrib and compare not in COMPARE_COMPATIBILITY[attrib]:
+            lint_ctx.error(
+                f'Test {test_idx}: Attribute {attrib} is incompatible with compare="{compare}".', node=element
+            )
+
+
 def lint_tests(tool_xml, lint_ctx):
     # determine node to report for general problems with tests
     tests = tool_xml.findall("./tests/test")
@@ -83,6 +99,13 @@ def lint_tests(tool_xml, lint_ctx):
                 )
                 continue
 
+            if output.tag == "output":
+                check_compare_attribs(output, lint_ctx, test_idx)
+            elements = output.findall("./element")
+            if elements:
+                for element in elements:
+                    check_compare_attribs(element, lint_ctx, test_idx)
+
             # check that
             # - test/output corresponds to outputs/data and
             # - test/collection to outputs/output_collection
@@ -108,7 +131,6 @@ def lint_tests(tool_xml, lint_ctx):
                             node=output,
                         )
                 else:
-                    elements = output.findall("./element")
                     if "count" not in output.attrib and len(elements) == 0:
                         lint_ctx.error(
                             f"Test {test_idx}: test collection '{name}' must have a 'count' attribute or 'element' children",

lib/galaxy/tool_util/verify/__init__.py

@@ -370,6 +370,7 @@ def files_re_match(file1, file2, attributes=None):
     )
     if attributes.get("sort", False):
         history_data.sort()
+        local_file.sort()
     lines_diff = int(attributes.get("lines_diff", 0))
     line_diff_count = 0
     diffs = []

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -1582,6 +1582,29 @@ temporary file, which will either be compared with the file named in the
 ``file`` attribute value or checked against assertions made by a child
 ``assert_contents`` tag to verify that the tool is functionally correct.
 
+Different methods can be chosen for the comparison with the local file specified
+by ``file`` using the ``compare`` attribute:
+
+- ``diff``: uses diff to compare the history data set and the file provided by
+  ``file``. Compressed files are decompressed before the compariopm if
+  ``decompress`` is set to ``true``. BAM files are converted to SAM before the
+  comparision and for pdf some special rules are implemented. The number of
+  allowed differences can be set with ``lines_diff``.  If ``sort="true"`` history
+  and local data is sorted before the comparison.
+- ``re_match``: each line of the history data set is compared to the regular
+  expression specified in the corresponding line of the ``file``. The allowed
+  number of non matching lines can be set with ``lines_diff`` and the history
+  dataset is sorted if ``sort`` is set to ``true``.
+- ``re_match_multiline``: it is checked if the history data sets matches the
+  multi line regular expression given in ``file``. The history dataset is sorted
+  before the comparison if the ``sort`` atrribute is set to ``true``.
+- ``contains``: check if each line in ``file`` is contained in the history data set.
+  The allowed number of lines that are not contained in the history dataset
+  can be set with ``lines_diff``.
+- ``sim_size``: compares the size of the history dataset and the ``file`` subject to
+  the values of the ``delta`` and ``delta_frac`` attributes. Note that a ``has_size``
+  content assertion should be preferred, because this avoids storing the test file.
+
         ]]></xs:documentation>
     </xs:annotation>
     <xs:sequence>
@@ -1631,9 +1654,12 @@ data type. If these do not match, the test will fail.
     </xs:attribute>
     <xs:attribute name="sort" type="PermissiveBoolean">
       <xs:annotation>
-        <xs:documentation xml:lang="en">This flag causes the lines of the output
-to be sorted before they are compared to the expected output. This could be
-useful for non-deterministic output.</xs:documentation>
+        <xs:documentation xml:lang="en"><![CDATA[
+
+Applies only if ``compare`` is ``diff``, ``re_match`` or ``re_match_multiline``. This flag causes the lines of the history data set to be sorted before the comparison. In case of ``diff`` and ``re_match`` also the local file is sorted. This could be
+useful for non-deterministic output.
+
+]]></xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="value" type="xs:string">
@@ -1668,7 +1694,7 @@ it may be inconvenient to upload the entiry file and this can be used instead.
     </xs:attribute>
     <xs:attribute name="lines_diff" type="xs:integer">
       <xs:annotation>
-        <xs:documentation xml:lang="en">If ``compare`` is set to ``diff``, the number of lines of difference to allow (each line with a modification is a line added and a line removed so this counts as two lines).</xs:documentation>
+        <xs:documentation xml:lang="en">Applies only if ``compare`` is set to ``diff``, ``re_match``, and ``contains``. If ``compare`` is set to ``diff``, the number of lines of difference to allow (each line with a modification is a line added and a line removed so this counts as two lines).</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="decompress" type="PermissiveBoolean">

test/unit/tool_util/test_tool_linters.py

@@ -763,6 +763,28 @@ TESTS_EXPECT_NUM_OUTPUTS_FILTER = """
 </tool>
 """
 
+TESTS_COMPARE_ATTRIB_INCOMPATIBILITY = """
+<tool>
+    <outputs>
+        <data name="data_name"/>
+        <collection name="collection_name" type="list:list"/>
+    </outputs>
+    <tests>
+        <test>
+            <output name="data_name" compare="re_match" decompress="true"/>
+            <output_collection name="collection_name">
+                <element compare="contains" sort="true" />
+            </output_collection>
+        </test>
+        <test>
+            <output name="data_name" compare="diff" lines_diff="2"/>
+            <output_collection name="collection_name">
+                <element compare="contains" lines_diff="2" />
+            </output_collection>
+        </test>
+    </tests>
+</tool>"""
+
 # tool xml for xml_order linter
 XML_ORDER = """
 <tool>
@@ -1608,6 +1630,17 @@ def test_tests_expect_num_outputs_filter(lint_ctx):
     assert len(lint_ctx.error_messages) == 0
 
 
+def test_tests_compare_attrib_incompatibility(lint_ctx):
+    tool_source = get_xml_tool_source(TESTS_COMPARE_ATTRIB_INCOMPATIBILITY)
+    run_lint(lint_ctx, tests.lint_tests, tool_source)
+    assert 'Test 1: Attribute decompress is incompatible with compare="re_match".' in lint_ctx.error_messages
+    assert 'Test 1: Attribute sort is incompatible with compare="contains".' in lint_ctx.error_messages
+    assert not lint_ctx.info_messages
+    assert len(lint_ctx.valid_messages) == 1
+    assert not lint_ctx.warn_messages
+    assert len(lint_ctx.error_messages) == 2
+
+
 def test_xml_order(lint_ctx):
     tool_source = get_xml_tool_source(XML_ORDER)
     run_lint(lint_ctx, xml_order.lint_xml_order, tool_source)