implement expect_test_failure attribute for tests

lib/galaxy/tool_util/parser/xml.py

@@ -598,6 +598,7 @@ def _test_elem_to_dict(test_elem, i, profile=None):
         stderr=__parse_assert_list_from_elem(test_elem.find("assert_stderr")),
         expect_exit_code=test_elem.get("expect_exit_code"),
         expect_failure=string_as_bool(test_elem.get("expect_failure", False)),
+        expect_test_failure=string_as_bool(test_elem.get("expect_test_failure", False)),
         maxseconds=test_elem.get("maxseconds", None),
     )
     _copy_to_dict_if_present(test_elem, rval, ["num_outputs"])

lib/galaxy/tool_util/parser/yaml.py

@@ -249,6 +249,7 @@ def _parse_test(i, test_dict):
     test_dict["stderr"] = __to_test_assert_list(test_dict.get("stderr", []))
     test_dict["expect_exit_code"] = test_dict.get("expect_exit_code", None)
     test_dict["expect_failure"] = test_dict.get("expect_failure", False)
+    test_dict["expect_test_failure"] = test_dict.get("expect_test_failure", False)
     return test_dict
 
 

lib/galaxy/tool_util/verify/interactor.py

@@ -1188,9 +1188,9 @@ def _verify_outputs(testdef, history, jobs, data_list, data_collection_list, gal
                     print(_format_stream(job_stdio[stream], stream=stream, format=True), file=sys.stderr)
         found_exceptions.append(e)
 
-    # if testdef.expect_failure:
-    #     if testdef.outputs:
-    #         raise Exception("Cannot specify outputs in a test expecting failure.")
+    if testdef.expect_failure:
+        if testdef.outputs:
+            raise Exception("Cannot specify outputs in a test expecting failure.")
 
     # Wait for the job to complete and register expections if the final
     # status was not what test was expecting.
@@ -1293,7 +1293,7 @@ def _verify_outputs(testdef, history, jobs, data_list, data_collection_list, gal
         except Exception as e:
             register_exception(e)
 
-    if found_exceptions and not testdef.expect_failure:
+    if found_exceptions and not testdef.expect_test_failure:
         raise JobOutputsError(found_exceptions, job_stdio)
     else:
         return job_stdio
@@ -1366,6 +1366,7 @@ class ToolTestDescription:
         self.stderr = processed_test_dict.get("stderr", None)
         self.expect_exit_code = processed_test_dict.get("expect_exit_code", None)
         self.expect_failure = processed_test_dict.get("expect_failure", False)
+        self.expect_test_failure = processed_test_dict.get("expect_test_failure", False)
 
     def test_data(self):
         """
@@ -1392,6 +1393,7 @@ class ToolTestDescription:
             "stderr": self.stderr,
             "expect_exit_code": self.expect_exit_code,
             "expect_failure": self.expect_failure,
+            "expect_test_failure": self.expect_test_failure,
             "name": self.name,
             "test_index": self.test_index,
             "tool_id": self.tool_id,

lib/galaxy/tool_util/xsd/galaxy.xsd

@@ -1106,6 +1106,12 @@ the expectation is for the job fail. If set to ``true`` no job output checks may
 be present in ``test`` definition.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="expect_test_failure" type="PermissiveBoolean" default="false">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Setting this to ``true`` indicates
+that at least one of the assumptions of the test is not met. This is most useful for internal testing.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="maxseconds" type="xs:integer">
       <xs:annotation>
         <xs:documentation xml:lang="en">Maximum amount of time to let test run.</xs:documentation>

lib/galaxy/tools/test.py

@@ -57,6 +57,7 @@ def description_from_tool_object(tool, test_index, raw_test_dict):
             "stderr": raw_test_dict.get("stderr", None),
             "expect_exit_code": raw_test_dict.get("expect_exit_code", None),
             "expect_failure": raw_test_dict.get("expect_failure", False),
+            "expect_test_failure": raw_test_dict.get("expect_test_failure", False),
             "required_files": required_files,
             "required_data_tables": required_data_tables,
             "required_loc_files": required_loc_files,

test/functional/tools/discover_sort_by.xml

@@ -70,7 +70,7 @@ done
       </output>
     </test>
     <!-- same test with wrong order should fail  -->
-    <test expect_num_outputs="4" expect_failure="true">
+    <test expect_num_outputs="4" expect_test_failure="true">
       <param name="input1" value="tinywga.fam" />
       <output_collection name="collection_numeric_name" type="list" count="10">
         <element name="1">