mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Merge pull request #17874 from bernt-matthias/assert-empty-output_24.0
[24.0] Missing outputs should be recorded as test errors
This commit is contained in:
Marius van den Beek
@@ -1430,8 +1430,6 @@ def verify_tool(
|
||||
raise e
|
||||
|
||||
if not expected_failure_occurred:
|
||||
assert data_list or data_collection_list
|
||||
|
||||
try:
|
||||
job_stdio = _verify_outputs(
|
||||
testdef, test_history, jobs, data_list, data_collection_list, galaxy_interactor, quiet=quiet
|
||||
@@ -1489,7 +1487,6 @@ def _handle_def_errors(testdef):
|
||||
def _verify_outputs(testdef, history, jobs, data_list, data_collection_list, galaxy_interactor, quiet=False):
|
||||
assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
|
||||
job = jobs[0]
|
||||
|
||||
found_exceptions: List[Exception] = []
|
||||
|
||||
def register_exception(e: Exception):
|
||||
@@ -1542,29 +1539,36 @@ def _verify_outputs(testdef, history, jobs, data_list, data_collection_list, gal
|
||||
outfile = output_dict["value"]
|
||||
attributes = output_dict["attributes"]
|
||||
output_testdef = Bunch(name=name, outfile=outfile, attributes=attributes)
|
||||
output_data = None
|
||||
try:
|
||||
output_data = data_list[name]
|
||||
except (TypeError, KeyError):
|
||||
# Legacy - fall back on ordered data list access if data_list is
|
||||
# just a list (case with twill variant or if output changes its
|
||||
# name).
|
||||
if hasattr(data_list, "values"):
|
||||
output_data = list(data_list.values())[output_index]
|
||||
else:
|
||||
output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
|
||||
assert output_data is not None
|
||||
try:
|
||||
galaxy_interactor.verify_output(
|
||||
history,
|
||||
jobs,
|
||||
output_data,
|
||||
output_testdef=output_testdef,
|
||||
tool_id=job["tool_id"],
|
||||
maxseconds=maxseconds,
|
||||
tool_version=testdef.tool_version,
|
||||
)
|
||||
except Exception as e:
|
||||
register_exception(e)
|
||||
try:
|
||||
if hasattr(data_list, "values"):
|
||||
output_data = list(data_list.values())[output_index]
|
||||
else:
|
||||
output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
|
||||
except IndexError:
|
||||
error = AssertionError(
|
||||
f"Tool did not produce an output with name '{name}' (or at index {output_index})"
|
||||
)
|
||||
register_exception(error)
|
||||
if output_data:
|
||||
try:
|
||||
galaxy_interactor.verify_output(
|
||||
history,
|
||||
jobs,
|
||||
output_data,
|
||||
output_testdef=output_testdef,
|
||||
tool_id=job["tool_id"],
|
||||
maxseconds=maxseconds,
|
||||
tool_version=testdef.tool_version,
|
||||
)
|
||||
except Exception as e:
|
||||
register_exception(e)
|
||||
|
||||
other_checks = {
|
||||
"command_line": "Command produced by the job",
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
<tool id="output_filter" name="output_filter" version="1.0.0">
|
||||
<description>test for output filtering and expect_num_outputs</description>
|
||||
<command><![CDATA[
|
||||
echo 'test' > 1 &&
|
||||
echo 'test' > 2 &&
|
||||
echo 'test' > 3 &&
|
||||
echo 'test' > 4 &&
|
||||
echo 'test' > 5 &&
|
||||
echo 'test' > p1.forward &&
|
||||
echo 'test' > p1.reverse &&
|
||||
echo 'test' > p2.forward &&
|
||||
echo 'test' > p2.reverse
|
||||
echo '1' > 1 &&
|
||||
echo '2' > 2 &&
|
||||
echo '3' > 3 &&
|
||||
echo '4' > 4 &&
|
||||
echo '5' > 5 &&
|
||||
echo 'p1.forward' > p1.forward &&
|
||||
echo 'p1.reverse' > p1.reverse &&
|
||||
echo 'p2.forward' > p2.forward &&
|
||||
echo 'p2.reverse' > p2.reverse
|
||||
]]></command>
|
||||
<inputs>
|
||||
<param name="produce_out_1" type="boolean" truevalue="true" falsevalue="false" checked="False" label="Do Filter 1" />
|
||||
@@ -26,7 +26,6 @@ echo 'test' > p2.reverse
|
||||
<!-- Must pass all filters... -->
|
||||
<filter>filter_text_1 == "foo"</filter>
|
||||
</data>
|
||||
<data name="out_3" format="txt" from_work_dir="3"/>
|
||||
<collection name="list_output" type="list" label="List">
|
||||
<discover_datasets pattern="(?P<identifier_0>[45])" ext="txt" visible="true" />
|
||||
<filter>produce_collection is True</filter>
|
||||
@@ -37,40 +36,31 @@ echo 'test' > p2.reverse
|
||||
</collection>
|
||||
</outputs>
|
||||
<tests>
|
||||
<test expect_num_outputs="3">
|
||||
<test expect_num_outputs="2">
|
||||
<param name="produce_out_1" value="true" />
|
||||
<param name="filter_text_1" value="foo" />
|
||||
<output name="out_1">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="1" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_2">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_3">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<test expect_num_outputs="2">
|
||||
<param name="produce_out_1" value="true" />
|
||||
<param name="filter_text_1" value="bar" /> <!-- fails second filter in out2 -->
|
||||
<output name="out_1">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_3">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="2" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<test expect_num_outputs="1">
|
||||
<param name="produce_out_1" value="true" />
|
||||
<param name="filter_text_1" value="bar" /> <!-- fails second filter in out2 -->
|
||||
<output name="out_1">
|
||||
<assert_contents>
|
||||
<has_line line="1" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
</test>
|
||||
<!-- tool runs with no outputs should fail -->
|
||||
<test expect_num_outputs="0" expect_test_failure="true">
|
||||
<param name="produce_out_1" value="false" />
|
||||
<param name="filter_text_1" value="not_foo_or_bar" />
|
||||
<output name="out_3">
|
||||
@@ -78,68 +68,61 @@ echo 'test' > p2.reverse
|
||||
<has_line line="test" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<assert_stdout>
|
||||
<has_n_lines n="0"/>
|
||||
</assert_stdout>
|
||||
</test>
|
||||
<test expect_num_outputs="4">
|
||||
<test expect_num_outputs="3">
|
||||
<param name="produce_out_1" value="true" />
|
||||
<param name="filter_text_1" value="foo" />
|
||||
<param name="produce_collection" value="true" />
|
||||
<output name="out_1">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="1" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_2">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_3">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="2" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output_collection name="list_output" type="list" count="2">
|
||||
<element name="4">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="4" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
<element name="5">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="5" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
<test expect_num_outputs="5">
|
||||
<test expect_num_outputs="4">
|
||||
<param name="produce_out_1" value="true" />
|
||||
<param name="filter_text_1" value="foo" />
|
||||
<param name="produce_collection" value="true" />
|
||||
<param name="produce_paired_collection" value="true" />
|
||||
<output name="out_1">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="1" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_2">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output name="out_3">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="2" />
|
||||
</assert_contents>
|
||||
</output>
|
||||
<output_collection name="list_output" type="list" count="2">
|
||||
<element name="4">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="4" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
<element name="5">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="5" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
</output_collection>
|
||||
@@ -147,24 +130,24 @@ echo 'test' > p2.reverse
|
||||
<element name="p1">
|
||||
<element name="forward">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="p1.forward" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="p1.reverse" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
</element>
|
||||
<element name="p2">
|
||||
<element name="forward">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="p2.forward" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<assert_contents>
|
||||
<has_line line="test" />
|
||||
<has_line line="p2.reverse" />
|
||||
</assert_contents>
|
||||
</element>
|
||||
</element>
|
||||
|
||||
Reference in New Issue
Block a user