mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-31 02:28:09 +08:00
tool testing add min/max attributes to test output collections
This commit is contained in:
Matthias Bernt
@@ -89,6 +89,8 @@ class ToolSourceTestOutputAttributes(TypedDict):
|
||||
metric: str
|
||||
pin_labels: Optional[Any]
|
||||
count: Optional[int]
|
||||
min: Optional[int]
|
||||
max: Optional[int]
|
||||
metadata: Dict[str, Any]
|
||||
md5: Optional[str]
|
||||
checksum: Optional[str]
|
||||
@@ -865,6 +867,10 @@ class TestCollectionOutputDef:
|
||||
else:
|
||||
count = attrib.get("count")
|
||||
self.count = int(count) if count is not None else None
|
||||
min = attrib.get("min")
|
||||
self.min = int(min) if min is not None else None
|
||||
max = attrib.get("max")
|
||||
self.max = int(max) if max is not None else None
|
||||
self.attrib = attrib
|
||||
self.element_tests = element_tests
|
||||
|
||||
|
||||
@@ -922,6 +922,16 @@ def __parse_test_attributes(
|
||||
count = int(attrib.pop("count"))
|
||||
except KeyError:
|
||||
pass
|
||||
min: Optional[int] = None
|
||||
try:
|
||||
min = int(attrib.pop("min"))
|
||||
except KeyError:
|
||||
pass
|
||||
max: Optional[int] = None
|
||||
try:
|
||||
max = int(attrib.pop("max"))
|
||||
except KeyError:
|
||||
pass
|
||||
extra_files: List[Dict[str, Any]] = []
|
||||
ftype: Optional[str] = None
|
||||
if "ftype" in attrib:
|
||||
@@ -966,6 +976,8 @@ def __parse_test_attributes(
|
||||
pin_labels=pin_labels,
|
||||
location=location,
|
||||
count=count,
|
||||
min=min,
|
||||
max=max,
|
||||
metadata=metadata,
|
||||
md5=md5sum,
|
||||
checksum=checksum,
|
||||
|
||||
@@ -332,6 +332,8 @@ class GalaxyInteractorApi:
|
||||
attributes = output_testdef.attributes
|
||||
name = output_testdef.name
|
||||
expected_count = attributes.get("count")
|
||||
min_count = attributes.get("min")
|
||||
max_count = attributes.get("max")
|
||||
hid = self.__output_id(output_data)
|
||||
# TODO: Twill version verifies dataset is 'ok' in here.
|
||||
try:
|
||||
@@ -357,6 +359,14 @@ class GalaxyInteractorApi:
|
||||
raise AssertionError(
|
||||
f"Output '{name}': expected to have '{expected_count}' datasets, but it had '{found_datasets}'"
|
||||
)
|
||||
if min_count is not None and min_count > found_datasets:
|
||||
raise AssertionError(
|
||||
f"Output '{name}': expected to have at least '{min_count}' datasets, but it had '{found_datasets}'"
|
||||
)
|
||||
if max_count is not None and max_count < found_datasets:
|
||||
raise AssertionError(
|
||||
f"Output '{name}': expected to have at most '{max_count}' datasets, but it had '{found_datasets}'"
|
||||
)
|
||||
for designation, (primary_outfile, primary_attributes) in primary_datasets.items():
|
||||
primary_output = None
|
||||
for output in outputs:
|
||||
@@ -1335,12 +1345,16 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
|
||||
message = f"Output collection '{name}': expected to be of type [{expected_collection_type}], was of type [{collection_type}]."
|
||||
raise AssertionError(message)
|
||||
|
||||
expected_element_count = output_collection_def.count
|
||||
if expected_element_count is not None:
|
||||
actual_element_count = len(data_collection["elements"])
|
||||
if expected_element_count != actual_element_count:
|
||||
message = f"Output collection '{name}': expected to have {expected_element_count} elements, but it had {actual_element_count}."
|
||||
raise AssertionError(message)
|
||||
actual_element_count = len(data_collection["elements"])
|
||||
if output_collection_def.count and output_collection_def.count != actual_element_count:
|
||||
message = f"Output collection '{name}': expected to have {output_collection_def.count} elements, but it had {actual_element_count}."
|
||||
raise AssertionError(message)
|
||||
if output_collection_def.min and output_collection_def.min > actual_element_count:
|
||||
message = f"Output collection '{name}': expected to have at least {output_collection_def.min} elements, but it had {actual_element_count}."
|
||||
raise AssertionError(message)
|
||||
if output_collection_def.max and output_collection_def.max < actual_element_count:
|
||||
message = f"Output collection '{name}': expected to have at most {output_collection_def.max} elements, but it had {actual_element_count}."
|
||||
raise AssertionError(message)
|
||||
|
||||
def get_element(elements, id):
|
||||
for element in elements:
|
||||
@@ -1356,7 +1370,6 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
|
||||
element_outfile, element_attrib = None, element_test
|
||||
else:
|
||||
element_outfile, element_attrib = element_test
|
||||
expected_count = element_attrib.get("count")
|
||||
if "expected_sort_order" in element_attrib:
|
||||
expected_sort_order[element_attrib["expected_sort_order"]] = element_identifier
|
||||
|
||||
@@ -1373,10 +1386,21 @@ def verify_collection(output_collection_def, data_collection, verify_dataset):
|
||||
elements = element["object"]["elements"]
|
||||
element_count = len(elements)
|
||||
verify_elements(elements, element_attrib.get("elements", {}))
|
||||
expected_count = element_attrib.get("count")
|
||||
if expected_count is not None and expected_count != element_count:
|
||||
raise AssertionError(
|
||||
f"Element '{element_identifier}': expected to have {expected_count} elements, but it had {element_count}"
|
||||
)
|
||||
max = element_attrib.get("max")
|
||||
if max is not None and max < element_count:
|
||||
raise AssertionError(
|
||||
f"Element '{element_identifier}': expected to have at most {max} elements, but it had {element_count}"
|
||||
)
|
||||
min = element_attrib.get("min")
|
||||
if min is not None and min > element_count:
|
||||
raise AssertionError(
|
||||
f"Element '{element_identifier}': expected to have at least {min} elements, but it had {element_count}"
|
||||
)
|
||||
|
||||
if len(expected_sort_order) > 0:
|
||||
generated_sort_order = [_["element_identifier"] for _ in element_objects]
|
||||
|
||||
@@ -1996,6 +1996,16 @@ This is available in Galaxy since release 17.05 and was introduced in [pull requ
|
||||
<xs:documentation xml:lang="en">Number or datasets for this output. Should be used for outputs with ``discover_datasets``</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="min" type="xs:integer">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Minimum number or datasets for this output. Should be used for outputs with ``discover_datasets``</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="max" type="xs:integer">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Maximum number or datasets for this output. Should be used for outputs with ``discover_datasets``</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="location" type="xs:anyURI" gxdocs:added="23.1">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">URL that points to a remote output file that will downloaded and used for output comparison.
|
||||
@@ -2348,6 +2358,16 @@ This value is the same as the value of the ``name`` attribute of the
|
||||
<xs:documentation xml:lang="en">Number of elements in output collection.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="min" type="xs:integer">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Minimum number of elements in output collection.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
<xs:attribute name="max" type="xs:integer">
|
||||
<xs:annotation>
|
||||
<xs:documentation xml:lang="en">Maximum number of elements in output collection.</xs:documentation>
|
||||
</xs:annotation>
|
||||
</xs:attribute>
|
||||
</xs:complexType>
|
||||
<xs:complexType name="TestAssertions">
|
||||
<xs:annotation>
|
||||
|
||||
@@ -157,6 +157,42 @@ true
|
||||
</discovered_dataset>
|
||||
</output>
|
||||
</test>
|
||||
<test expect_num_outputs="1">
|
||||
<param name="produce_discovered_dataset_invisible" value="true" />
|
||||
<output name="discovered_dataset_invisible" min="3" max="3">
|
||||
<expand macro="content_assertion"/>
|
||||
<discovered_dataset designation="2" ftype="txt">
|
||||
<expand macro="content_assertion"/>
|
||||
</discovered_dataset>
|
||||
<discovered_dataset designation="3" ftype="txt">
|
||||
<expand macro="content_assertion"/>
|
||||
</discovered_dataset>
|
||||
</output>
|
||||
</test>
|
||||
<test expect_num_outputs="1" expect_test_failure="true">
|
||||
<param name="produce_discovered_dataset_invisible" value="true" />
|
||||
<output name="discovered_dataset_invisible" min="4">
|
||||
<expand macro="content_assertion"/>
|
||||
<discovered_dataset designation="2" ftype="txt">
|
||||
<expand macro="content_assertion"/>
|
||||
</discovered_dataset>
|
||||
<discovered_dataset designation="3" ftype="txt">
|
||||
<expand macro="content_assertion"/>
|
||||
</discovered_dataset>
|
||||
</output>
|
||||
</test>
|
||||
<test expect_num_outputs="1" expect_test_failure="true">
|
||||
<param name="produce_discovered_dataset_invisible" value="true" />
|
||||
<output name="discovered_dataset_invisible" max="2">
|
||||
<expand macro="content_assertion"/>
|
||||
<discovered_dataset designation="2" ftype="txt">
|
||||
<expand macro="content_assertion"/>
|
||||
</discovered_dataset>
|
||||
<discovered_dataset designation="3" ftype="txt">
|
||||
<expand macro="content_assertion"/>
|
||||
</discovered_dataset>
|
||||
</output>
|
||||
</test>
|
||||
|
||||
<!-- discovered datasets invisible -->
|
||||
<test expect_num_outputs="1">
|
||||
@@ -322,5 +358,120 @@ true
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
|
||||
<!-- successful test of min/max of output_collection and elements -->
|
||||
<test expect_num_outputs="1">
|
||||
<param name="produce_paired_list" value="true" />
|
||||
<output_collection name="paired_list" type="list:paired" min="2" max="2">
|
||||
<element name="p1" min="2" max="2">
|
||||
<element name="forward" min="1" max="1">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse" min="1" max="1">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
<element name="p2" min="2" max="2">
|
||||
<element name="forward" min="1" max="1">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse" min="1" max="1">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
|
||||
<!-- unsuccessful test of min of output_collection -->
|
||||
<test expect_num_outputs="1" expect_test_failure="true">
|
||||
<param name="produce_paired_list" value="true" />
|
||||
<output_collection name="paired_list" type="list:paired" min="3">
|
||||
<element name="p1">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
<element name="p2">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
<!-- unsuccessful test of max of output_collection -->
|
||||
<test expect_num_outputs="1" expect_test_failure="true">
|
||||
<param name="produce_paired_list" value="true" />
|
||||
<output_collection name="paired_list" type="list:paired" max="1">
|
||||
<!-- TODO allow testing without all this -->
|
||||
<element name="p1">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
<element name="p2">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
|
||||
<!-- unsuccessful test of min of element -->
|
||||
<test expect_num_outputs="1" expect_test_failure="true">
|
||||
<param name="produce_paired_list" value="true" />
|
||||
<output_collection name="paired_list" type="list:paired" min="2" max="2">
|
||||
<element name="p1" min="3">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
<element name="p2" min="3">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
<!-- unsuccessful test of min of element -->
|
||||
<test expect_num_outputs="1" expect_test_failure="true">
|
||||
<param name="produce_paired_list" value="true" />
|
||||
<output_collection name="paired_list" type="list:paired" min="2" max="2">
|
||||
<element name="p1" max="1">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
<element name="p2" max="1">
|
||||
<element name="forward">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
<element name="reverse">
|
||||
<expand macro="content_assertion"/>
|
||||
</element>
|
||||
</element>
|
||||
</output_collection>
|
||||
</test>
|
||||
|
||||
</tests>
|
||||
</tool>
|
||||
|
||||
Reference in New Issue
Block a user