opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

full test coverage for tests linter

Browse Source
This commit is contained in:
Matthias Bernt
2022-01-17 11:07:48 +01:00
parent a73de59c9d
commit 16070ccbb5
2 changed files with 60 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_util/linters/tests.py
+18 -20
View File
@@ -18,10 +18,12 @@ def lint_tsts(tool_xml, lint_ctx):
pass
tests = tool_xml.findall("./tests/test")
datasource = is_datasource(tool_xml)
if not tests and not datasource:
lint_ctx.warn("No tests found, most tools should define test cases.", line=tests_line, xpath=tests_path)
elif datasource:
lint_ctx.info("No tests found, that should be OK for data_sources.", line=tests_line, xpath=tests_path)
if not tests:
if not datasource:
lint_ctx.warn("No tests found, most tools should define test cases.", line=tests_line, xpath=tests_path)
elif datasource:
lint_ctx.info("No tests found, that should be OK for data_sources.", line=tests_line, xpath=tests_path)
return
num_valid_tests = 0
for test_idx, test in enumerate(tests, start=1):
@@ -63,23 +65,22 @@ def lint_tsts(tool_xml, lint_ctx):
output_data_names, output_collection_names = _collect_output_names(tool_xml)
found_output_test = False
for output in test.findall("output"):
for output in test.findall("output") + test.findall("output_collection"):
found_output_test = True
name = output.attrib.get("name", None)
if not name:
lint_ctx.warn(f"Test {test_idx}: Found output tag without a name defined.", line=output.sourceline, xpath=tool_xml.getpath(output))
if output.tag == "output":
valid_names = output_data_names
else:
if name not in output_data_names:
lint_ctx.error(f"Test {test_idx}: Found output tag with unknown name [{name}], valid names [{output_data_names}]", line=output.sourceline, xpath=tool_xml.getpath(output))
valid_names = output_collection_names
if not name:
lint_ctx.error(f"Test {test_idx}: Found {output.tag} tag without a name defined.", line=output.sourceline, xpath=tool_xml.getpath(output))
else:
if name not in valid_names:
lint_ctx.error(f"Test {test_idx}: Found {output.tag} tag with unknown name [{name}], valid names [{valid_names}]", line=output.sourceline, xpath=tool_xml.getpath(output))
for output_collection in test.findall("output_collection"):
found_output_test = True
name = output_collection.attrib.get("name", None)
if not name:
lint_ctx.warn(f"Test {test_idx}: Found output_collection tag without a name defined.", line=output_collection.sourceline, xpath=tool_xml.getpath(output_collection))
else:
if name not in output_collection_names:
lint_ctx.warn(f"Test {test_idx}: Found output_collection tag with unknown name [{name}], valid names [{output_collection_names}]", line=output_collection.sourceline, xpath=tool_xml.getpath(output_collection))
if "expect_failure" in test.attrib and found_output_test:
lint_ctx.error(f"Test {test_idx}: Cannot specify outputs in a test expecting failure.")
continue
has_test = has_test or found_output_test
if not has_test:
@@ -87,9 +88,6 @@ def lint_tsts(tool_xml, lint_ctx):
else:
num_valid_tests += 1
if "expect_failure" in test.attrib and found_output_test:
lint_ctx.error(f"Test {test_idx}: Cannot specify outputs in a test expecting failure.")
if num_valid_tests or datasource:
lint_ctx.valid(f"{num_valid_tests} test(s) found.", line=tests_line, xpath=tests_path)
else:
test/unit/tool_util/test_tool_linters.py
+42 -10
View File
@@ -491,6 +491,12 @@ STDIO_INVALID_CHILD_OR_ATTRIB = """
"""
# check that linter does complain about tests wo assumptions
TESTS_ABSENT = """
<tool/>
"""
TESTS_ABSENT_DATA_SOURCE = """
<tool tool_type="data_source"/>
"""
TESTS_WO_EXPECTATIONS = """
<tool>
<tests>
@@ -500,7 +506,7 @@ TESTS_WO_EXPECTATIONS = """
</tool>
"""
TESTS_PARAM = """
TESTS_PARAM_OUTPUT_NAMES = """
<tool>
<inputs>
<param argument="--existent-test-name"/>
@@ -512,14 +518,20 @@ TESTS_PARAM = """
</inputs>
<outputs>
<data name="existent_output"/>
<collection name="existent_collection"/>
</outputs>
<tests>
<test expect_num_outputs="1">
<param/>
<param name="existent_test_name"/>
<param name="cond_name|another_existent_test_name"/>
<param name="non_existent_test_name"/>
<output/>
<output name="existent_output"/>
<output name="nonexistent_output"/>
<output_collection/>
<output_collection name="existent_collection"/>
<output_collection name="nonexistent_collection"/>
</test>
</tests>
</tool>
@@ -890,25 +902,43 @@ TESTS = [
and len(x.info_messages) == 0 and len(x.warn_messages) == 3 and len(x.error_messages) == 0
),
(
TESTS_ABSENT, tests.lint_tsts,
lambda x:
'No tests found, most tools should define test cases.' in x.warn_messages
and len(x.info_messages) == 0 and len(x.valid_messages) == 0 and len(x.warn_messages) == 1 and len(x.error_messages) == 0
),
(
TESTS_ABSENT_DATA_SOURCE, tests.lint_tsts,
lambda x:
'No tests found, that should be OK for data_sources.' in x.info_messages
and len(x.info_messages) == 1 and len(x.valid_messages) == 0 and len(x.warn_messages) == 0 and len(x.error_messages) == 0
),
(
TESTS_WO_EXPECTATIONS, tests.lint_tsts,
lambda x:
'Test 1: No outputs or expectations defined for tests, this test is likely invalid.' in x.warn_messages
and 'No valid test(s) found.' in x.warn_messages
and len(x.warn_messages) == 2 and len(x.error_messages) == 0
and len(x.info_messages) == 0 and len(x.valid_messages) == 0 and len(x.warn_messages) == 2 and len(x.error_messages) == 0
),
(
TESTS_PARAM, tests.lint_tsts,
TESTS_PARAM_OUTPUT_NAMES, tests.lint_tsts,
lambda x:
"Test 1: Test param non_existent_test_name not found in the inputs" in x.error_messages
'1 test(s) found.' in x.valid_messages
and "Test 1: Found test param tag without a name defined." in x.error_messages
and "Test 1: Test param non_existent_test_name not found in the inputs" in x.error_messages
and "Test 1: Found output tag without a name defined." in x.error_messages
and "Test 1: Found output tag with unknown name [nonexistent_output], valid names [['existent_output']]" in x.error_messages
and len(x.warn_messages) == 0 and len(x.error_messages) == 2
and "Test 1: Found output_collection tag without a name defined." in x.error_messages
and "Test 1: Found output_collection tag with unknown name [nonexistent_collection], valid names [['existent_collection']]" in x.error_messages
and len(x.info_messages) == 0 and len(x.valid_messages) == 1 and len(x.warn_messages) == 0 and len(x.error_messages) == 6
),
(
TESTS_EXPECT_FAILURE_OUTPUT, tests.lint_tsts,
lambda x:
"Test 1: Cannot specify outputs in a test expecting failure." in x.error_messages
and len(x.warn_messages) == 0 and len(x.error_messages) == 1
'No valid test(s) found.' in x.warn_messages
and "Test 1: Cannot specify outputs in a test expecting failure." in x.error_messages
and len(x.info_messages) == 0 and len(x.valid_messages) == 0 and len(x.warn_messages) == 1 and len(x.error_messages) == 1
),
(
XML_ORDER, xml_order.lint_xml_order,
@@ -964,9 +994,11 @@ TEST_IDS = [
'stdio: default for non-legacy profile',
'stdio: multiple stdio',
'stdio: invalid tag or attribute',
'test without expectations',
'test param missing from inputs',
'test expecting failure with outputs',
'tests: absent',
'tests: absent data_source',
'tests: without expectations',
'tests: param and output names',
'tests: expecting failure with outputs',
'xml_order'
]