opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 13:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
85dcd60d4cb9073ea86a54360780d87441206641
galaxy/scripts/cwl_conformance_to_test_cases.py
T
Dannon Baker 623a5e165a Apply black formatting. 
Apply isort.
2022-02-03 07:42:20 -05:00

424 lines
16 KiB
Python
Raw Blame History

import os
import string
import sys
import yaml
THIS_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
GALAXY_ROOT_DIR = os.path.abspath(os.path.join(THIS_DIRECTORY, os.pardir))
CWL_API_TESTS_DIRECTORY = os.path.join(GALAXY_ROOT_DIR, "lib", "galaxy_test", "api", "cwl")
TEST_FILE_TEMPLATE = string.Template(
'''"""Test CWL conformance for version ${version}."""
import pytest
from ..test_workflows_cwl import BaseCwlWorkflowTestCase
class CwlConformanceTestCase(BaseCwlWorkflowTestCase):
"""Test case mapping to CWL conformance tests for version ${version}."""
$tests'''
)
TEST_TEMPLATE = string.Template(
'''
${marks} def test_conformance_${version_simple}_${label}(self):
"""${doc}
Generated from::
${cwl_test_def}
""" # noqa: W293
self.cwl_populator.run_conformance_test("""${version}""", """${doc}""")
'''
)
RED_TESTS = {
"v1.0": [
# required
"directory_literal_with_literal_file_nostdin",
"format_checking",
"format_checking_equivalentclass",
"format_checking_subclass",
"stdin_from_directory_literal_with_literal_file",
"stdin_from_directory_literal_with_local_file",
"step_input_default_value_overriden_2nd_step_noexp",
"step_input_default_value_overriden_noexp",
# not required
"clt_any_input_with_mixed_array_provided",
"clt_any_input_with_record_provided",
"clt_optional_union_input_file_or_files_with_array_of_one_file_provided",
"clt_optional_union_input_file_or_files_with_many_files_provided",
"directory_secondaryfiles",
"docker_entrypoint",
"dockeroutputdir",
"dynamic_initial_workdir",
"dynamic_resreq_filesizes",
"dynamic_resreq_wf_optional_file_default",
"dynamic_resreq_wf_optional_file_wf_default",
"env_home_tmpdir",
"env_home_tmpdir_docker",
"env_home_tmpdir_docker_complex",
"filesarray_secondaryfiles",
"initialworkdir_nesteddir",
"input_dir_recurs_copy_writable",
"job_input_secondary_subdirs",
"job_input_subdir_primary_and_secondary_subdirs",
"record_output_binding",
"resreq_step_overrides_wf",
"step_input_default_value_overriden",
"step_input_default_value_overriden_2nd_step",
"valuefrom_wf_step",
"valuefrom_wf_step_multiple",
"wf_multiplesources_multipletypes",
"wf_multiplesources_multipletypes_noexp",
"wf_scatter_flat_crossproduct_oneempty",
"wf_scatter_nested_crossproduct_firstempty",
"wf_scatter_nested_crossproduct_secondempty",
"wf_scatter_oneparam_valuefrom",
"wf_scatter_oneparam_valuefrom_inputs",
"wf_scatter_two_flat_crossproduct",
"wf_scatter_two_nested_crossproduct",
"wf_scatter_twoparam_dotproduct_valuefrom",
"wf_scatter_twoparam_flat_crossproduct_valuefrom",
"wf_scatter_twoparam_nested_crossproduct_valuefrom",
"workflow_any_input_with_record_provided",
"workflow_embedded_subworkflow_embedded_subsubworkflow",
"workflow_embedded_subworkflow_with_subsubworkflow_and_tool",
"workflow_embedded_subworkflow_with_tool_and_subsubworkflow",
"workflow_file_array_output",
"workflow_file_input_default_unspecified",
"workflow_integer_input_optional_unspecified",
"workflow_records_inputs_and_outputs",
],
"v1.1": [
# required
"any_input_param_graph_no_default",
"any_input_param_graph_no_default_hashmain",
"directory_literal_with_literal_file_nostdin",
"fail_glob_outside_output_dir",
"format_checking",
"format_checking_equivalentclass",
"format_checking_subclass",
"input_records_file_entry_with_format",
"outputEval_exitCode",
"outputbinding_glob_directory",
"secondary_files_in_output_records",
"secondary_files_in_unnamed_records",
"secondary_files_missing",
"stage_file_array_to_dir",
"stage_file_array_to_dir_basename",
"stage_file_array_to_dir_basename_entryname",
"stdin_from_directory_literal_with_literal_file",
"stdin_from_directory_literal_with_local_file",
"step_input_default_value_overriden_2nd_step_noexp",
"step_input_default_value_overriden_noexp",
# not required
"clt_any_input_with_mixed_array_provided",
"clt_any_input_with_record_provided",
"clt_optional_union_input_file_or_files_with_array_of_one_file_provided",
"clt_optional_union_input_file_or_files_with_many_files_provided",
"cwl_requirements_addition",
"cwl_requirements_override_expression",
"cwl_requirements_override_static",
"directory_secondaryfiles",
"docker_entrypoint",
"dockeroutputdir",
"dynamic_resreq_filesizes",
"dynamic_resreq_wf_optional_file_default",
"dynamic_resreq_wf_optional_file_wf_default",
"embedded_subworkflow",
"env_home_tmpdir",
"env_home_tmpdir_docker",
"env_home_tmpdir_docker_no_return_code",
"filesarray_secondaryfiles",
"filesarray_secondaryfiles2",
"initialworkdir_nesteddir",
"inplace_update_on_dir_content",
"inplace_update_on_file_content",
"input_dir_recurs_copy_writable",
"job_input_secondary_subdirs",
"job_input_subdir_primary_and_secondary_subdirs",
"listing_requirement_deep",
"listing_requirement_none",
"listing_requirement_shallow",
"networkaccess",
"networkaccess_disabled",
"record_output_binding",
"record_output_file_entry_format",
"resreq_step_overrides_wf",
"scatter_embedded_subworkflow",
"scatter_multi_input_embedded_subworkflow",
"secondary_files_in_named_records",
"stage_array_dirs",
"stage_null_array",
"stdin_shorcut",
"step_input_default_value_overriden",
"step_input_default_value_overriden_2nd_step",
"symlink_to_file_out_of_workdir_illegal",
"timelimit_expressiontool",
"timelimit_invalid_wf",
"timelimit_zero_unlimited",
"timelimit_zero_unlimited_wf",
"tmpdir_is_not_outdir",
"valuefrom_wf_step",
"valuefrom_wf_step_multiple",
"wf_multiplesources_multipletypes",
"wf_multiplesources_multipletypes_noexp",
"wf_scatter_flat_crossproduct_oneempty",
"wf_scatter_nested_crossproduct_firstempty",
"wf_scatter_nested_crossproduct_secondempty",
"wf_scatter_oneparam_valuefrom",
"wf_scatter_oneparam_valuefrom_inputs",
"wf_scatter_two_flat_crossproduct",
"wf_scatter_two_nested_crossproduct",
"wf_scatter_twoparam_dotproduct_valuefrom",
"wf_scatter_twoparam_flat_crossproduct_valuefrom",
"wf_scatter_twoparam_nested_crossproduct_valuefrom",
"workflow_any_input_with_record_provided",
"workflow_embedded_subworkflow_embedded_subsubworkflow",
"workflow_embedded_subworkflow_with_subsubworkflow_and_tool",
"workflow_embedded_subworkflow_with_tool_and_subsubworkflow",
"workflow_file_array_output",
"workflow_file_input_default_unspecified",
"workflow_input_inputBinding_loadContents",
"workflow_input_loadContents_without_inputBinding",
"workflow_integer_input_optional_unspecified",
"workflow_records_inputs_and_outputs",
"workflow_step_in_loadContents",
],
"v1.2": [
# required
"any_input_param_graph_no_default",
"any_input_param_graph_no_default_hashmain",
"directory_literal_with_literal_file_nostdin",
"format_checking",
"format_checking_equivalentclass",
"format_checking_subclass",
"glob_outside_outputs_fails",
"input_records_file_entry_with_format",
"outputEval_exitCode",
"outputbinding_glob_directory",
"secondary_files_in_output_records",
"secondary_files_in_unnamed_records",
"secondary_files_missing",
"stage_file_array",
"stage_file_array_basename",
"stage_file_array_entryname_overrides",
"stdin_from_directory_literal_with_literal_file",
"stdin_from_directory_literal_with_local_file",
"step_input_default_value_overriden_2nd_step_noexp",
"step_input_default_value_overriden_noexp",
# not required
"225",
"226",
"233",
"235",
"236",
"306",
"307",
"all_non_null_all_null",
"all_non_null_all_null_nojs",
"all_non_null_multi_non_null",
"all_non_null_multi_non_null_nojs",
"all_non_null_one_non_null",
"all_non_null_one_non_null_nojs",
"clt_any_input_with_mixed_array_provided",
"clt_any_input_with_record_provided",
"clt_any_input_with_string_provided",
"clt_optional_union_input_file_or_files_with_array_of_one_file_provided",
"clt_optional_union_input_file_or_files_with_many_files_provided",
"command_input_file_expression",
"condifional_scatter_on_nonscattered_false",
"condifional_scatter_on_nonscattered_false_nojs",
"condifional_scatter_on_nonscattered_true_nojs",
"conditionals_multi_scatter",
"conditionals_multi_scatter_nojs",
"conditionals_nested_cross_scatter",
"conditionals_nested_cross_scatter_nojs",
"conditionals_non_boolean_fail",
"conditionals_non_boolean_fail_nojs",
"cwl_requirements_addition",
"cwl_requirements_override_expression",
"cwl_requirements_override_static",
"cwloutput_nolimit",
"direct_optional_nonnull_result_nojs",
"direct_optional_null_result",
"direct_required",
"direct_required_nojs",
"directory_secondaryfiles",
"docker_entrypoint",
"dockeroutputdir",
"dynamic_resreq_filesizes",
"dynamic_resreq_wf_optional_file_default",
"dynamic_resreq_wf_optional_file_wf_default",
"embedded_subworkflow",
"env_home_tmpdir",
"env_home_tmpdir_docker",
"env_home_tmpdir_docker_no_return_code",
"filesarray_secondaryfiles",
"filesarray_secondaryfiles2",
"first_non_null_first_non_null",
"first_non_null_first_non_null_nojs",
"first_non_null_second_non_null",
"first_non_null_second_non_null_nojs",
"initialworkdir_nesteddir",
"input_dir_recurs_copy_writable",
"iwd-container-entryname1",
"iwd-fileobjs1",
"iwd-fileobjs2",
"iwd-passthrough1",
"iwd-passthrough3",
"iwd-passthrough4",
"job_input_secondary_subdirs",
"job_input_subdir_primary_and_secondary_subdirs",
"listing_requirement_deep",
"listing_requirement_none",
"listing_requirement_shallow",
"networkaccess",
"networkaccess_disabled",
"pass_through_required_false_when",
"pass_through_required_false_when_nojs",
"pass_through_required_the_only_non_null",
"pass_through_required_the_only_non_null_nojs",
"pass_through_required_true_when",
"pass_through_required_true_when_nojs",
"record_output_binding",
"record_output_file_entry_format",
"resreq_step_overrides_wf",
"scatter_embedded_subworkflow",
"scatter_multi_input_embedded_subworkflow",
"scatter_on_scattered_conditional",
"scatter_on_scattered_conditional_nojs",
"secondary_files_in_named_records",
"step_input_default_value_overriden",
"step_input_default_value_overriden_2nd_step",
"storage_float",
"the_only_non_null_single_true",
"the_only_non_null_single_true_nojs",
"timelimit_expressiontool",
"timelimit_invalid_wf",
"timelimit_zero_unlimited",
"timelimit_zero_unlimited_wf",
"tmpdir_is_not_outdir",
"valuefrom_wf_step",
"valuefrom_wf_step_multiple",
"wf_multiplesources_multipletypes",
"wf_multiplesources_multipletypes_noexp",
"wf_scatter_flat_crossproduct_oneempty",
"wf_scatter_nested_crossproduct_firstempty",
"wf_scatter_nested_crossproduct_secondempty",
"wf_scatter_oneparam_valuefrom",
"wf_scatter_oneparam_valuefrom_inputs",
"wf_scatter_two_flat_crossproduct",
"wf_scatter_two_nested_crossproduct",
"wf_scatter_twoparam_dotproduct_valuefrom",
"wf_scatter_twoparam_flat_crossproduct_valuefrom",
"wf_scatter_twoparam_nested_crossproduct_valuefrom",
"wf_wc_nomultiple_merge_nested",
"workflow_any_input_with_record_provided",
"workflow_embedded_subworkflow_embedded_subsubworkflow",
"workflow_embedded_subworkflow_with_subsubworkflow_and_tool",
"workflow_embedded_subworkflow_with_tool_and_subsubworkflow",
"workflow_file_array_output",
"workflow_file_input_default_unspecified",
"workflow_input_inputBinding_loadContents",
"workflow_input_loadContents_without_inputBinding",
"workflow_integer_input_optional_unspecified",
"workflow_records_inputs_and_outputs",
"workflow_step_in_loadContents",
],
}
def conformance_tests_gen(directory, filename="conformance_tests.yaml"):
conformance_tests_path = os.path.join(directory, filename)
with open(conformance_tests_path) as f:
conformance_tests = yaml.safe_load(f)
for conformance_test in conformance_tests:
if "$import" in conformance_test:
import_path = conformance_test["$import"]
yield from conformance_tests_gen(directory, import_path)
else:
yield conformance_test
def main():
if len(sys.argv) != 3:
raise Exception("Expecting 2 arguments: conformance_tests_dir version")
conformance_tests_dir = sys.argv[1]
version = sys.argv[2]
version_simple = version.replace(".", "_")
red_tests_list = RED_TESTS[version]
red_tests_found = set()
all_tests_found = set()
tests = ""
for i, conformance_test in enumerate(conformance_tests_gen(os.path.join(conformance_tests_dir, version))):
test_with_doc = conformance_test.copy()
if "doc" not in test_with_doc:
raise Exception(f"No doc in test [{test_with_doc}]")
del test_with_doc["doc"]
cwl_test_def = yaml.dump(test_with_doc, default_flow_style=False)
cwl_test_def = "\n".join(f" {line}" for line in cwl_test_def.splitlines())
label = conformance_test.get("label", str(i))
tags = conformance_test.get("tags", [])
is_red = label in red_tests_list
marks = " @pytest.mark.cwl_conformance\n"
marks += f" @pytest.mark.cwl_conformance_{version_simple}\n"
for tag in tags:
marks += f" @pytest.mark.{tag}\n"
if is_red:
marks += " @pytest.mark.red\n"
else:
marks += " @pytest.mark.green\n"
if not {"command_line_tool", "expression_tool", "workflow"}.intersection(tags):
print(
f"PROBLEM - test [{label}] tagged with neither command_line_tool, expression_tool, nor workflow",
file=sys.stderr,
)
template_kwargs = {
"version_simple": version_simple,
"version": version,
"doc": conformance_test["doc"],
"cwl_test_def": cwl_test_def,
"label": label.replace("-", "_"),
"marks": marks,
}
test_body = TEST_TEMPLATE.safe_substitute(template_kwargs)
tests += test_body
if label in all_tests_found:
print(f"PROBLEM - Duplicate label found [{label}]", file=sys.stderr)
all_tests_found.add(label)
if is_red:
red_tests_found.add(label)
test_file_contents = TEST_FILE_TEMPLATE.safe_substitute(
{
"version": version,
"version_simple": version_simple,
"tests": tests,
}
)
test_file = os.path.join(CWL_API_TESTS_DIRECTORY, f"test_cwl_conformance_{version_simple}.py")
with open(test_file, "w") as f:
f.write(test_file_contents)
print(f"Finished writing {test_file}")
for red_test in red_tests_list:
if red_test not in red_tests_found:
print(f"PROBLEM - Failed to find annotated red test [{red_test}]", file=sys.stderr)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink