From 0a607640099884b800d047deb1afb1ba7f3dab2c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Feb 2026 17:35:41 -0500 Subject: [PATCH 01/14] Fix collection semantics YAML bugs and improve LaTeX rendering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML: fix wf_editor→workflow_editor key, swap paired/unpaired test refs, fix extra bracket typos, normalize collection syntax, fix prose typos. Python: add missing WORDS_TO_TEXTIFY entries, handle paired_or_unpaired as atomic token to avoid split into paired+unpaired during replacement. Co-Authored-By: Claude Opus 4.6 --- .../types/collection_semantics.yml | 40 +++++++++---------- .../dataset_collections/types/semantics.py | 9 ++++- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index d044ba11bb5..3629338865b 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -9,7 +9,7 @@ Any significantly sophisticated workflow language will have ways to collect data into arrays or vectors or dictionaries and apply operations across this data (mapping) - or reduce the dimensionality of this data (reductions). Typically, this explicitly + or reduce the dimensionality of this data (reductions). Typically, this is explicitly annotated with map functions or for loops. Galaxy however is designed to be a point and click interface for connecting steps and running tools. It is important that steps just connect and just do the most natural thing - and this is what Galaxy does. @@ -42,7 +42,7 @@ tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_collection" - wf_editor: "accepts paired data -> data connection" + workflow_editor: "accepts paired data -> data connection" - example: label: BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED @@ -56,7 +56,7 @@ then: "tool(i=mapOver(C)) ~> {o: collection}" tests: tool_runtime: - api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" workflow_editor: "accepts paired_or_unpaired data -> data connection" - example: @@ -71,7 +71,7 @@ then: "tool(i=mapOver(C)) ~> {o: collection}" tests: tool_runtime: - api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" + api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" workflow_editor: "accepts paired_or_unpaired data -> data connection" - example: @@ -83,7 +83,7 @@ out: {o: dataset} - collections: C: [list, {i1: "d_1", ..., in: "d_n"}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: "tool(i=mapOver(C)) ~> {o: collection}" tests: workflow_editor: "accepts collection data -> data connection" @@ -168,7 +168,7 @@ - collections: C1: ["list", {i1: d1_1, ..., in: d1_n}] C2: ["list", {i1: d2_1, ..., in: d2_n}] - then: "tool(i=mapOver(C1), i2=mapOver(C2)) ~> {o: collection}" + then: "tool(i=mapOver(C1), i2=mapOver(C2)) ~> {o: collection}" tests: tool_runtime: api_test: test_tools.py::TestToolsApi::test_map_over_two_collections @@ -181,7 +181,7 @@ Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime - this is often referred to a "reduction" in the code. + this is often referred to as a "reduction" in the code. - example: label: COLLECTION_INPUT_PAIRED @@ -201,7 +201,7 @@ - example: label: COLLECTION_INPUT_LIST assumptions: - - datasets: ["d1,...,dn"] + - datasets: ["d_1,...,d_n"] - tool: in: {i: "collection"} out: {o: dataset} @@ -300,7 +300,7 @@ then: "tool(i=C) == tool(i=[d_1,...,d_n])" tests: tool_runtime: - api_test: "test_tools.py::TestToolsApi::test_tools.py::test_reduce_collections" + api_test: "test_tools.py::TestToolsApi::test_reduce_collections" workflow_editor: "treats multi data input as list input" - doc: | @@ -329,7 +329,7 @@ in: {i: "dataset"} out: {o: dataset} - collections: - C: [paired_or_unpaired,{forward=d_f, reverse=d_r}] + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] then: "tool(i=C)" is_valid: false tests: @@ -356,7 +356,7 @@ workflow_editor: "accepts list:paired -> paired connection" - doc: | - The natural extension of multiple data input parameters consuming list collections as describe + The natural extension of multiple data input parameters consuming list collections as described above when discussing reductions is that nested lists of lists (``list:list``) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list @@ -371,7 +371,7 @@ out: {o: dataset} - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - then: "tool(i=mapOver(C, 'list')) ~> {o: collection}" + then: "tool(i=mapOver(C, 'list')) ~> {o: collection}" tests: workflow_editor: "maps list:list over multi data input" @@ -388,7 +388,7 @@ in: {i: "dataset"} out: {o: dataset} - collections: - C: [list:paired, {forward: d_f, reverse: d_r}] + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] then: "tool(i=mapOver(C, 'paired'))" is_valid: false tests: @@ -402,7 +402,7 @@ in: {i: "dataset"} out: {o: dataset} - collections: - C: ["list:paired_or_unpaired", {forward: d_f, reverse: d_r}] + C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] then: "tool(i=mapOver(C, 'paired_or_unpaired'))" is_valid: false tests: @@ -437,7 +437,7 @@ - doc: | - This inverse of this doesn't work intentionally. In some ways a ``paired`` collection + The inverse of this doesn't work intentionally. In some ways a ``paired`` collection acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist @@ -451,7 +451,7 @@ in: {i: "collection"} out: {o: dataset} - collections: - C: [paired_or_unpaired, {forward=d_f, reverse: d_r}] + C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] then: "tool(i=C) is invalid" tests: workflow_editor: "rejects paired_or_unpaired -> paired connection" @@ -485,7 +485,7 @@ in: {i: "collection"} out: {o: dataset} - collections: - C: [list:paired_or_unpaired, {el: {forward: f, reverse: r}}] + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] then: "tool(i=mapOver(C)) is invalid" tests: workflow_editor: "rejects list:paired_or_unpaired -> paired connection" @@ -498,7 +498,7 @@ in: {i: "collection"} out: {o: dataset} - collections: - C: [list:paired_or_unpaired, {el: {forward: f, reverse: r}}] + C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] then: "tool(i=mapOver(C)) is invalid" tests: workflow_editor: "rejects list:paired_or_unpaired -> list connection" @@ -529,7 +529,7 @@ - collections: C: [list, {i1: d_1, ..., in: d_n}] - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" - then: "tool(i=mapOver(C, 'single_datasets')) ~> {o: collection}" + then: "tool(i=mapOver(C, 'single_datasets')) ~> {o: collection}" tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" @@ -554,7 +554,7 @@ workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" - doc: | - Due only implementation time, the special casing of allowing paired_or_unpaired + Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired input, a paired:list cannot be consumed by a paired_or_unpaired:list input though diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index 8005ccbc169..48436256c67 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -117,7 +117,8 @@ class ExampleEntry(BaseModel): YAMLRootModel = RootModel[list[Union[DocEntry, ExampleEntry]]] -WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver"] +WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver", "collection", "dataset", "inner"] +_POU_PLACEHOLDER = "\x00POU\x00" def expression_to_latex(expression: str, wrap: bool = True): @@ -125,7 +126,11 @@ def expression_to_latex(expression: str, wrap: bool = True): expression = expression.replace("~>", "\\mapsto") expression = expression.replace("{", "\\left\\{") expression = expression.replace("}", "\\right\\}") - expression = expression.replace("paired_or_unpaired", "paired\\_or\\_unpaired") + expression = expression.replace("single_datasets", "\\text{single\\_datasets}") + expression = expression.replace("paired_or_unpaired", _POU_PLACEHOLDER) + expression = expression.replace("unpaired", "\\text{unpaired}") + expression = expression.replace("paired", "\\text{paired}") + expression = expression.replace(_POU_PLACEHOLDER, "\\text{paired\\_or\\_unpaired}") for word in WORDS_TO_TEXTIFY: expression = expression.replace(word, "\\text{" + word + "}") if wrap: From 36908bb11aedb67ca3cef730bd3d6b8aeed44f19 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Feb 2026 19:02:17 -0500 Subject: [PATCH 02/14] Add collection semantics validation and unit tests Implement check() with 3 validators: api_test refs (ast-based), framework tool XML existence, workflow_editor it() description matching. Add ExampleTests extra='forbid' to reject unknown keys. 39 unit tests covering models, LaTeX helpers, and all validators. Co-Authored-By: Claude Opus 4.6 --- .../dataset_collections/types/semantics.py | 98 +++++- .../data/model/test_collection_semantics.py | 323 ++++++++++++++++++ 2 files changed, 418 insertions(+), 3 deletions(-) create mode 100644 test/unit/data/model/test_collection_semantics.py diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index 48436256c67..afbc902a5d4 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -2,8 +2,10 @@ # how to use this function... # PYTHONPATH=lib python lib/galaxy/model/dataset_collections/types/semantics.py +import ast import argparse import os +import re import sys from io import StringIO from typing import ( @@ -16,6 +18,7 @@ from typing import ( import yaml from pydantic import ( BaseModel, + ConfigDict, Field, RootModel, ) @@ -43,6 +46,8 @@ ToolRuntimeTest = Union[ToolRuntimeApi, ToolRuntimeFramework] class ExampleTests(BaseModel): + model_config = ConfigDict(extra="forbid") + tool_runtime: Optional[ToolRuntimeTest] = None workflow_editor: Optional[str] = None @@ -170,9 +175,96 @@ def main(argv=None) -> None: generate_docs() -def check(): - # todo - pass +def _load_examples() -> list["Example"]: + semantics_yaml = yaml.safe_load( + resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml") + ) + root = YAMLRootModel.model_validate(semantics_yaml) + return [e.example for e in root.root if isinstance(e, ExampleEntry)] + + +def check() -> list[str]: + examples = _load_examples() + errors: list[str] = [] + errors.extend(validate_api_test_refs(examples)) + errors.extend(validate_tool_refs(examples)) + errors.extend(validate_workflow_editor_refs(examples)) + return errors + + +def validate_api_test_refs(examples: list["Example"]) -> list[str]: + errors: list[str] = [] + api_test_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "api") + for ex in examples: + if not ex.tests or not ex.tests.tool_runtime: + continue + if not isinstance(ex.tests.tool_runtime, ToolRuntimeApi): + continue + ref = ex.tests.tool_runtime.api_test + parts = ref.split("::") + filename = parts[0] + filepath = os.path.join(api_test_dir, filename) + if not os.path.exists(filepath): + errors.append(f"[{ex.label}] api_test file not found: {filename}") + continue + with open(filepath) as f: + tree = ast.parse(f.read(), filename=filepath) + if len(parts) == 2: + func_name = parts[1] + found = any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name + for node in ast.walk(tree) + ) + if not found: + errors.append(f"[{ex.label}] api_test function not found: {ref}") + elif len(parts) == 3: + class_name, method_name = parts[1], parts[2] + found = False + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + found = any( + isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name == method_name + for child in node.body + ) + break + if not found: + errors.append(f"[{ex.label}] api_test method not found: {ref}") + return errors + + +def validate_tool_refs(examples: list["Example"]) -> list[str]: + errors: list[str] = [] + tools_dir = os.path.join(galaxy_directory(), "test", "functional", "tools") + for ex in examples: + if not ex.tests or not ex.tests.tool_runtime: + continue + if not isinstance(ex.tests.tool_runtime, ToolRuntimeFramework): + continue + tool_id = ex.tests.tool_runtime.tool + tool_path = os.path.join(tools_dir, f"{tool_id}.xml") + if not os.path.exists(tool_path): + errors.append(f"[{ex.label}] framework tool XML not found: {tool_id}.xml") + return errors + + +def validate_workflow_editor_refs(examples: list["Example"]) -> list[str]: + errors: list[str] = [] + test_file = os.path.join( + galaxy_directory(), + "client", "src", "components", "Workflow", "Editor", "modules", "terminals.test.ts", + ) + if not os.path.exists(test_file): + return [f"workflow editor test file not found: {test_file}"] + with open(test_file) as f: + content = f.read() + it_descriptions = set(re.findall(r'it\(["\'](.+?)["\']\s*,', content)) + for ex in examples: + if not ex.tests or not ex.tests.workflow_editor: + continue + desc = ex.tests.workflow_editor + if desc not in it_descriptions: + errors.append(f"[{ex.label}] workflow_editor test not found: \"{desc}\"") + return errors def generate_docs(): diff --git a/test/unit/data/model/test_collection_semantics.py b/test/unit/data/model/test_collection_semantics.py new file mode 100644 index 00000000000..53c7c224b08 --- /dev/null +++ b/test/unit/data/model/test_collection_semantics.py @@ -0,0 +1,323 @@ +import pytest +import yaml +from pydantic import ValidationError + +from galaxy.model.dataset_collections.types.semantics import ( + CollectionDefinition, + DatasetsDeclaration, + DocEntry, + Example, + ExampleEntry, + ExampleTests, + ToolDefinition, + ToolRuntimeApi, + ToolRuntimeFramework, + YAMLRootModel, + arg_parser, + check, + collect_docs_with_examples, + elements_to_latex, + expression_to_latex, + validate_api_test_refs, + validate_tool_refs, + validate_workflow_editor_refs, +) +from galaxy.util.resources import resource_string + + + +def _load_root_model() -> YAMLRootModel: + raw = yaml.safe_load( + resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml") + ) + return YAMLRootModel.model_validate(raw) + + +def test_yaml_loads_and_parses(): + root = _load_root_model() + assert len(root.root) > 0 + + + +def test_all_entries_are_doc_or_example(): + root = _load_root_model() + for entry in root.root: + assert isinstance(entry, (DocEntry, ExampleEntry)) + + +def test_has_both_doc_and_example_entries(): + root = _load_root_model() + has_doc = any(isinstance(e, DocEntry) for e in root.root) + has_example = any(isinstance(e, ExampleEntry) for e in root.root) + assert has_doc + assert has_example + + + +@pytest.mark.parametrize( + "input_expr, expected_substr", + [ + ("->", "\\rightarrow"), + ("~>", "\\mapsto"), + ("list", "\\text{list}"), + ("forward", "\\text{forward}"), + ("reverse", "\\text{reverse}"), + ("mapOver", "\\text{mapOver}"), + ("collection", "\\text{collection}"), + ("dataset", "\\text{dataset}"), + ], +) +def test_expression_to_latex_replacements(input_expr, expected_substr): + result = expression_to_latex(input_expr) + assert expected_substr in result + + +def test_expression_to_latex_paired_or_unpaired_atomic(): + result = expression_to_latex("paired_or_unpaired") + assert "\\text{paired\\_or\\_unpaired}" in result + # should NOT contain bare \text{paired} or \text{unpaired} separately + without_pou = result.replace("\\text{paired\\_or\\_unpaired}", "") + assert "\\text{paired}" not in without_pou + assert "\\text{unpaired}" not in without_pou + + +def test_expression_to_latex_wrap_true(): + result = expression_to_latex("x", wrap=True) + assert result.startswith("$ ") + assert result.endswith(" $") + + +def test_expression_to_latex_wrap_false(): + result = expression_to_latex("x", wrap=False) + assert not result.startswith("$ ") + + + +def test_elements_to_latex_single_none(): + result = elements_to_latex({"a": None}) + assert "a" in result + assert "\\left\\{" in result + assert "\\right\\}" in result + + +def test_elements_to_latex_nested_dict(): + result = elements_to_latex({"outer": {"inner": None}}) + assert "\\text{ outer }" in result + assert "inner" in result + + +def test_elements_to_latex_multiple_keys(): + result = elements_to_latex({"a": None, "b": None}) + assert "a" in result + assert "b" in result + assert "," in result + + + +def test_datasets_declaration_as_latex_single(): + d = DatasetsDeclaration(datasets=["x"]) + assert d.as_latex() == "$ x $" + + +def test_datasets_declaration_as_latex_multiple(): + d = DatasetsDeclaration(datasets=["x", "y"]) + result = d.as_latex() + assert "$ x $" in result + assert "$ y $" in result + + +def test_tool_definition_as_latex(): + t = ToolDefinition(**{"in": {"i": "dataset"}, "out": {"o": "dataset"}}) + result = t.as_latex() + assert "\\Rightarrow" in result + assert "i:" in result + assert "o:" in result + + +def test_collection_definition_as_latex(): + c = CollectionDefinition(collection_type="paired", elements={"forward": None, "reverse": None}) + result = c.as_latex() + assert "paired" in result + assert "forward" in result + assert "reverse" in result + + + +def _make_root(entries: list) -> YAMLRootModel: + return YAMLRootModel.model_validate(entries) + + +def test_collect_doc_with_no_examples(): + root = _make_root([{"doc": "Some doc text"}]) + result = collect_docs_with_examples(root) + assert len(result) == 1 + doc, examples = result[0] + assert doc.doc == "Some doc text" + assert examples == [] + + +def test_collect_doc_with_examples(): + root = _make_root([ + {"doc": "Doc"}, + {"example": {"label": "EX1", "then": "a -> b"}}, + ]) + result = collect_docs_with_examples(root) + assert len(result) == 1 + _, examples = result[0] + assert len(examples) == 1 + assert examples[0].example.label == "EX1" + + +def test_collect_example_without_then_excluded(): + root = _make_root([ + {"doc": "Doc"}, + {"example": {"label": "NO_THEN"}}, + {"example": {"label": "HAS_THEN", "then": "x -> y"}}, + ]) + result = collect_docs_with_examples(root) + _, examples = result[0] + assert len(examples) == 1 + assert examples[0].example.label == "HAS_THEN" + + +def test_collect_multiple_doc_sections(): + root = _make_root([ + {"doc": "Doc1"}, + {"example": {"label": "EX1", "then": "a"}}, + {"doc": "Doc2"}, + {"example": {"label": "EX2", "then": "b"}}, + ]) + result = collect_docs_with_examples(root) + assert len(result) == 2 + assert result[0][0].doc == "Doc1" + assert result[1][0].doc == "Doc2" + assert len(result[0][1]) == 1 + assert len(result[1][1]) == 1 + + + +def test_arg_parser_default(): + args = arg_parser().parse_args([]) + assert args.check is False + + +def test_arg_parser_check_flag(): + args = arg_parser().parse_args(["--check"]) + assert args.check is True + + +def test_arg_parser_check_short_flag(): + args = arg_parser().parse_args(["-c"]) + assert args.check is True + + + +def test_example_tests_rejects_unknown_keys(): + with pytest.raises(ValidationError): + ExampleTests(**{"wf_editor": "some value"}) + + +def test_example_tests_accepts_valid_keys(): + et = ExampleTests(workflow_editor="some test description") + assert et.workflow_editor == "some test description" + + + +def _extract_examples(root: YAMLRootModel) -> list[Example]: + return [e.example for e in root.root if isinstance(e, ExampleEntry)] + + +def test_validate_api_test_refs_clean(): + """Current YAML should have no api_test reference errors.""" + root = _load_root_model() + examples = _extract_examples(root) + errors = validate_api_test_refs(examples) + assert errors == [] + + +def test_validate_api_test_refs_bad_file(): + """Mutated api_test ref with nonexistent file should be caught.""" + ex = Example( + label="BAD_FILE", + tests=ExampleTests( + tool_runtime=ToolRuntimeApi(api_test="nonexistent_file.py::some_func") + ), + ) + errors = validate_api_test_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_file.py" in errors[0] + + +def test_validate_api_test_refs_bad_function(): + """Mutated api_test ref with real file but nonexistent function should be caught.""" + ex = Example( + label="BAD_FUNC", + tests=ExampleTests( + tool_runtime=ToolRuntimeApi(api_test="test_tool_execute.py::nonexistent_function") + ), + ) + errors = validate_api_test_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_function" in errors[0] + + +def test_validate_api_test_refs_bad_method(): + """Mutated api_test ref with real class but nonexistent method should be caught.""" + ex = Example( + label="BAD_METHOD", + tests=ExampleTests( + tool_runtime=ToolRuntimeApi(api_test="test_tools.py::TestToolsApi::nonexistent_method") + ), + ) + errors = validate_api_test_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_method" in errors[0] + + + +def test_validate_tool_refs_clean(): + """Current YAML should have no framework tool reference errors.""" + root = _load_root_model() + examples = _extract_examples(root) + errors = validate_tool_refs(examples) + assert errors == [] + + +def test_validate_tool_refs_bad_tool(): + """Nonexistent tool XML should be caught.""" + ex = Example( + label="BAD_TOOL", + tests=ExampleTests( + tool_runtime=ToolRuntimeFramework(tool="nonexistent_tool_xyz") + ), + ) + errors = validate_tool_refs([ex]) + assert len(errors) == 1 + assert "nonexistent_tool_xyz" in errors[0] + + + +def test_validate_workflow_editor_refs_clean(): + """Current YAML should have no workflow_editor reference errors.""" + root = _load_root_model() + examples = _extract_examples(root) + errors = validate_workflow_editor_refs(examples) + assert errors == [] + + +def test_validate_workflow_editor_refs_bad_ref(): + """Nonexistent workflow_editor description should be caught.""" + ex = Example( + label="BAD_WF", + tests=ExampleTests(workflow_editor="totally nonexistent test description xyz"), + ) + errors = validate_workflow_editor_refs([ex]) + assert len(errors) == 1 + assert "totally nonexistent test description xyz" in errors[0] + + + +def test_check_returns_no_errors(): + """check() against current YAML should return empty error list.""" + errors = check() + assert errors == [] From efb5477f62c323080d3ea55635a2855baf85ed57 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Feb 2026 19:14:28 -0500 Subject: [PATCH 03/14] Normalize spacing in collection semantics then expressions Standardize collection} with no trailing spaces. Co-Authored-By: Claude Opus 4.6 --- .../model/dataset_collections/types/collection_semantics.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 3629338865b..8675bee9ae6 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -38,7 +38,7 @@ out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C)) ~> {o: collection }" + then: "tool(i=mapOver(C)) ~> {o: collection}" tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_collection" @@ -349,7 +349,7 @@ - collections: C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] "C\\_PAIRED": [paired, {forward: d_f,reverse: d_r}] - then: "tool(i=mapOver(C, 'paired')) ~> {o: collection}" + then: "tool(i=mapOver(C, 'paired')) ~> {o: collection}" tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_subcollection_mapping" From f83436743c2384d4ec5d09516337a056bc808c45 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Feb 2026 21:33:30 -0500 Subject: [PATCH 04/14] Add then expressions to 5 test-only collection semantics examples Co-Authored-By: Claude Opus 4.6 --- .../types/collection_semantics.yml | 45 +++++++++++++++++++ .../data/model/test_collection_semantics.py | 12 +++++ 2 files changed, 57 insertions(+) diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 8675bee9ae6..42262bce58b 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -275,11 +275,29 @@ - example: label: COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: "tool(i=C)" + is_valid: false tests: workflow_editor: "rejects paired:paired -> list:paired connection" - example: label: COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] + then: "tool(i=C)" + is_valid: false tests: workflow_editor: "rejects paired:paired -> list:paired_or_unpaired connection" @@ -510,6 +528,15 @@ - example: label: MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list:paired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + C_AS_MIXED: ["list:list:paired_or_unpaired", {o1: {el1: {forward: d_f, reverse: d_r}}}] + then: "tool(i=mapOver(C)) == tool(i=mapOver(C_AS_MIXED))" tests: workflow_editor: "accepts list:list:paired -> paired_or_unpaired connection" @@ -545,11 +572,29 @@ - example: label: MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_UNPAIRED_j = CollectionInstance for each dataset dj in C" + then: "tool(i=mapOver(C, 'single_datasets')) ~> {o: collection}" tests: workflow_editor: "accepts list:list -> paired_or_unpaired connection" - example: label: MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] + - "C_AS_LPU_i = inner list of C at position i, treated as list:paired_or_unpaired via single_datasets" + then: "tool(i=mapOver(C, 'list:paired_or_unpaired')) ~> {o: collection}" tests: workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" diff --git a/test/unit/data/model/test_collection_semantics.py b/test/unit/data/model/test_collection_semantics.py index 53c7c224b08..ead367d2618 100644 --- a/test/unit/data/model/test_collection_semantics.py +++ b/test/unit/data/model/test_collection_semantics.py @@ -317,6 +317,18 @@ def test_validate_workflow_editor_refs_bad_ref(): +def test_all_tested_examples_have_then(): + """Every example with test references should also have a then expression.""" + root = _load_root_model() + missing = [] + for entry in root.root: + if isinstance(entry, ExampleEntry): + ex = entry.example + if ex.tests and not ex.then: + missing.append(ex.label) + assert missing == [], f"Examples with tests but no then: {missing}" + + def test_check_returns_no_errors(): """check() against current YAML should return empty error list.""" errors = check() From 74fefc89d8933a0fc588f15fbe6f4e2a5c53df07 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 10 Feb 2026 09:58:52 -0500 Subject: [PATCH 05/14] Add structured YAML then expressions with bug fix and polish Convert 32 free-form then string expressions in collection_semantics.yml to structured typed YAML with 14 Pydantic models and as_latex() methods. Bug fix: NestedElements.elements typed as dict[str, "OutputBinding"] with model_rebuild() to resolve forward ref (was dict[str, Any]). Polish: discriminated unions via Annotated[..., Field(discriminator="type")] on all 4 union type aliases, rename elements_to_latex to _assumption_elements_to_latex. 61 tests (was 40): smoke tests for all as_latex() and generate_docs(), unit tests for EllipsisMarker, DatasetListInput, NestedElements, CollectionOutput with ellipsis, compound sub_collection_type, MapOverThen producing DatasetOutput. Co-Authored-By: Claude Opus 4.6 --- doc/source/dev/collection_semantics.md | 178 ++++++-- .../types/collection_semantics.yml | 415 ++++++++++++++++-- .../dataset_collections/types/semantics.py | 188 +++++++- .../data/model/test_collection_semantics.py | 312 ++++++++++++- 4 files changed, 1004 insertions(+), 89 deletions(-) diff --git a/doc/source/dev/collection_semantics.md b/doc/source/dev/collection_semantics.md index 2386e289c88..e1486e7e506 100644 --- a/doc/source/dev/collection_semantics.md +++ b/doc/source/dev/collection_semantics.md @@ -8,7 +8,7 @@ In particular it describes how they operate within Galaxy tools and workflows. Any significantly sophisticated workflow language will have ways to collect data into arrays or vectors or dictionaries and apply operations across this data (mapping) -or reduce the dimensionality of this data (reductions). Typically, this explicitly +or reduce the dimensionality of this data (reductions). Typically, this is explicitly annotated with map functions or for loops. Galaxy however is designed to be a point and click interface for connecting steps and running tools. It is important that steps just connect and just do the most natural thing - and this is what Galaxy does. @@ -47,7 +47,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection \right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired},\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$ ::: @@ -65,7 +65,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or\_unpaired},\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}>\right\}$$ ::: @@ -83,7 +83,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or\_unpaired},\left\{\text{unpaired}=tool(i=d_u)[o]\right\}>\right\}$$ ::: @@ -101,7 +101,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d_1)[o],...,in=tool(i=d_n)[o]]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ ::: @@ -131,7 +131,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list}:\text{list},\left\{o1=\left\{inner=tool(i=d_1)[o]\right\}\right\},...,\left\{on=\left\{inner=tool(i=d_n)[o]\right\}\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\text{list},\left\{\text{o1}=\left\{\text{inner}=tool(i=d_1)[o]\right\},...,\text{on}=\left\{\text{inner}=tool(i=d_n)[o]\right\}\right\}>\right\}$$ ::: @@ -149,7 +149,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: collection<\text{list}:paired\_or\_unpaired,\left\{el1=\left\{\text{forward}=tool(i=d_f)[o],\text{reverse}=tool(i=d_r)[o]\right\}\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\text{paired\_or\_unpaired},\left\{\text{el1}=\left\{\text{forward}=tool(i=d_f)[o], \text{reverse}=tool(i=d_r)[o]\right\}\right\}>\right\}$$ ::: @@ -180,7 +180,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C),i2=d_o) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d_1, i2=d_o)[o],...,in=tool(i=d_n, i2=d_o)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C), i2=d_o) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d_1, i2=d_o)[o],...,\text{in}=tool(i=d_n, i2=d_o)[o]\right\}>\right\}$$ ::: @@ -223,7 +223,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=d1_1, i2=d2_1)[o],...,in=tool(i=d1_n, i2=d2_n)[o]]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=d1_1, i2=d2_1)[o],...,\text{in}=tool(i=d1_n, i2=d2_n)[o]\right\}>\right\}$$ ::: @@ -241,7 +241,7 @@ collections directly and do not necessarily result in mapping over. Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime -this is often referred to a "reduction" in the code. +this is often referred to as a "reduction" in the code. (COLLECTION_INPUT_PAIRED)= @@ -262,7 +262,7 @@ Assuming, then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -273,14 +273,14 @@ $$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ Assuming, -* $ d1,...,dn $ are datasets +* $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ * $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -298,7 +298,7 @@ Assuming, then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -316,7 +316,7 @@ Assuming, then -$$tool(i=C) \rightarrow \left\{o: dataset\right\}$$ +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: @@ -332,6 +332,8 @@ then collection inputs must match every part of the collection type input defini (COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS)= (COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST)= +(COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= +(COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)=
Examples :::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` @@ -370,6 +372,42 @@ $$tool(i=C)\text{ is invalid}$$ +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + + +

@@ -437,7 +475,7 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ forward=d_f, reverse=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -473,7 +511,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C, 'paired')) \mapsto \left\{o: collection<\text{list}, \left\{el1: tool(i=C\_PAIRED)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ ::: @@ -483,7 +521,7 @@ $$tool(i=\text{mapOver}(C, 'paired')) \mapsto \left\{o: collection<\text{list}, -The natural extension of multiple data input parameters consuming list collections as describe +The natural extension of multiple data input parameters consuming list collections as described above when discussing reductions is that nested lists of lists (``list:list``) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list @@ -505,7 +543,7 @@ Assuming, then -$$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: collection<\text{list},\left\{o1: tool(i=[d_1])[o]\right\},...,on: tool(i=[d_n])[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{o1}=tool(i=[d_1])[o],...,\text{on}=tool(i=[d_n])[o]\right\}>\right\}$$ ::: @@ -531,12 +569,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C, 'paired'))\text{ is invalid}$$ +$$tool(i=\text{mapOver}(C, '\text{paired}'))\text{ is invalid}$$ ::: @@ -549,12 +587,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C, 'paired\_or\_unpaired'))\text{ is invalid}$$ +$$tool(i=\text{mapOver}(C, '\text{paired\_or\_unpaired}'))\text{ is invalid}$$ ::: @@ -587,7 +625,7 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ * $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ -* $ C_AS_MIXED = CollectionInstance $ +* $ C_AS_MIXED = CollectionInstance<\text{paired\_or\_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ then @@ -603,7 +641,7 @@ $$tool(i=C) == tool(i=C_AS_MIXED)$$ -This inverse of this doesn't work intentionally. In some ways a ``paired`` collection +The inverse of this doesn't work intentionally. In some ways a ``paired`` collection acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist @@ -620,12 +658,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ forward=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then -$$tool(i=C) is invalid$$ +$$tool(i=C)\text{ is invalid}$$ ::: @@ -673,12 +711,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=f, \text{ reverse }=r \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) is invalid$$ +$$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: @@ -691,12 +729,12 @@ Assuming, * $ d_f $, $ d_r $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=f, \text{ reverse }=r \right\} \right\}\text{>} $ +* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then -$$tool(i=\text{mapOver}(C)) is invalid$$ +$$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: @@ -711,6 +749,32 @@ can be mapped over either a ``paired_or_unpaired`` input to produce a nested lis or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). +(MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= +
Examples + +:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}list:list:paired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:list:paired\_or\_unpaired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ + + +then + +$$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ + +::: + + + +

+ + + In order for ``paired_or_unpaired`` collections to also act as a single dataset, a flat list can be mapped over a such an input with a special sub collection mapping @@ -728,12 +792,12 @@ Assuming, * $ d_1,...,d_n $ are datasets * $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ * $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -* $ C_AS_UNPAIRED_i = CollectionInstance for i from 1...n $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ then -$$tool(i=\text{mapOver}(C, 'single_datasets')) \mapsto \left\{o: collection<\text{list},\left\{i1=tool(i=C_AS_UNPAIRED_1)[o],...,in=tool(i=C_AS_UNPAIRED_n)[o]]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ ::: @@ -751,7 +815,53 @@ a ``list:paired_or_unpaired`` input to produce a flat list with the same structu as the outer list of the input. -Due only implementation time, the special casing of allowing paired_or_unpaired +(MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED)= +(MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED)= +
Examples + +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{un\text{paired}}=dj\right\}> for each \text{dataset} dj in C $ + + +then + +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{list}:\text{list},\left\{\text{o1}=\left\{\text{inner}=tool(i=C_AS_UNPAIRED_1)[o]\right\},...,\text{on}=\left\{\text{inner}=tool(i=C_AS_UNPAIRED_n)[o]\right\}\right\}>\right\}$$ + +::: + + + +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired\_or\_unpaired} via \text{single\_\text{dataset}s} $ + + +then + +$$tool(i=\text{mapOver}(C, '\text{list}:\text{paired\_or\_unpaired}')) \mapsto \left\{o: \text{collection}<\text{list},\left\{\text{o1}=tool(i=C_AS_LPU_1)[o],...,\text{on}=tool(i=C_AS_LPU_n)[o]\right\}>\right\}$$ + +::: + + + +

+ + + +Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired input, a paired:list cannot be consumed by a paired_or_unpaired:list input though diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 42262bce58b..3aacb55ff75 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -38,7 +38,24 @@ out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_collection" @@ -53,7 +70,24 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" @@ -68,7 +102,20 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {unpaired: d_u}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: paired_or_unpaired + elements: + unpaired: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_u}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" @@ -83,7 +130,25 @@ out: {o: dataset} - collections: C: [list, {i1: "d_1", ..., in: "d_n"}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o tests: workflow_editor: "accepts collection data -> data connection" @@ -100,7 +165,31 @@ out: {o: dataset} - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_map_over_nested_collections" @@ -115,7 +204,27 @@ out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: "list:paired_or_unpaired" + elements: + el1: + type: nested_elements + elements: + forward: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_f}}} + output: o + reverse: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_r}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_list_paired_or_unpaired" @@ -136,7 +245,26 @@ out: {o: dataset} - collections: C: ["list", {i1: d_1, ..., in: d_n}] - then: "tool(i=mapOver(C),i2=d_o) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + i2: {type: dataset, ref: d_o} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}, i2: {type: dataset, ref: d_o}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}, i2: {type: dataset, ref: d_o}}} + output: o tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_map_over_collected_and_individual_datasets" @@ -168,7 +296,26 @@ - collections: C1: ["list", {i1: d1_1, ..., in: d1_n}] C2: ["list", {i1: d2_1, ..., in: d2_n}] - then: "tool(i=mapOver(C1), i2=mapOver(C2)) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C1} + i2: {type: map_over, collection: C2} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_1}, i2: {type: dataset, ref: d2_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d1_n}, i2: {type: dataset, ref: d2_n}}} + output: o tests: tool_runtime: api_test: test_tools.py::TestToolsApi::test_map_over_two_collections @@ -192,9 +339,15 @@ out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: - tool_runtime: + tool_runtime: tool: collection_paired_test workflow_editor: "accepts paired -> paired connection" @@ -207,7 +360,13 @@ out: {o: dataset} - collections: C: [list, {el1: d_1, ..., eln: d_n}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: workflow_editor: "accepts list -> list connection" @@ -220,7 +379,13 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: tool_runtime: tool: collection_paired_or_unpaired @@ -235,7 +400,13 @@ out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] - then: "tool(i=C) -> {o: dataset}" + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} tests: tool_runtime: tool: collection_list_paired_or_unpaired @@ -250,11 +421,15 @@ assumptions: - datasets: [d_f, d_r] - tool: - in: {i: "collection"} + in: {i: "collection"} out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects connecting paired -> list" @@ -264,11 +439,15 @@ assumptions: - datasets: ["d_1,...,d_n"] - tool: - in: {i: "collection"} + in: {i: "collection"} out: {o: dataset} - collections: C: [list, {i1: d_1, ..., in: d_n}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects connecting list -> paired" @@ -282,7 +461,11 @@ out: {o: dataset} - collections: C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects paired:paired -> list:paired connection" @@ -296,7 +479,11 @@ out: {o: dataset} - collections: C: ["paired:paired", {forward: {forward: d_f, reverse: d_r}, reverse: {forward: d_f, reverse: d_r}}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects paired:paired -> list:paired_or_unpaired connection" @@ -315,7 +502,14 @@ out: {o: dataset} - collections: C: [list, {i1: d_1, ..., in: d_n}] - then: "tool(i=C) == tool(i=[d_1,...,d_n])" + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: dataset_list, refs: ["d_1", "...", "d_n"]} tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_reduce_collections" @@ -329,12 +523,16 @@ label: PAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: C: [paired, {forward: d_f, reverse: d_r}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects paired input on multi-data input" @@ -343,12 +541,16 @@ label: PAIRED_OR_UNPAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] - then: "tool(i=C)" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} is_valid: false tests: workflow_editor: "rejects paired_or_unpaired input on multi-data input" @@ -367,7 +569,20 @@ - collections: C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] "C\\_PAIRED": [paired, {forward: d_f,reverse: d_r}] - then: "tool(i=mapOver(C, 'paired')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: list + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_subcollection_mapping" @@ -389,7 +604,25 @@ out: {o: dataset} - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - then: "tool(i=mapOver(C, 'list')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: list} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_1"]}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: dataset_list, refs: ["d_n"]}}} + output: o tests: workflow_editor: "maps list:list over multi data input" @@ -402,12 +635,16 @@ label: LIST_PAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C, 'paired'))" + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} is_valid: false tests: workflow_editor: "rejects list:paired input on multi-data input" @@ -416,12 +653,16 @@ label: LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID assumptions: - datasets: [d_f, d_r] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C, 'paired_or_unpaired'))" + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired_or_unpaired} is_valid: false tests: workflow_editor: "rejects list:paired_or_unpaired input on multi-data input" @@ -447,7 +688,14 @@ - collections: C: [paired, {forward: d_f, reverse: d_r}] - "C_AS_MIXED = CollectionInstance" - then: "tool(i=C) == tool(i=C_AS_MIXED)" + then: + type: equivalence + left: + inputs: + i: {type: collection, ref: C} + right: + inputs: + i: {type: collection, ref: C_AS_MIXED} tests: tool_runtime: tool: collection_paired_or_unpaired @@ -470,7 +718,12 @@ out: {o: dataset} - collections: C: [paired_or_unpaired, {forward: d_f, reverse: d_r}] - then: "tool(i=C) is invalid" + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false tests: workflow_editor: "rejects paired_or_unpaired -> paired connection" @@ -491,7 +744,14 @@ - collections: C: ["list:paired", {el: {forward: d_f, reverse: d_r}}] C_AS_MIXED: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C)) == tool(i=mapOver(C_AS_MIXED))" + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} tests: workflow_editor: "accepts list:paired -> paired_or_unpaired connection" @@ -504,7 +764,12 @@ out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C)) is invalid" + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false tests: workflow_editor: "rejects list:paired_or_unpaired -> paired connection" @@ -517,7 +782,12 @@ out: {o: dataset} - collections: C: ["list:paired_or_unpaired", {el: {forward: d_f, reverse: d_r}}] - then: "tool(i=mapOver(C)) is invalid" + then: + type: invalid + invocation: + inputs: + i: {type: map_over, collection: C} + is_valid: false tests: workflow_editor: "rejects list:paired_or_unpaired -> list connection" @@ -536,7 +806,14 @@ - collections: C: ["list:list:paired", {o1: {el1: {forward: d_f, reverse: d_r}}}] C_AS_MIXED: ["list:list:paired_or_unpaired", {o1: {el1: {forward: d_f, reverse: d_r}}}] - then: "tool(i=mapOver(C)) == tool(i=mapOver(C_AS_MIXED))" + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} tests: workflow_editor: "accepts list:list:paired -> paired_or_unpaired connection" @@ -556,7 +833,25 @@ - collections: C: [list, {i1: d_1, ..., in: d_n}] - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" - then: "tool(i=mapOver(C, 'single_datasets')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: list + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" @@ -580,7 +875,31 @@ - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - "C_AS_UNPAIRED_j = CollectionInstance for each dataset dj in C" - then: "tool(i=mapOver(C, 'single_datasets')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: "list:list" + elements: + o1: + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + "on": + type: nested_elements + elements: + inner: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o tests: workflow_editor: "accepts list:list -> paired_or_unpaired connection" @@ -594,7 +913,25 @@ - collections: C: ["list:list", {o1: {inner: d_1}, ..., "on": {inner: d_n}}] - "C_AS_LPU_i = inner list of C at position i, treated as list:paired_or_unpaired via single_datasets" - then: "tool(i=mapOver(C, 'list:paired_or_unpaired')) ~> {o: collection}" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: "list:paired_or_unpaired"} + produces: + o: + type: collection + collection_type: list + elements: + o1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_1}}} + output: o + "...": {type: ellipsis} + "on": + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_n}}} + output: o tests: workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index afbc902a5d4..45792eb4d12 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -9,7 +9,9 @@ import re import sys from io import StringIO from typing import ( + Annotated, Any, + Literal, NamedTuple, Optional, Union, @@ -76,14 +78,14 @@ class ToolDeclaration(BaseModel): return self.tool.as_latex() -def elements_to_latex(elements: dict[str, Any]): +def _assumption_elements_to_latex(elements: dict[str, Any]): elements_as_strings = [] for identifier, value in elements.items(): if value is None: elements_as_strings.append(identifier) else: if isinstance(value, dict): - value_str = elements_to_latex(value) + value_str = _assumption_elements_to_latex(value) else: value_str = str(value) elements_as_strings.append(f"\\text{{ {identifier} }}={value_str}") @@ -97,7 +99,7 @@ class CollectionDefinition(NamedTuple): def as_latex(self) -> str: collection_type = self.collection_type.replace("_", "\\_") - return f"\\text{{CollectionInstance<}}{collection_type},{elements_to_latex(self.elements)}\\text{{>}}" + return f"\\text{{CollectionInstance<}}{collection_type},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" class CollectionDeclarations(BaseModel): @@ -107,10 +109,185 @@ class CollectionDeclarations(BaseModel): Expression = Union[str, DatasetsDeclaration, ToolDeclaration, CollectionDeclarations] +# --- Structured Then Expression Models --- + + +def _latex_type_word(word: str) -> str: + if word == "paired_or_unpaired": + return "\\text{paired\\_or\\_unpaired}" + if word == "single_datasets": + return "\\text{single\\_datasets}" + return "\\text{" + word + "}" + + +def _latex_collection_type(ct: str) -> str: + return ":".join(_latex_type_word(p) for p in ct.split(":")) + + +def _output_elements_to_latex(elements: dict) -> str: + parts = [] + for key, value in elements.items(): + if key == "...": + parts.append("...") + else: + parts.append(f"\\text{{{key}}}={value.as_latex()}") + # Join without spaces around ellipsis entries + result = "" + for i, part in enumerate(parts): + if i > 0: + if part == "..." or parts[i - 1] == "...": + result += "," + else: + result += ", " + result += part + return "\\left\\{" + result + "\\right\\}" + + +def _produces_to_latex(produces: dict) -> str: + parts = [] + for key, value in produces.items(): + parts.append(f"{key}: {value.as_latex()}") + return "\\left\\{" + ", ".join(parts) + "\\right\\}" + + +class DatasetInput(BaseModel): + type: Literal["dataset"] + ref: str + + def as_latex(self) -> str: + return self.ref + + +class MapOverInput(BaseModel): + type: Literal["map_over"] + collection: str + sub_collection_type: Optional[str] = None + + def as_latex(self) -> str: + if self.sub_collection_type: + sct = _latex_collection_type(self.sub_collection_type) + return f"\\text{{mapOver}}({self.collection}, '{sct}')" + return f"\\text{{mapOver}}({self.collection})" + + +class CollectionInput(BaseModel): + type: Literal["collection"] + ref: str + + def as_latex(self) -> str: + return self.ref + + +class DatasetListInput(BaseModel): + type: Literal["dataset_list"] + refs: list[str] + + def as_latex(self) -> str: + return "[" + ",".join(self.refs) + "]" + + +InputBinding = Annotated[Union[DatasetInput, MapOverInput, CollectionInput, DatasetListInput], Field(discriminator="type")] + + +class ToolInvocation(BaseModel): + inputs: dict[str, InputBinding] + + def as_latex(self) -> str: + args = ", ".join(f"{k}={v.as_latex()}" for k, v in self.inputs.items()) + return f"tool({args})" + + +class DatasetOutput(BaseModel): + type: Literal["dataset"] + + def as_latex(self) -> str: + return "\\text{dataset}" + + +class EllipsisMarker(BaseModel): + type: Literal["ellipsis"] + + def as_latex(self) -> str: + return "..." + + +class ToolOutputRef(BaseModel): + type: Literal["tool_output_ref"] + invocation: ToolInvocation + output: str + + def as_latex(self) -> str: + return f"{self.invocation.as_latex()}[{self.output}]" + + +class NestedElements(BaseModel): + type: Literal["nested_elements"] + elements: dict[str, "OutputBinding"] + + def as_latex(self) -> str: + return _output_elements_to_latex(self.elements) + + +OutputBinding = Annotated[Union[DatasetOutput, EllipsisMarker, ToolOutputRef, NestedElements], Field(discriminator="type")] +NestedElements.model_rebuild() + + +class CollectionOutput(BaseModel): + type: Literal["collection"] + collection_type: str + elements: dict[str, OutputBinding] + + def as_latex(self) -> str: + ct = _latex_collection_type(self.collection_type) + el = _output_elements_to_latex(self.elements) + return f"\\text{{collection}}<{ct},{el}>" + + +OutputSpec = Annotated[Union[DatasetOutput, CollectionOutput], Field(discriminator="type")] + + +class MapOverThen(BaseModel): + type: Literal["map_over"] + invocation: ToolInvocation + produces: dict[str, OutputSpec] + + def as_latex(self) -> str: + return f"{self.invocation.as_latex()} \\mapsto {_produces_to_latex(self.produces)}" + + +class ReductionThen(BaseModel): + type: Literal["reduction"] + invocation: ToolInvocation + produces: dict[str, OutputSpec] + + def as_latex(self) -> str: + return f"{self.invocation.as_latex()} \\rightarrow {_produces_to_latex(self.produces)}" + + +class EquivalenceThen(BaseModel): + type: Literal["equivalence"] + left: ToolInvocation + right: ToolInvocation + + def as_latex(self) -> str: + return f"{self.left.as_latex()} == {self.right.as_latex()}" + + +class InvalidThen(BaseModel): + type: Literal["invalid"] + invocation: ToolInvocation + + def as_latex(self) -> str: + return self.invocation.as_latex() + + +ThenExpression = Annotated[Union[MapOverThen, ReductionThen, EquivalenceThen, InvalidThen], Field(discriminator="type")] + + class Example(BaseModel): label: str assumptions: Optional[list[Expression]] = None - then: Optional[str] = None + then: Optional[ThenExpression] = None is_valid: bool = True tests: Optional[ExampleTests] = None @@ -312,8 +489,9 @@ def generate_docs(): validity_str = "" if not example.is_valid: validity_str = "\\text{ is invalid}" + then_latex = example.then.as_latex() markdown_content.write( - f"\n\nthen\n\n$${expression_to_latex(example.then, wrap=False)}{validity_str}$$\n\n" + f"\n\nthen\n\n$${then_latex}{validity_str}$$\n\n" ) markdown_content.write(":::\n\n") markdown_content.write("\n\n
\n\n") diff --git a/test/unit/data/model/test_collection_semantics.py b/test/unit/data/model/test_collection_semantics.py index ead367d2618..a8d697feb6b 100644 --- a/test/unit/data/model/test_collection_semantics.py +++ b/test/unit/data/model/test_collection_semantics.py @@ -4,20 +4,35 @@ from pydantic import ValidationError from galaxy.model.dataset_collections.types.semantics import ( CollectionDefinition, + CollectionInput, + CollectionOutput, DatasetsDeclaration, + DatasetInput, + DatasetListInput, + DatasetOutput, DocEntry, + EllipsisMarker, + EquivalenceThen, Example, ExampleEntry, ExampleTests, + InvalidThen, + MapOverInput, + MapOverThen, + NestedElements, + ReductionThen, ToolDefinition, + ToolInvocation, + ToolOutputRef, ToolRuntimeApi, ToolRuntimeFramework, YAMLRootModel, arg_parser, check, collect_docs_with_examples, - elements_to_latex, + _assumption_elements_to_latex, expression_to_latex, + generate_docs, validate_api_test_refs, validate_tool_refs, validate_workflow_editor_refs, @@ -93,21 +108,21 @@ def test_expression_to_latex_wrap_false(): -def test_elements_to_latex_single_none(): - result = elements_to_latex({"a": None}) +def test_assumption_elements_to_latex_single_none(): + result = _assumption_elements_to_latex({"a": None}) assert "a" in result assert "\\left\\{" in result assert "\\right\\}" in result -def test_elements_to_latex_nested_dict(): - result = elements_to_latex({"outer": {"inner": None}}) +def test_assumption_elements_to_latex_nested_dict(): + result = _assumption_elements_to_latex({"outer": {"inner": None}}) assert "\\text{ outer }" in result assert "inner" in result -def test_elements_to_latex_multiple_keys(): - result = elements_to_latex({"a": None, "b": None}) +def test_assumption_elements_to_latex_multiple_keys(): + result = _assumption_elements_to_latex({"a": None, "b": None}) assert "a" in result assert "b" in result assert "," in result @@ -156,10 +171,13 @@ def test_collect_doc_with_no_examples(): assert examples == [] +_SIMPLE_THEN = {"type": "invalid", "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}} + + def test_collect_doc_with_examples(): root = _make_root([ {"doc": "Doc"}, - {"example": {"label": "EX1", "then": "a -> b"}}, + {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, ]) result = collect_docs_with_examples(root) assert len(result) == 1 @@ -172,7 +190,7 @@ def test_collect_example_without_then_excluded(): root = _make_root([ {"doc": "Doc"}, {"example": {"label": "NO_THEN"}}, - {"example": {"label": "HAS_THEN", "then": "x -> y"}}, + {"example": {"label": "HAS_THEN", "then": _SIMPLE_THEN}}, ]) result = collect_docs_with_examples(root) _, examples = result[0] @@ -183,9 +201,9 @@ def test_collect_example_without_then_excluded(): def test_collect_multiple_doc_sections(): root = _make_root([ {"doc": "Doc1"}, - {"example": {"label": "EX1", "then": "a"}}, + {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, {"doc": "Doc2"}, - {"example": {"label": "EX2", "then": "b"}}, + {"example": {"label": "EX2", "then": _SIMPLE_THEN}}, ]) result = collect_docs_with_examples(root) assert len(result) == 2 @@ -333,3 +351,275 @@ def test_check_returns_no_errors(): """check() against current YAML should return empty error list.""" errors = check() assert errors == [] + + +# --- Structured Then Expression Model Tests --- + + +def test_invalid_then_collection_input(): + """InvalidThen with collection input: tool(i=C)""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + ) + assert then.as_latex() == expression_to_latex("tool(i=C)", wrap=False) + + +def test_invalid_then_map_over_with_sub_collection(): + """InvalidThen with mapOver sub-collection: tool(i=mapOver(C, 'paired'))""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired")} + ), + ) + assert then.as_latex() == expression_to_latex("tool(i=mapOver(C, 'paired'))", wrap=False) + + +def test_invalid_then_map_over_paired_or_unpaired(): + """InvalidThen with mapOver paired_or_unpaired sub-collection.""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired_or_unpaired")} + ), + ) + assert then.as_latex() == expression_to_latex("tool(i=mapOver(C, 'paired_or_unpaired'))", wrap=False) + + +def test_invalid_then_map_over_no_sub_collection(): + """InvalidThen with bare mapOver: tool(i=mapOver(C))""" + then = InvalidThen( + type="invalid", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C")} + ), + ) + assert then.as_latex() == expression_to_latex("tool(i=mapOver(C))", wrap=False) + + +def test_reduction_then(): + """ReductionThen: tool(i=C) -> {o: dataset}""" + then = ReductionThen( + type="reduction", + invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + produces={"o": DatasetOutput(type="dataset")}, + ) + assert then.as_latex() == expression_to_latex("tool(i=C) -> {o: dataset}", wrap=False) + + +def test_equivalence_then_list_reduction(): + """EquivalenceThen: tool(i=C) == tool(i=[d_1,...,d_n])""" + then = EquivalenceThen( + type="equivalence", + left=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + right=ToolInvocation(inputs={"i": DatasetListInput(type="dataset_list", refs=["d_1", "...", "d_n"])}), + ) + assert then.as_latex() == expression_to_latex("tool(i=C) == tool(i=[d_1,...,d_n])", wrap=False) + + +def test_equivalence_then_collection_refs(): + """EquivalenceThen: tool(i=C) == tool(i=C_AS_MIXED)""" + then = EquivalenceThen( + type="equivalence", + left=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C")}), + right=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C_AS_MIXED")}), + ) + assert then.as_latex() == expression_to_latex("tool(i=C) == tool(i=C_AS_MIXED)", wrap=False) + + +def test_map_over_then_paired(): + """MapOverThen for BASIC_MAPPING_PAIRED pattern.""" + then = MapOverThen( + type="map_over", + invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}), + produces={ + "o": CollectionOutput( + type="collection", + collection_type="paired", + elements={ + "forward": ToolOutputRef( + type="tool_output_ref", + invocation=ToolInvocation(inputs={"i": DatasetInput(type="dataset", ref="d_f")}), + output="o", + ), + "reverse": ToolOutputRef( + type="tool_output_ref", + invocation=ToolInvocation(inputs={"i": DatasetInput(type="dataset", ref="d_r")}), + output="o", + ), + }, + ) + }, + ) + result = then.as_latex() + assert "\\text{mapOver}(C)" in result + assert "\\text{paired}" in result + assert "\\mapsto" in result + assert "tool(i=d_f)[o]" in result + assert "tool(i=d_r)[o]" in result + + +def test_map_over_then_sub_collection(): + """MapOverThen for sub-collection mapping (MAPPING_LIST_PAIRED_OVER_PAIRED).""" + then = MapOverThen( + type="map_over", + invocation=ToolInvocation( + inputs={"i": MapOverInput(type="map_over", collection="C", sub_collection_type="paired")} + ), + produces={ + "o": CollectionOutput( + type="collection", + collection_type="list", + elements={ + "el1": ToolOutputRef( + type="tool_output_ref", + invocation=ToolInvocation( + inputs={"i": CollectionInput(type="collection", ref="C\\_PAIRED")} + ), + output="o", + ), + }, + ) + }, + ) + result = then.as_latex() + assert "\\text{mapOver}(C, '\\text{paired}')" in result + assert "\\text{list}" in result + assert "C\\_PAIRED" in result + + +def test_pydantic_parse_invalid_then_yaml(): + """Structured InvalidThen parses from dict (simulating YAML load).""" + data = { + "type": "invalid", + "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}, + } + then = InvalidThen.model_validate(data) + assert isinstance(then.invocation.inputs["i"], CollectionInput) + assert then.invocation.inputs["i"].ref == "C" + + +def test_pydantic_parse_reduction_then_yaml(): + """Structured ReductionThen parses from dict.""" + data = { + "type": "reduction", + "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}, + "produces": {"o": {"type": "dataset"}}, + } + then = ReductionThen.model_validate(data) + assert isinstance(then.produces["o"], DatasetOutput) + + +def test_example_accepts_structured_then(): + """Example model accepts structured then expression.""" + data = { + "label": "TEST", + "then": { + "type": "invalid", + "invocation": {"inputs": {"i": {"type": "collection", "ref": "C"}}}, + }, + "is_valid": False, + } + ex = Example.model_validate(data) + assert isinstance(ex.then, InvalidThen) + assert ex.is_valid is False + + +def test_example_rejects_string_then(): + """Example model no longer accepts string then after migration.""" + with pytest.raises(ValidationError): + Example(label="TEST", then="tool(i=C) -> {o: dataset}") + + +# --- Smoke Tests (Steps 3-4) --- + + +def test_all_then_as_latex_succeeds(): + """Every then expression produces valid LaTeX without error.""" + root = _load_root_model() + for entry in root.root: + if isinstance(entry, ExampleEntry) and entry.example.then: + latex = entry.example.then.as_latex() + assert isinstance(latex, str) + assert len(latex) > 0, f"{entry.example.label} produced empty LaTeX" + + +def test_generate_docs_succeeds(): + """generate_docs() runs to completion without errors.""" + generate_docs() + + +# --- Unit Tests for Uncovered Models (Step 5) --- + + +def test_ellipsis_marker_as_latex(): + m = EllipsisMarker(type="ellipsis") + assert m.as_latex() == "..." + + +def test_dataset_list_input_as_latex(): + d = DatasetListInput(type="dataset_list", refs=["d_1", "...", "d_n"]) + assert d.as_latex() == "[d_1,...,d_n]" + + +def test_nested_elements_as_latex(): + ne = NestedElements.model_validate({ + "type": "nested_elements", + "elements": { + "forward": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_f"}}}, + "output": "o", + }, + "reverse": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_r"}}}, + "output": "o", + }, + }, + }) + result = ne.as_latex() + assert "\\text{forward}=" in result + assert "\\text{reverse}=" in result + assert "tool(i=d_f)[o]" in result + + +def test_collection_output_with_ellipsis_as_latex(): + co = CollectionOutput.model_validate({ + "type": "collection", + "collection_type": "list", + "elements": { + "el1": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_1"}}}, + "output": "o", + }, + "...": {"type": "ellipsis"}, + "eln": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_n"}}}, + "output": "o", + }, + }, + }) + result = co.as_latex() + assert "\\text{list}" in result + assert ",...," in result + + +def test_map_over_input_compound_sub_collection_type(): + mi = MapOverInput(type="map_over", collection="C", sub_collection_type="list:paired_or_unpaired") + result = mi.as_latex() + assert "\\text{list}:\\text{paired\\_or\\_unpaired}" in result + + +def test_map_over_then_produces_dataset(): + then = MapOverThen( + type="map_over", + invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}), + produces={"o": DatasetOutput(type="dataset")}, + ) + result = then.as_latex() + assert "\\mapsto" in result + assert "\\text{dataset}" in result From 436b6a7a047a3106f6c413964497f3be5e933347 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 10 Feb 2026 13:14:47 -0500 Subject: [PATCH 06/14] Lint fixes. --- .../dataset_collections/types/semantics.py | 28 +++- .../data/model/test_collection_semantics.py | 156 ++++++++---------- 2 files changed, 89 insertions(+), 95 deletions(-) diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index 45792eb4d12..ac77086f44f 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -2,8 +2,8 @@ # how to use this function... # PYTHONPATH=lib python lib/galaxy/model/dataset_collections/types/semantics.py -import ast import argparse +import ast import os import re import sys @@ -99,7 +99,9 @@ class CollectionDefinition(NamedTuple): def as_latex(self) -> str: collection_type = self.collection_type.replace("_", "\\_") - return f"\\text{{CollectionInstance<}}{collection_type},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" + return ( + f"\\text{{CollectionInstance<}}{collection_type},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" + ) class CollectionDeclarations(BaseModel): @@ -186,7 +188,9 @@ class DatasetListInput(BaseModel): return "[" + ",".join(self.refs) + "]" -InputBinding = Annotated[Union[DatasetInput, MapOverInput, CollectionInput, DatasetListInput], Field(discriminator="type")] +InputBinding = Annotated[ + Union[DatasetInput, MapOverInput, CollectionInput, DatasetListInput], Field(discriminator="type") +] class ToolInvocation(BaseModel): @@ -228,7 +232,9 @@ class NestedElements(BaseModel): return _output_elements_to_latex(self.elements) -OutputBinding = Annotated[Union[DatasetOutput, EllipsisMarker, ToolOutputRef, NestedElements], Field(discriminator="type")] +OutputBinding = Annotated[ + Union[DatasetOutput, EllipsisMarker, ToolOutputRef, NestedElements], Field(discriminator="type") +] NestedElements.model_rebuild() @@ -428,7 +434,13 @@ def validate_workflow_editor_refs(examples: list["Example"]) -> list[str]: errors: list[str] = [] test_file = os.path.join( galaxy_directory(), - "client", "src", "components", "Workflow", "Editor", "modules", "terminals.test.ts", + "client", + "src", + "components", + "Workflow", + "Editor", + "modules", + "terminals.test.ts", ) if not os.path.exists(test_file): return [f"workflow editor test file not found: {test_file}"] @@ -440,7 +452,7 @@ def validate_workflow_editor_refs(examples: list["Example"]) -> list[str]: continue desc = ex.tests.workflow_editor if desc not in it_descriptions: - errors.append(f"[{ex.label}] workflow_editor test not found: \"{desc}\"") + errors.append(f'[{ex.label}] workflow_editor test not found: "{desc}"') return errors @@ -490,9 +502,7 @@ def generate_docs(): if not example.is_valid: validity_str = "\\text{ is invalid}" then_latex = example.then.as_latex() - markdown_content.write( - f"\n\nthen\n\n$${then_latex}{validity_str}$$\n\n" - ) + markdown_content.write(f"\n\nthen\n\n$${then_latex}{validity_str}$$\n\n") markdown_content.write(":::\n\n") markdown_content.write("\n\n
\n\n") markdown_content.write("\n\n") diff --git a/test/unit/data/model/test_collection_semantics.py b/test/unit/data/model/test_collection_semantics.py index a8d697feb6b..a12f24f96b6 100644 --- a/test/unit/data/model/test_collection_semantics.py +++ b/test/unit/data/model/test_collection_semantics.py @@ -3,19 +3,25 @@ import yaml from pydantic import ValidationError from galaxy.model.dataset_collections.types.semantics import ( + _assumption_elements_to_latex, + arg_parser, + check, + collect_docs_with_examples, CollectionDefinition, CollectionInput, CollectionOutput, - DatasetsDeclaration, DatasetInput, DatasetListInput, DatasetOutput, + DatasetsDeclaration, DocEntry, EllipsisMarker, EquivalenceThen, Example, ExampleEntry, ExampleTests, + expression_to_latex, + generate_docs, InvalidThen, MapOverInput, MapOverThen, @@ -26,25 +32,16 @@ from galaxy.model.dataset_collections.types.semantics import ( ToolOutputRef, ToolRuntimeApi, ToolRuntimeFramework, - YAMLRootModel, - arg_parser, - check, - collect_docs_with_examples, - _assumption_elements_to_latex, - expression_to_latex, - generate_docs, validate_api_test_refs, validate_tool_refs, validate_workflow_editor_refs, + YAMLRootModel, ) from galaxy.util.resources import resource_string - def _load_root_model() -> YAMLRootModel: - raw = yaml.safe_load( - resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml") - ) + raw = yaml.safe_load(resource_string("galaxy.model.dataset_collections.types", "collection_semantics.yml")) return YAMLRootModel.model_validate(raw) @@ -53,7 +50,6 @@ def test_yaml_loads_and_parses(): assert len(root.root) > 0 - def test_all_entries_are_doc_or_example(): root = _load_root_model() for entry in root.root: @@ -68,7 +64,6 @@ def test_has_both_doc_and_example_entries(): assert has_example - @pytest.mark.parametrize( "input_expr, expected_substr", [ @@ -107,7 +102,6 @@ def test_expression_to_latex_wrap_false(): assert not result.startswith("$ ") - def test_assumption_elements_to_latex_single_none(): result = _assumption_elements_to_latex({"a": None}) assert "a" in result @@ -128,7 +122,6 @@ def test_assumption_elements_to_latex_multiple_keys(): assert "," in result - def test_datasets_declaration_as_latex_single(): d = DatasetsDeclaration(datasets=["x"]) assert d.as_latex() == "$ x $" @@ -157,7 +150,6 @@ def test_collection_definition_as_latex(): assert "reverse" in result - def _make_root(entries: list) -> YAMLRootModel: return YAMLRootModel.model_validate(entries) @@ -175,10 +167,12 @@ _SIMPLE_THEN = {"type": "invalid", "invocation": {"inputs": {"i": {"type": "coll def test_collect_doc_with_examples(): - root = _make_root([ - {"doc": "Doc"}, - {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, - ]) + root = _make_root( + [ + {"doc": "Doc"}, + {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, + ] + ) result = collect_docs_with_examples(root) assert len(result) == 1 _, examples = result[0] @@ -187,11 +181,13 @@ def test_collect_doc_with_examples(): def test_collect_example_without_then_excluded(): - root = _make_root([ - {"doc": "Doc"}, - {"example": {"label": "NO_THEN"}}, - {"example": {"label": "HAS_THEN", "then": _SIMPLE_THEN}}, - ]) + root = _make_root( + [ + {"doc": "Doc"}, + {"example": {"label": "NO_THEN"}}, + {"example": {"label": "HAS_THEN", "then": _SIMPLE_THEN}}, + ] + ) result = collect_docs_with_examples(root) _, examples = result[0] assert len(examples) == 1 @@ -199,12 +195,14 @@ def test_collect_example_without_then_excluded(): def test_collect_multiple_doc_sections(): - root = _make_root([ - {"doc": "Doc1"}, - {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, - {"doc": "Doc2"}, - {"example": {"label": "EX2", "then": _SIMPLE_THEN}}, - ]) + root = _make_root( + [ + {"doc": "Doc1"}, + {"example": {"label": "EX1", "then": _SIMPLE_THEN}}, + {"doc": "Doc2"}, + {"example": {"label": "EX2", "then": _SIMPLE_THEN}}, + ] + ) result = collect_docs_with_examples(root) assert len(result) == 2 assert result[0][0].doc == "Doc1" @@ -213,7 +211,6 @@ def test_collect_multiple_doc_sections(): assert len(result[1][1]) == 1 - def test_arg_parser_default(): args = arg_parser().parse_args([]) assert args.check is False @@ -229,7 +226,6 @@ def test_arg_parser_check_short_flag(): assert args.check is True - def test_example_tests_rejects_unknown_keys(): with pytest.raises(ValidationError): ExampleTests(**{"wf_editor": "some value"}) @@ -240,7 +236,6 @@ def test_example_tests_accepts_valid_keys(): assert et.workflow_editor == "some test description" - def _extract_examples(root: YAMLRootModel) -> list[Example]: return [e.example for e in root.root if isinstance(e, ExampleEntry)] @@ -257,9 +252,7 @@ def test_validate_api_test_refs_bad_file(): """Mutated api_test ref with nonexistent file should be caught.""" ex = Example( label="BAD_FILE", - tests=ExampleTests( - tool_runtime=ToolRuntimeApi(api_test="nonexistent_file.py::some_func") - ), + tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="nonexistent_file.py::some_func")), ) errors = validate_api_test_refs([ex]) assert len(errors) == 1 @@ -270,9 +263,7 @@ def test_validate_api_test_refs_bad_function(): """Mutated api_test ref with real file but nonexistent function should be caught.""" ex = Example( label="BAD_FUNC", - tests=ExampleTests( - tool_runtime=ToolRuntimeApi(api_test="test_tool_execute.py::nonexistent_function") - ), + tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="test_tool_execute.py::nonexistent_function")), ) errors = validate_api_test_refs([ex]) assert len(errors) == 1 @@ -283,16 +274,13 @@ def test_validate_api_test_refs_bad_method(): """Mutated api_test ref with real class but nonexistent method should be caught.""" ex = Example( label="BAD_METHOD", - tests=ExampleTests( - tool_runtime=ToolRuntimeApi(api_test="test_tools.py::TestToolsApi::nonexistent_method") - ), + tests=ExampleTests(tool_runtime=ToolRuntimeApi(api_test="test_tools.py::TestToolsApi::nonexistent_method")), ) errors = validate_api_test_refs([ex]) assert len(errors) == 1 assert "nonexistent_method" in errors[0] - def test_validate_tool_refs_clean(): """Current YAML should have no framework tool reference errors.""" root = _load_root_model() @@ -305,16 +293,13 @@ def test_validate_tool_refs_bad_tool(): """Nonexistent tool XML should be caught.""" ex = Example( label="BAD_TOOL", - tests=ExampleTests( - tool_runtime=ToolRuntimeFramework(tool="nonexistent_tool_xyz") - ), + tests=ExampleTests(tool_runtime=ToolRuntimeFramework(tool="nonexistent_tool_xyz")), ) errors = validate_tool_refs([ex]) assert len(errors) == 1 assert "nonexistent_tool_xyz" in errors[0] - def test_validate_workflow_editor_refs_clean(): """Current YAML should have no workflow_editor reference errors.""" root = _load_root_model() @@ -334,7 +319,6 @@ def test_validate_workflow_editor_refs_bad_ref(): assert "totally nonexistent test description xyz" in errors[0] - def test_all_tested_examples_have_then(): """Every example with test references should also have a then expression.""" root = _load_root_model() @@ -391,9 +375,7 @@ def test_invalid_then_map_over_no_sub_collection(): """InvalidThen with bare mapOver: tool(i=mapOver(C))""" then = InvalidThen( type="invalid", - invocation=ToolInvocation( - inputs={"i": MapOverInput(type="map_over", collection="C")} - ), + invocation=ToolInvocation(inputs={"i": MapOverInput(type="map_over", collection="C")}), ) assert then.as_latex() == expression_to_latex("tool(i=mapOver(C))", wrap=False) @@ -474,9 +456,7 @@ def test_map_over_then_sub_collection(): elements={ "el1": ToolOutputRef( type="tool_output_ref", - invocation=ToolInvocation( - inputs={"i": CollectionInput(type="collection", ref="C\\_PAIRED")} - ), + invocation=ToolInvocation(inputs={"i": CollectionInput(type="collection", ref="C\\_PAIRED")}), output="o", ), }, @@ -564,21 +544,23 @@ def test_dataset_list_input_as_latex(): def test_nested_elements_as_latex(): - ne = NestedElements.model_validate({ - "type": "nested_elements", - "elements": { - "forward": { - "type": "tool_output_ref", - "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_f"}}}, - "output": "o", + ne = NestedElements.model_validate( + { + "type": "nested_elements", + "elements": { + "forward": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_f"}}}, + "output": "o", + }, + "reverse": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_r"}}}, + "output": "o", + }, }, - "reverse": { - "type": "tool_output_ref", - "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_r"}}}, - "output": "o", - }, - }, - }) + } + ) result = ne.as_latex() assert "\\text{forward}=" in result assert "\\text{reverse}=" in result @@ -586,23 +568,25 @@ def test_nested_elements_as_latex(): def test_collection_output_with_ellipsis_as_latex(): - co = CollectionOutput.model_validate({ - "type": "collection", - "collection_type": "list", - "elements": { - "el1": { - "type": "tool_output_ref", - "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_1"}}}, - "output": "o", + co = CollectionOutput.model_validate( + { + "type": "collection", + "collection_type": "list", + "elements": { + "el1": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_1"}}}, + "output": "o", + }, + "...": {"type": "ellipsis"}, + "eln": { + "type": "tool_output_ref", + "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_n"}}}, + "output": "o", + }, }, - "...": {"type": "ellipsis"}, - "eln": { - "type": "tool_output_ref", - "invocation": {"inputs": {"i": {"type": "dataset", "ref": "d_n"}}}, - "output": "o", - }, - }, - }) + } + ) result = co.as_latex() assert "\\text{list}" in result assert ",...," in result From 513cef7506d17033e5f554a6685a4c0921078f8d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 16 Feb 2026 11:58:02 -0500 Subject: [PATCH 07/14] Add workflow runtime API tests for collection semantics with mapping fix Add 14 workflow runtime tests covering basic mapping, reduction, and subtyping scenarios for collection semantics. Wire tests into collection_semantics.yml via new WorkflowRuntimeTest model. Fix bug in effective_collection_type where list:paired mapped over paired_or_unpaired produced empty string instead of "list" - the string-length subtraction assumed the subcollection type was a literal suffix but paired_or_unpaired (19 chars) != paired (6 chars). Co-Authored-By: Claude Opus 4.6 --- .../dataset_collections/type_description.py | 6 + .../types/collection_semantics.yml | 32 ++++- .../dataset_collections/types/semantics.py | 99 ++++++++++----- lib/galaxy_test/base/populators.py | 13 ++ .../collection_semantics_cat.gxwf-tests.yml | 110 +++++++++++++++++ .../collection_semantics_cat.gxwf.yml | 11 ++ ...on_semantics_cat_collection.gxwf-tests.yml | 21 ++++ ...llection_semantics_cat_collection.gxwf.yml | 13 ++ ...on_semantics_cat_two_inputs.gxwf-tests.yml | 76 ++++++++++++ ...llection_semantics_cat_two_inputs.gxwf.yml | 13 ++ ...ics_list_paired_or_unpaired.gxwf-tests.yml | 25 ++++ ...semantics_list_paired_or_unpaired.gxwf.yml | 13 ++ ...mantics_multi_data_optional.gxwf-tests.yml | 21 ++++ ...ion_semantics_multi_data_optional.gxwf.yml | 13 ++ ...emantics_paired_or_unpaired.gxwf-tests.yml | 116 ++++++++++++++++++ ...tion_semantics_paired_or_unpaired.gxwf.yml | 13 ++ .../test_type_descriptions.py | 9 ++ 17 files changed, 569 insertions(+), 35 deletions(-) create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index 77e7cec37e9..af392b4b796 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -71,6 +71,12 @@ class CollectionTypeDescription: if subcollection_type == "single_datasets": return self.collection_type + # When the subcollection type is paired_or_unpaired but the collection + # actually ends with :paired (a subtype match), strip the actual suffix + # rather than the longer paired_or_unpaired string. + if subcollection_type == "paired_or_unpaired" and self.collection_type.endswith(":paired"): + return self.collection_type[: -len(":paired")] + return self.collection_type[: -(len(subcollection_type) + 1)] def has_subcollections_of_type(self, other_collection_type) -> bool: diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 3aacb55ff75..7de23fa3778 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -59,6 +59,8 @@ tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_collection" + workflow_runtime: + framework_test: "collection_semantics_cat_0" workflow_editor: "accepts paired data -> data connection" - example: @@ -91,6 +93,8 @@ tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_paired" + workflow_runtime: + framework_test: "collection_semantics_cat_1" workflow_editor: "accepts paired_or_unpaired data -> data connection" - example: @@ -119,6 +123,8 @@ tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_paired_or_unpaired_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_2" workflow_editor: "accepts paired_or_unpaired data -> data connection" - example: @@ -228,6 +234,8 @@ tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_data_with_list_paired_or_unpaired" + workflow_runtime: + framework_test: "collection_semantics_cat_3" workflow_editor: "accepts list:paired_or_unpaired data -> data connection" - doc: | @@ -268,6 +276,8 @@ tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_map_over_collected_and_individual_datasets" + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_0" - doc: | If a tool consumes two input datasets and produces one output dataset, you can map two @@ -319,9 +329,11 @@ tests: tool_runtime: api_test: test_tools.py::TestToolsApi::test_map_over_two_collections + workflow_runtime: + framework_test: "collection_semantics_cat_two_inputs_1" - doc: "## Reduction" -- doc: | +- doc: | Not all tool executions result in implicit collections and mapping over inputs. Tool inputs of ``type`` ``data_collection`` can consume collections directly and do not necessarily result in mapping over. @@ -368,6 +380,8 @@ produces: o: {type: dataset} tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_0" workflow_editor: "accepts list -> list connection" - example: @@ -389,6 +403,8 @@ tests: tool_runtime: tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_0" workflow_editor: "accepts paired_or_unpaired data -> paired_or_unpaired connection" - example: @@ -410,6 +426,8 @@ tests: tool_runtime: tool: collection_list_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_0" workflow_editor: "accepts list:paired_or_unpaired data -> list:paired_or_unpaired connection" - doc: | @@ -497,7 +515,7 @@ label: LIST_REDUCTION assumptions: - datasets: ["d_1,...,d_n"] - - tool: + - tool: in: {i: "dataset"} out: {o: dataset} - collections: @@ -513,6 +531,8 @@ tests: tool_runtime: api_test: "test_tools.py::TestToolsApi::test_reduce_collections" + workflow_runtime: + framework_test: "collection_semantics_multi_data_optional_0" workflow_editor: "treats multi data input as list input" - doc: | @@ -699,6 +719,8 @@ tests: tool_runtime: tool: collection_paired_or_unpaired + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_1" workflow_editor: "accepts paired -> paired_or_unpaired connection" - doc: | @@ -753,6 +775,8 @@ inputs: i: {type: map_over, collection: C_AS_MIXED} tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_2" workflow_editor: "accepts list:paired -> paired_or_unpaired connection" - example: @@ -815,6 +839,8 @@ inputs: i: {type: map_over, collection: C_AS_MIXED} tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_3" workflow_editor: "accepts list:list:paired -> paired_or_unpaired connection" - doc: | @@ -855,6 +881,8 @@ tests: tool_runtime: api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_4" workflow_editor: "accepts list -> paired_or_unpaired connection" - doc: | diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index ac77086f44f..42f947b8b20 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -47,10 +47,16 @@ class ToolRuntimeFramework(BaseModel): ToolRuntimeTest = Union[ToolRuntimeApi, ToolRuntimeFramework] +class WorkflowRuntimeTest(BaseModel): + api_test: Optional[str] = None + framework_test: Optional[str] = None + + class ExampleTests(BaseModel): model_config = ConfigDict(extra="forbid") tool_runtime: Optional[ToolRuntimeTest] = None + workflow_runtime: Optional[WorkflowRuntimeTest] = None workflow_editor: Optional[str] = None @@ -375,43 +381,70 @@ def check() -> list[str]: return errors +def _validate_api_test_ref(label: str, ref: str, api_test_dir: str) -> Optional[str]: + parts = ref.split("::") + filename = parts[0] + filepath = os.path.join(api_test_dir, filename) + if not os.path.exists(filepath): + return f"[{label}] api_test file not found: {filename}" + with open(filepath) as f: + tree = ast.parse(f.read(), filename=filepath) + if len(parts) == 2: + func_name = parts[1] + found = any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name + for node in ast.walk(tree) + ) + if not found: + return f"[{label}] api_test function not found: {ref}" + elif len(parts) == 3: + class_name, method_name = parts[1], parts[2] + found = False + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + found = any( + isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name == method_name + for child in node.body + ) + break + if not found: + return f"[{label}] api_test method not found: {ref}" + return None + + +def _validate_framework_test_ref(label: str, ref: str, workflow_dir: str) -> Optional[str]: + """Validate a framework test ref like 'collection_semantics_cat_0'.""" + parts = ref.rsplit("_", 1) + if len(parts) != 2 or not parts[1].isdigit(): + return f"[{label}] framework_test must be 'workflow_name_index': {ref}" + workflow_name = parts[0] + workflow_file = os.path.join(workflow_dir, f"{workflow_name}.gxwf.yml") + if not os.path.exists(workflow_file): + return f"[{label}] framework_test workflow not found: {workflow_name}.gxwf.yml" + test_file = os.path.join(workflow_dir, f"{workflow_name}.gxwf-tests.yml") + if not os.path.exists(test_file): + return f"[{label}] framework_test test file not found: {workflow_name}.gxwf-tests.yml" + return None + + def validate_api_test_refs(examples: list["Example"]) -> list[str]: errors: list[str] = [] api_test_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "api") + workflow_dir = os.path.join(galaxy_directory(), "lib", "galaxy_test", "workflow") for ex in examples: - if not ex.tests or not ex.tests.tool_runtime: - continue - if not isinstance(ex.tests.tool_runtime, ToolRuntimeApi): - continue - ref = ex.tests.tool_runtime.api_test - parts = ref.split("::") - filename = parts[0] - filepath = os.path.join(api_test_dir, filename) - if not os.path.exists(filepath): - errors.append(f"[{ex.label}] api_test file not found: {filename}") - continue - with open(filepath) as f: - tree = ast.parse(f.read(), filename=filepath) - if len(parts) == 2: - func_name = parts[1] - found = any( - isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func_name - for node in ast.walk(tree) - ) - if not found: - errors.append(f"[{ex.label}] api_test function not found: {ref}") - elif len(parts) == 3: - class_name, method_name = parts[1], parts[2] - found = False - for node in ast.walk(tree): - if isinstance(node, ast.ClassDef) and node.name == class_name: - found = any( - isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name == method_name - for child in node.body - ) - break - if not found: - errors.append(f"[{ex.label}] api_test method not found: {ref}") + if ex.tests and ex.tests.tool_runtime and isinstance(ex.tests.tool_runtime, ToolRuntimeApi): + error = _validate_api_test_ref(ex.label, ex.tests.tool_runtime.api_test, api_test_dir) + if error: + errors.append(error) + if ex.tests and ex.tests.workflow_runtime: + if ex.tests.workflow_runtime.api_test: + error = _validate_api_test_ref(ex.label, ex.tests.workflow_runtime.api_test, api_test_dir) + if error: + errors.append(error) + if ex.tests.workflow_runtime.framework_test: + error = _validate_framework_test_ref(ex.label, ex.tests.workflow_runtime.framework_test, workflow_dir) + if error: + errors.append(error) return errors diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 6e5776f11ae..ebd5b0d1a87 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -4062,6 +4062,10 @@ def load_data_dict( fetch_response = dataset_collection_populator.create_list_of_pairs_in_history( history_id, contents=elements, wait=True, **new_collection_kwds ).json() + elif collection_type == "list:paired_or_unpaired": + fetch_response = dataset_collection_populator.create_list_of_paired_and_unpaired_in_history( + history_id, wait=True, **new_collection_kwds + ).json() elif collection_type and ":" in collection_type: fetch_response = { "outputs": [ @@ -4074,6 +4078,15 @@ def load_data_dict( fetch_response = dataset_collection_populator.create_list_in_history( history_id, contents=elements, direct_upload=True, wait=True, **new_collection_kwds ).json() + elif collection_type == "paired_or_unpaired": + if elements: + fetch_response = dataset_collection_populator.upload_collection( + history_id, "paired_or_unpaired", elements=elements, wait=True, **new_collection_kwds + ).json() + else: + fetch_response = dataset_collection_populator.create_paired_or_unpaired_pair_in_history( + history_id, wait=True, **new_collection_kwds + ).json() else: fetch_response = dataset_collection_populator.create_pair_in_history( history_id, contents=elements or None, direct_upload=True, wait=True, **new_collection_kwds diff --git a/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml new file mode 100644 index 00000000000..8139f677173 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml @@ -0,0 +1,110 @@ +- doc: | + BASIC_MAPPING_PAIRED: A paired collection mapped over a data input produces + a paired output collection. + job: + input1: + class: Collection + collection_type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: paired + elements: + forward: + asserts: + - that: has_text + text: "forward content" + reverse: + asserts: + - that: has_text + text: "reverse content" + +- doc: | + BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED: A paired_or_unpaired collection (paired variant) + mapped over a data input produces a paired_or_unpaired output collection with 2 elements. + job: + input1: + class: Collection + collection_type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: paired_or_unpaired + elements: + forward: + asserts: + - that: has_text + text: "forward content" + reverse: + asserts: + - that: has_text + text: "reverse content" + +- doc: | + BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED: A paired_or_unpaired collection (unpaired variant) + mapped over a data input produces a paired_or_unpaired output collection with 1 element. + job: + input1: + class: Collection + collection_type: paired_or_unpaired + elements: + - identifier: unpaired + class: File + contents: "unpaired content" + outputs: + wf_output: + class: Collection + collection_type: paired_or_unpaired + elements: + unpaired: + asserts: + - that: has_text + text: "unpaired content" + +- doc: | + BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection mapped over + a data input produces a list:paired_or_unpaired output collection. + job: + input1: + class: Collection + collection_type: list:paired_or_unpaired + elements: + - identifier: el1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: list:paired_or_unpaired + elements: + el1: + elements: + forward: + asserts: + - that: has_text + text: "forward content" + reverse: + asserts: + - that: has_text + text: "reverse content" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml new file mode 100644 index 00000000000..b76cf93b744 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf.yml @@ -0,0 +1,11 @@ +class: GalaxyWorkflow +inputs: + input1: data +outputs: + wf_output: + outputSource: cat_step/out_file1 +steps: + cat_step: + tool_id: cat + in: + input1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml new file mode 100644 index 00000000000..086cbb77fa0 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml @@ -0,0 +1,21 @@ +- doc: | + COLLECTION_INPUT_LIST: A list collection consumed by a tool expecting a list + collection input produces a single dataset. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "element 1" + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml new file mode 100644 index 00000000000..25ddcacdc7e --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list +outputs: + wf_output: + outputSource: tool_step/out_file1 +steps: + tool_step: + tool_id: cat_collection + in: + input1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml new file mode 100644 index 00000000000..b3d1a0f3cdb --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf-tests.yml @@ -0,0 +1,76 @@ +- doc: | + BASIC_MAPPING_INCLUDING_SINGLE_DATASET: Mapping a list collection over a data input + while another input is a single dataset produces a list output collection. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "list element 1" + - identifier: el2 + class: File + contents: "list element 2" + input2: + class: File + contents: "single dataset content" + outputs: + wf_output: + class: Collection + collection_type: list + elements: + el1: + asserts: + - that: has_text + text: "list element 1" + - that: has_text + text: "single dataset content" + el2: + asserts: + - that: has_text + text: "list element 2" + - that: has_text + text: "single dataset content" + +- doc: | + BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE: Mapping two list collections with + identical structure over two data inputs produces a list output collection. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "collection 1 element 1" + - identifier: el2 + class: File + contents: "collection 1 element 2" + input2: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "collection 2 element 1" + - identifier: el2 + class: File + contents: "collection 2 element 2" + outputs: + wf_output: + class: Collection + collection_type: list + elements: + el1: + asserts: + - that: has_text + text: "collection 1 element 1" + - that: has_text + text: "collection 2 element 1" + el2: + asserts: + - that: has_text + text: "collection 1 element 2" + - that: has_text + text: "collection 2 element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml new file mode 100644 index 00000000000..86a90d7aef7 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_two_inputs.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: data + input2: data +outputs: + wf_output: + outputSource: cat_step/out_file1 +steps: + cat_step: + tool_id: cat + in: + input1: input1 + queries_0|input2: input2 diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml new file mode 100644 index 00000000000..4f760a7639f --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml @@ -0,0 +1,25 @@ +- doc: | + COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection + consumed by a tool expecting list:paired_or_unpaired produces a single dataset. + job: + input1: + class: Collection + collection_type: list:paired_or_unpaired + elements: + - identifier: el1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml new file mode 100644 index 00000000000..cd5937397a8 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list:paired_or_unpaired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_list_paired_or_unpaired + in: + f1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml new file mode 100644 index 00000000000..30685576196 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf-tests.yml @@ -0,0 +1,21 @@ +- doc: | + LIST_REDUCTION: A list collection consumed by a multi-data input (reduction) + produces a single dataset containing all elements. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "element 1" + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml new file mode 100644 index 00000000000..3f60c35b20f --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_multi_data_optional.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: multi_data_optional + in: + input1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml new file mode 100644 index 00000000000..8db03048fac --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml @@ -0,0 +1,116 @@ +- doc: | + COLLECTION_INPUT_PAIRED_OR_UNPAIRED: A paired_or_unpaired collection consumed + directly by a tool expecting paired_or_unpaired produces a single dataset. + job: + input1: + class: Collection + collection_type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" + +- doc: | + PAIRED_OR_UNPAIRED_CONSUMES_PAIRED: A paired collection can be consumed by a + paired_or_unpaired tool input (subtype compatibility). + job: + input1: + class: Collection + collection_type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" + +- doc: | + MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED: A list:paired collection mapped over + a paired_or_unpaired tool input produces a list output collection (subtype + mapping). + job: + input1: + class: Collection + collection_type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" + outputs: + wf_output: + class: Collection + collection_type: list + +- doc: | + MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED: A list:list:paired collection mapped + over a paired_or_unpaired tool input produces a list:list output collection. + job: + input1: + class: Collection + collection_type: list:list:paired + elements: + - identifier: test_level_2 + class: Collection + type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" + outputs: + wf_output: + class: Collection + collection_type: list:list + +- doc: | + MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection mapped over + a paired_or_unpaired tool input produces a list output collection. + job: + input1: + class: Collection + collection_type: list:paired_or_unpaired + elements: + - identifier: test_level_1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" + outputs: + wf_output: + class: Collection + collection_type: list diff --git a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml new file mode 100644 index 00000000000..98252c3723c --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: paired_or_unpaired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_paired_or_unpaired + in: + f1: input1 diff --git a/test/unit/data/dataset_collections/test_type_descriptions.py b/test/unit/data/dataset_collections/test_type_descriptions.py index 8416ed2edb2..d7cf727f593 100644 --- a/test/unit/data/dataset_collections/test_type_descriptions.py +++ b/test/unit/data/dataset_collections/test_type_descriptions.py @@ -86,6 +86,15 @@ def test_sample_sheet_acts_like_list(): assert sample_sheet_paired.effective_collection_type(paired_type) == "sample_sheet" +def test_effective_collection_type_paired_or_unpaired_over_paired(): + # list:paired mapped over paired_or_unpaired should yield "list" + assert c_t("list:paired").effective_collection_type("paired_or_unpaired") == "list" + # list:list:paired mapped over paired_or_unpaired should yield "list:list" + assert c_t("list:list:paired").effective_collection_type("paired_or_unpaired") == "list:list" + # list:paired_or_unpaired mapped over paired_or_unpaired should still yield "list" + assert c_t("list:paired_or_unpaired").effective_collection_type("paired_or_unpaired") == "list" + + def test_validate(): c_t("list").validate() c_t("list:paired").validate() From e0c1a0067858380747799a7bac35faeac1dca240 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 20 Feb 2026 14:14:49 -0500 Subject: [PATCH 08/14] Add sample_sheet collection semantics examples for workflow editor tests Co-Authored-By: Claude Opus 4.6 --- doc/source/dev/collection_semantics.md | 706 ++++++++++-------- .../types/collection_semantics.yml | 253 +++++++ 2 files changed, 643 insertions(+), 316 deletions(-) diff --git a/doc/source/dev/collection_semantics.md b/doc/source/dev/collection_semantics.md index e1486e7e506..1e3ad18695d 100644 --- a/doc/source/dev/collection_semantics.md +++ b/doc/source/dev/collection_semantics.md @@ -28,22 +28,21 @@ mapped over. Each element of the implicit collections correspond to their own jo Galaxy very naturally and intuitively parallelizes jobs without extra work from the user and without any knowledge of the tool. - (BASIC_MAPPING_PAIRED)= (BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED)= (BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED)= (BASIC_MAPPING_LIST)= +
Examples -:::{admonition} Example: `BASIC_MAPPING_PAIRED` +:::{admonition} Example: `BASIC_MAPPING_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -51,17 +50,14 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired},\l ::: - - -:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED` +:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -69,17 +65,14 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or ::: - - -:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED` +:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED` :class: note Assuming, -* $ d_u $ is a dataset -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ unpaired }=d_u \right\}\text{>} $ - +- $ d_u $ is a dataset +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ unpaired }=d_u \right\}\text{>} $ then @@ -87,17 +80,14 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or ::: - - -:::{admonition} Example: `BASIC_MAPPING_LIST` +:::{admonition} Example: `BASIC_MAPPING_LIST` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then @@ -105,29 +95,24 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list},\lef ::: - -

- - The above description of mapping over inputs works naturally and as expected for nested collections. - (NESTED_LIST_MAPPING)= (BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED)= +
Examples -:::{admonition} Example: `NESTED_LIST_MAPPING` +:::{admonition} Example: `NESTED_LIST_MAPPING` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ then @@ -135,17 +120,14 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\tex ::: - - -:::{admonition} Example: `BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -153,30 +135,24 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\tex ::: - -

- - - For tools with multiple data inputs, the tool can be executed with individual datasets for the non-mapped over input and each tool execution will just be executed with that dataset. The dataset not mapped over serves as the input for each execution. - (BASIC_MAPPING_INCLUDING_SINGLE_DATASET)= +
Examples -:::{admonition} Example: `BASIC_MAPPING_INCLUDING_SINGLE_DATASET` +:::{admonition} Example: `BASIC_MAPPING_INCLUDING_SINGLE_DATASET` :class: note Assuming, -* $ d_1,...,d_n $, $ d_o $ are datasets -* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ - +- $ d_1,...,d_n $, $ d_o $ are datasets +- $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then @@ -184,12 +160,8 @@ $$tool(i=\text{mapOver}(C), i2=d_o) \mapsto \left\{o: \text{collection}<\text{li ::: - -

- - If a tool consumes two input datasets and produces one output dataset, you can map two collections with identical structure (same element identifiers in the same order) over the respective inputs and the result is an implicit collection with the same structure @@ -206,20 +178,19 @@ of map over operations on tools - the results will all continue to match and wor very naturally - again without extra work by the user and without extra knowledge by the tool author. - (BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE)= +
Examples -:::{admonition} Example: `BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE` +:::{admonition} Example: `BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE` :class: note Assuming, -* $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets -* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C1 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ -* $ C2 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ - +- $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets +- $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C1 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ +- $ C2 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ then @@ -227,38 +198,33 @@ $$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: \text{coll ::: - -

- - ## Reduction Not all tool executions result in implicit collections and mapping -over inputs. Tool inputs of ``type`` ``data_collection`` can consume +over inputs. Tool inputs of `type` `data_collection` can consume collections directly and do not necessarily result in mapping over. Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime this is often referred to as a "reduction" in the code. - (COLLECTION_INPUT_PAIRED)= (COLLECTION_INPUT_LIST)= (COLLECTION_INPUT_PAIRED_OR_UNPAIRED)= (COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED)= +
Examples -:::{admonition} Example: `COLLECTION_INPUT_PAIRED` +:::{admonition} Example: `COLLECTION_INPUT_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -266,17 +232,14 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: - - -:::{admonition} Example: `COLLECTION_INPUT_LIST` +:::{admonition} Example: `COLLECTION_INPUT_LIST` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ then @@ -284,17 +247,14 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: - - -:::{admonition} Example: `COLLECTION_INPUT_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `COLLECTION_INPUT_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -302,17 +262,14 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: - - -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -320,31 +277,26 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: - -

- - -For nested collections where each rank is a ``list`` or a ``paired`` collection, +For nested collections where each rank is a `list` or a `paired` collection, then collection inputs must match every part of the collection type input definition. - (COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS)= (COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST)= (COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= (COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= +
Examples -:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` +:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -352,17 +304,14 @@ $$tool(i=C)\text{ is invalid}$$ ::: - - -:::{admonition} Example: `COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST` +:::{admonition} Example: `COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then @@ -370,17 +319,14 @@ $$tool(i=C)\text{ is invalid}$$ ::: - - -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -388,17 +334,14 @@ $$tool(i=C)\text{ is invalid}$$ ::: - - -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -406,29 +349,24 @@ $$tool(i=C)\text{ is invalid}$$ ::: - -

- - -In addition to explicit collection inputs, tool inputs of ``type`` ``data`` -where ``multiple="true"`` can consume lists directly. This is likewise a +In addition to explicit collection inputs, tool inputs of `type` `data` +where `multiple="true"` can consume lists directly. This is likewise a "reduction" and does not result in implicit collection creation. - (LIST_REDUCTION)= +
Examples -:::{admonition} Example: `LIST_REDUCTION` +:::{admonition} Example: `LIST_REDUCTION` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ then @@ -436,29 +374,24 @@ $$tool(i=C) == tool(i=[d_1,...,d_n])$$ ::: - -

- - -Paired collections cannot be reduced this way. ``paired`` is not meant +Paired collections cannot be reduced this way. `paired` is not meant to represent a list/array/vector data structure - it is more like a tuple. - (PAIRED_REDUCTION_INVALID)= (PAIRED_OR_UNPAIRED_REDUCTION_INVALID)= +
Examples -:::{admonition} Example: `PAIRED_REDUCTION_INVALID` +:::{admonition} Example: `PAIRED_REDUCTION_INVALID` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -466,17 +399,14 @@ $$tool(i=C)\text{ is invalid}$$ ::: - - -:::{admonition} Example: `PAIRED_OR_UNPAIRED_REDUCTION_INVALID` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_REDUCTION_INVALID` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -484,30 +414,25 @@ $$tool(i=C)\text{ is invalid}$$ ::: - -

- - ## Sub-collection Mapping ![](https://planemo.readthedocs.io/en/master/_images/subcollection_mapping_identifiers.svg) - (MAPPING_LIST_PAIRED_OVER_PAIRED)= +
Examples -:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED` +:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -* $ C\_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +- $ C_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -515,31 +440,26 @@ $$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection} ::: - -

- - The natural extension of multiple data input parameters consuming list collections as described -above when discussing reductions is that nested lists of lists (``list:list``) can be mapped +above when discussing reductions is that nested lists of lists (`list:list`) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list of the input collection. - (NESTED_LIST_REDUCTION)= +
Examples -:::{admonition} Example: `NESTED_LIST_REDUCTION` +:::{admonition} Example: `NESTED_LIST_REDUCTION` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ then @@ -547,30 +467,25 @@ $$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: \text{collection}<\ ::: - -

- - Just as a paired collection won't be reduced by a multiple data input, any sort of nested collection ending in a paired collection cannot be mapped over such an input. So a multiple -data input parameter cannot be mapped over by a list of pairs (``list:paired``) for instance. - +data input parameter cannot be mapped over by a list of pairs (`list:paired`) for instance. (LIST_PAIRED_REDUCTION_INVALID)= (LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID)= +
Examples -:::{admonition} Example: `LIST_PAIRED_REDUCTION_INVALID` +:::{admonition} Example: `LIST_PAIRED_REDUCTION_INVALID` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -578,17 +493,14 @@ $$tool(i=\text{mapOver}(C, '\text{paired}'))\text{ is invalid}$$ ::: - - -:::{admonition} Example: `LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID` +:::{admonition} Example: `LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -596,37 +508,32 @@ $$tool(i=\text{mapOver}(C, '\text{paired\_or\_unpaired}'))\text{ is invalid}$$ ::: - -

- - ## paired_or_unpaired Collections -The collection type ``paired_or_unpaired`` is meant to serve as a stand-in for -an entity that can be either a single dataset or what is effectively a ``paired`` +The collection type `paired_or_unpaired` is meant to serve as a stand-in for +an entity that can be either a single dataset or what is effectively a `paired` dataset collection. These collections either have one element with identifier -``unpaired`` or two elements with identifiers ``forward`` and ``reverse``. - -Tools can declare a data_collection input with collection type ``paired_or_unpaired`` -and that input will consume either an explicit ``paired_or_unpaired`` collection -normally or can consume a ``paired`` input. +`unpaired` or two elements with identifiers `forward` and `reverse`. +Tools can declare a data_collection input with collection type `paired_or_unpaired` +and that input will consume either an explicit `paired_or_unpaired` collection +normally or can consume a `paired` input. (PAIRED_OR_UNPAIRED_CONSUMES_PAIRED)= +
Examples -:::{admonition} Example: `PAIRED_OR_UNPAIRED_CONSUMES_PAIRED` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_CONSUMES_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ -* $ C_AS_MIXED = CollectionInstance<\text{paired\_or\_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +- $ C_AS_MIXED = CollectionInstance<\text{paired_or_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ then @@ -634,32 +541,26 @@ $$tool(i=C) == tool(i=C_AS_MIXED)$$ ::: - -

- - - -The inverse of this doesn't work intentionally. In some ways a ``paired`` collection -acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired -collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` -dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist -in ``paired_or_unpaired`` collection. - +The inverse of this doesn't work intentionally. In some ways a `paired` collection +acts as a `paired_or_unpaired` collection but a `paired_or_unpaired` is not a paired +collection. This makes a lot of sense in terms of tools - a tool consuming a `paired` +dataset expects to find both a `forward` and `reverse` element but these may not exist +in `paired_or_unpaired` collection. (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED)= +
Examples -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}paired\_or\_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ then @@ -667,34 +568,28 @@ $$tool(i=C)\text{ is invalid}$$ ::: - -

- - - -The same logic holds for mapping, lists of paired datasets (``list:paired``) can be mapped over these -``paired_or_unpaired`` inputs and mixed lists of pairs (``list:paired_or_unpaired``) cannot -be mapped over a ``paired`` input. Following the same logic, ``list:paired_or_unpaired`` cannot -be mapped over a ``list`` input or multiple data input. - +The same logic holds for mapping, lists of paired datasets (`list:paired`) can be mapped over these +`paired_or_unpaired` inputs and mixed lists of pairs (`list:paired_or_unpaired`) cannot +be mapped over a `paired` input. Following the same logic, `list:paired_or_unpaired` cannot +be mapped over a `list` input or multiple data input. (MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING)= (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING)= +
Examples -:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -* $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +- $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -702,17 +597,14 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: - - -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -720,17 +612,14 @@ $$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: - - -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:paired\_or\_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ then @@ -738,30 +627,25 @@ $$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: - -

- - -This logic extends naturally into higher dimensional collections. A ``list:list:paired`` -can be mapped over either a ``paired_or_unpaired`` input to produce a nested list (``list:list``) -or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). - +This logic extends naturally into higher dimensional collections. A `list:list:paired` +can be mapped over either a `paired_or_unpaired` input to produce a nested list (`list:list`) +or a `list:paired_or_unpaired` input to produce a flat list (`list`). (MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= +
Examples -:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_f $, $ d_r $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list:paired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ -* $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:list:paired\_or\_unpaired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ - +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:list:paired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +- $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:list:paired_or_unpaired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ then @@ -769,31 +653,25 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: - -

- - - -In order for ``paired_or_unpaired`` collections to also act as a single dataset, +In order for `paired_or_unpaired` collections to also act as a single dataset, a flat list can be mapped over a such an input with a special sub collection mapping type of 'single_datasets'. - (MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED)= +
Examples -:::{admonition} Example: `MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +- $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ then @@ -801,34 +679,29 @@ $$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{c ::: - -

- - This treatment of lists without pairing extends to nested structures naturally. -For instance, a list of list of datasets (``list:list``) can be mapped over a -``paired_or_unpaired`` input to produce a nested list of lists (``list:list``) +For instance, a list of list of datasets (`list:list`) can be mapped over a +`paired_or_unpaired` input to produce a nested list of lists (`list:list`) with a structure matching the input. Likewise, the nested list can be mapped over -a ``list:paired_or_unpaired`` input to produce a flat list with the same structure +a `list:paired_or_unpaired` input to produce a flat list with the same structure as the outer list of the input. - (MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED)= (MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED)= +
Examples -:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ -* $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{un\text{paired}}=dj\right\}> for each \text{dataset} dj in C $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +- $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=dj\right\}> for each \text{dataset} dj in C $ then @@ -836,18 +709,15 @@ $$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{c ::: - - -:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -* $ d_1,...,d_n $ are datasets -* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -* $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ -* $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired\_or\_unpaired} via \text{single\_\text{dataset}s} $ - +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +- $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired_or_unpaired} via \text{single\_\text{dataset}s} $ then @@ -855,12 +725,8 @@ $$tool(i=\text{mapOver}(C, '\text{list}:\text{paired\_or\_unpaired}')) \mapsto \ ::: - -

- - Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired @@ -869,4 +735,212 @@ it should be able to for consistency. We have focused our time on data structure more likely to be used in actual Galaxy analyses given current and guessed future usage. +## sample_sheet Collections +The collection type `sample_sheet` attaches typed, columnar metadata to each +element of a dataset collection. For mapping and type matching purposes, +`sample_sheet` behaves identically to `list` - it can be mapped over tool +inputs, matched against `list` collection inputs, and composed with inner types +(`sample_sheet:paired`, `sample_sheet:paired_or_unpaired`). The key asymmetry +is that while a `sample_sheet` output can satisfy a `list` input, a `list` +output cannot satisfy a `sample_sheet` input - sample sheets carry metadata +that plain lists do not. + +(SAMPLE_SHEET_MAPPING)= +(SAMPLE_SHEET_MATCHES_LIST)= + +
Examples + +:::{admonition} Example: `SAMPLE_SHEET_MAPPING` +:class: note + +Assuming, + +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + +then + +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ + +::: + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST` +:class: note + +Assuming, + +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + +

+ +Sub-collection mapping works the same as for `list` composites. A +`sample_sheet:paired` can be mapped over a `paired` collection input, +extracting each inner pair and producing a `sample_sheet` implicit output. + +(SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED)= +(SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED)= + +
Examples + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED` +:class: note + +Assuming, + +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +- $ C_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + +then + +$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ + +::: + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED` +:class: note + +Assuming, + +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + +

+ +The `paired_or_unpaired` integration rules carry over from `list` to +`sample_sheet`. A flat `sample_sheet` can be mapped over a +`paired_or_unpaired` input via `single_datasets` sub-collection mapping, +and `sample_sheet:paired` can be mapped over `paired_or_unpaired` just as +`list:paired` can. + +(SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED)= +(SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED)= +(SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED)= + +
Examples + +:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +- $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ + +then + +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ + +::: + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +- $ C_AS_MIXED $ is $ \text{CollectionInstance<}sample_sheet:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + +then + +$$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ + +::: + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED` +:class: note + +Assuming, + +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + +

+ +The type matching is asymmetric: a `sample_sheet` output can satisfy a `list` +input because sample sheets carry all the structural information lists have (plus +metadata). However, a `list` output cannot satisfy a `sample_sheet` input because +lists lack the `column_definitions` and per-element `columns` metadata that +sample sheet consumers expect. + +(LIST_NOT_MATCHES_SAMPLE_SHEET)= +(LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED)= +(SAMPLE_SHEET_MATCHES_SAMPLE_SHEET)= + +
Examples + +:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET` +:class: note + +Assuming, + +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + +:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED` +:class: note + +Assuming, + +- $ d_f $, $ d_r $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + +then + +$$tool(i=C)\text{ is invalid}$$ + +::: + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET` +:class: note + +Assuming, + +- $ d_1,...,d_n $ are datasets +- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + +then + +$$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ + +::: + +

diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 7de23fa3778..10e28ba627b 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -971,3 +971,256 @@ it should be able to for consistency. We have focused our time on data structures more likely to be used in actual Galaxy analyses given current and guessed future usage. + +- doc: "## sample_sheet Collections" +- doc: | + The collection type ``sample_sheet`` attaches typed, columnar metadata to each + element of a dataset collection. For mapping and type matching purposes, + ``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool + inputs, matched against ``list`` collection inputs, and composed with inner types + (``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry + is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` + output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata + that plain lists do not. + +- example: + label: SAMPLE_SHEET_MAPPING + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: dataset} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: "d_1", ..., in: "d_n"}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: dataset, ref: d_n}}} + output: o + tests: + workflow_editor: "accepts sample_sheet data -> data connection (maps like list)" + +- example: + label: SAMPLE_SHEET_MATCHES_LIST + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_editor: "accepts sample_sheet -> list connection (canMatch)" + +- doc: | + Sub-collection mapping works the same as for ``list`` composites. A + ``sample_sheet:paired`` can be mapped over a ``paired`` collection input, + extracting each inner pair and producing a ``sample_sheet`` implicit output. + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + "C\\_PAIRED": [paired, {forward: d_f, reverse: d_r}] + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: paired} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + el1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} + output: o + tests: + workflow_editor: "accepts sample_sheet:paired -> paired connection (maps over like list:paired)" + +- example: + label: SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_editor: "accepts sample_sheet:paired -> list:paired connection (canMatch)" + +- doc: | + The ``paired_or_unpaired`` integration rules carry over from ``list`` to + ``sample_sheet``. A flat ``sample_sheet`` can be mapped over a + ``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, + and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as + ``list:paired`` can. + +- example: + label: SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + - "C_AS_UNPAIRED_i = CollectionInstance for i from 1...n" + then: + type: map_over + invocation: + inputs: + i: {type: map_over, collection: C, sub_collection_type: single_datasets} + produces: + o: + type: collection + collection_type: sample_sheet + elements: + i1: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_1}}} + output: o + "...": {type: ellipsis} + in: + type: tool_output_ref + invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} + output: o + tests: + workflow_editor: "accepts sample_sheet -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired", {el1: {forward: d_f, reverse: d_r}}] + C_AS_MIXED: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: equivalence + left: + inputs: + i: {type: map_over, collection: C} + right: + inputs: + i: {type: map_over, collection: C_AS_MIXED} + tests: + workflow_editor: "accepts sample_sheet:paired -> paired_or_unpaired connection" + +- example: + label: SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["sample_sheet:paired_or_unpaired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_editor: "accepts sample_sheet:paired_or_unpaired -> list:paired_or_unpaired connection (canMatch)" + +- doc: | + The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` + input because sample sheets carry all the structural information lists have (plus + metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because + lists lack the ``column_definitions`` and per-element ``columns`` metadata that + sample sheet consumers expect. + +- example: + label: LIST_NOT_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [list, {i1: d_1, ..., in: d_n}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list -> sample_sheet connection (asymmetry)" + +- example: + label: LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED + assumptions: + - datasets: [d_f, d_r] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: ["list:paired", {el1: {forward: d_f, reverse: d_r}}] + then: + type: invalid + invocation: + inputs: + i: {type: collection, ref: C} + is_valid: false + tests: + workflow_editor: "rejects list:paired -> sample_sheet:paired connection (asymmetry)" + +- example: + label: SAMPLE_SHEET_MATCHES_SAMPLE_SHEET + assumptions: + - datasets: ["d_1,...,d_n"] + - tool: + in: {i: "collection"} + out: {o: dataset} + - collections: + C: [sample_sheet, {i1: d_1, ..., in: d_n}] + then: + type: reduction + invocation: + inputs: + i: {type: collection, ref: C} + produces: + o: {type: dataset} + tests: + workflow_editor: "accepts sample_sheet -> sample_sheet connection" From 494722a792f2c361ed2f1ac30fc6950324d4675b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 23 Feb 2026 22:41:28 -0500 Subject: [PATCH 09/14] Add sample_sheet workflow runtime framework tests 7 new framework workflow tests for sample_sheet collection semantics: - SAMPLE_SHEET_MAPPING (cat workflow, map over data) - SAMPLE_SHEET_MATCHES_LIST (cat_collection, canMatch reduction) - SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED (paired_or_unpaired workflow) - SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED (new list_paired workflow) - SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED - SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED - SAMPLE_SHEET_MATCHES_SAMPLE_SHEET (new cat_sample_sheet workflow using __SAMPLE_SHEET_TO_TABULAR__ tool) Make rows optional in galactic_job_json() replacement_collection() (value.get("rows") instead of value["rows"]). Test data still provides rows: {} per element since server-side SampleSheetDatasetCollectionType requires it. Add 3 unit tests for sample_sheet collection creation in test_cwl_util.py. Add framework_test references to collection_semantics.yml for 7 examples. Co-Authored-By: Claude Opus 4.6 --- .../types/collection_semantics.yml | 14 +++ lib/galaxy/tool_util/cwl/util.py | 2 +- .../collection_semantics_cat.gxwf-tests.yml | 31 ++++++ ...on_semantics_cat_collection.gxwf-tests.yml | 25 +++++ ..._semantics_cat_sample_sheet.gxwf-tests.yml | 24 ++++ ...ection_semantics_cat_sample_sheet.gxwf.yml | 13 +++ ...ction_semantics_list_paired.gxwf-tests.yml | 27 +++++ .../collection_semantics_list_paired.gxwf.yml | 13 +++ ...ics_list_paired_or_unpaired.gxwf-tests.yml | 28 +++++ ...emantics_paired_or_unpaired.gxwf-tests.yml | 50 +++++++++ test/unit/tool_util/test_cwl_util.py | 104 ++++++++++++++++++ 11 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 10e28ba627b..b4bfbff4e9e 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -1012,6 +1012,8 @@ invocation: {inputs: {i: {type: dataset, ref: d_n}}} output: o tests: + workflow_runtime: + framework_test: "collection_semantics_cat_4" workflow_editor: "accepts sample_sheet data -> data connection (maps like list)" - example: @@ -1031,6 +1033,8 @@ produces: o: {type: dataset} tests: + workflow_runtime: + framework_test: "collection_semantics_cat_collection_1" workflow_editor: "accepts sample_sheet -> list connection (canMatch)" - doc: | @@ -1063,6 +1067,8 @@ invocation: {inputs: {i: {type: collection, ref: "C\\_PAIRED"}}} output: o tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_5" workflow_editor: "accepts sample_sheet:paired -> paired connection (maps over like list:paired)" - example: @@ -1082,6 +1088,8 @@ produces: o: {type: dataset} tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_0" workflow_editor: "accepts sample_sheet:paired -> list:paired connection (canMatch)" - doc: | @@ -1142,6 +1150,8 @@ inputs: i: {type: map_over, collection: C_AS_MIXED} tests: + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_6" workflow_editor: "accepts sample_sheet:paired -> paired_or_unpaired connection" - example: @@ -1161,6 +1171,8 @@ produces: o: {type: dataset} tests: + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_1" workflow_editor: "accepts sample_sheet:paired_or_unpaired -> list:paired_or_unpaired connection (canMatch)" - doc: | @@ -1223,4 +1235,6 @@ produces: o: {type: dataset} tests: + workflow_runtime: + framework_test: "collection_semantics_cat_sample_sheet_0" workflow_editor: "accepts sample_sheet -> sample_sheet connection" diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index 2a455085aa0..fde6669af57 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -365,7 +365,7 @@ def galactic_job_json( elements = to_elements(value, collection_type) kwds = {} if collection_type.startswith("sample_sheet"): - kwds["rows"] = value["rows"] + kwds["rows"] = value.get("rows") if "name" in value: kwds["name"] = value["name"] collection = collection_create_func(elements, collection_type, **kwds) diff --git a/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml index 8139f677173..8b682a6ab98 100644 --- a/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/collection_semantics_cat.gxwf-tests.yml @@ -108,3 +108,34 @@ asserts: - that: has_text text: "reverse content" + +- doc: | + SAMPLE_SHEET_MAPPING: A sample_sheet mapped over a data input produces + a sample_sheet output collection. + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet + elements: + el1: + asserts: + - that: has_text + text: "element 1" + el2: + asserts: + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml index 086cbb77fa0..36b84a84ceb 100644 --- a/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/collection_semantics_cat_collection.gxwf-tests.yml @@ -19,3 +19,28 @@ text: "element 1" - that: has_text text: "element 2" + +- doc: | + SAMPLE_SHEET_MATCHES_LIST: A sample_sheet collection consumed by a list + collection input via canMatch (reduction). + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "element 1" + - that: has_text + text: "element 2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml new file mode 100644 index 00000000000..dc70eb23d0d --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf-tests.yml @@ -0,0 +1,24 @@ +- doc: | + SAMPLE_SHEET_MATCHES_SAMPLE_SHEET: A sample_sheet consumed by a sample_sheet + collection input (identity match, reduction). + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + asserts: + - that: has_text + text: "el1" + - that: has_text + text: "el2" diff --git a/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml new file mode 100644 index 00000000000..033361052f2 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_cat_sample_sheet.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: sample_sheet +outputs: + wf_output: + outputSource: tool_step/output +steps: + tool_step: + tool_id: __SAMPLE_SHEET_TO_TABULAR__ + in: + input: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml new file mode 100644 index 00000000000..13aca3bc2f7 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf-tests.yml @@ -0,0 +1,27 @@ +- doc: | + SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED: A sample_sheet:paired consumed by a + list:paired collection input via canMatch (reduction). + job: + input1: + class: Collection + collection_type: sample_sheet:paired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "identifier is el1" + - that: has_text + text: "collection_type" diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml new file mode 100644 index 00000000000..d1671ce66e2 --- /dev/null +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: list:paired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_paired_or_list_paired_input + in: + f1: input1 diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml index 4f760a7639f..9a31554ecac 100644 --- a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml @@ -23,3 +23,31 @@ text: "forward content" - that: has_text text: "reverse content" + +- doc: | + SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED: A + sample_sheet:paired_or_unpaired consumed by list:paired_or_unpaired via canMatch. + job: + input1: + class: Collection + collection_type: sample_sheet:paired_or_unpaired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired_or_unpaired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + asserts: + - that: has_text + text: "forward content" + - that: has_text + text: "reverse content" diff --git a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml index 8db03048fac..cb3a59908ff 100644 --- a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml @@ -114,3 +114,53 @@ wf_output: class: Collection collection_type: list + +- doc: | + SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED: A sample_sheet:paired mapped over + a paired_or_unpaired tool input produces a sample_sheet output. + job: + input1: + class: Collection + collection_type: sample_sheet:paired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet + +- doc: | + SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED: A sample_sheet:paired + mapped over paired_or_unpaired (equiv to sample_sheet:paired_or_unpaired mapping). + job: + input1: + class: Collection + collection_type: sample_sheet:paired + rows: + el1: [] + elements: + - identifier: el1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "forward content" + - identifier: reverse + class: File + contents: "reverse content" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet diff --git a/test/unit/tool_util/test_cwl_util.py b/test/unit/tool_util/test_cwl_util.py index 57881e15981..84c513c76bb 100644 --- a/test/unit/tool_util/test_cwl_util.py +++ b/test/unit/tool_util/test_cwl_util.py @@ -151,3 +151,107 @@ def test_galactic_job_json_collection_element_filetype(): for target in captured_targets: assert isinstance(target, FileLiteralTarget) assert target.properties.get("filetype") == "fastqsanger" + + +def test_galactic_job_json_sample_sheet_collection_without_rows(): + """sample_sheet collection works without rows metadata.""" + created_collections = [] + + def collection_create_func(element_identifiers, collection_type, rows=None, name=None): + created_collections.append( + {"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name} + ) + return {"id": f"collection_{collection_type}"} + + job = { + "input1": { + "class": "Collection", + "collection_type": "sample_sheet", + "elements": [ + {"identifier": "el1", "class": "File", "contents": "element 1"}, + {"identifier": "el2", "class": "File", "contents": "element 2"}, + ], + } + } + result_job, datasets = galactic_job_json( + job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow" + ) + assert len(created_collections) == 1 + coll = created_collections[0] + assert coll["collection_type"] == "sample_sheet" + assert coll["rows"] is None + assert len(coll["element_identifiers"]) == 2 + assert coll["element_identifiers"][0]["name"] == "el1" + assert coll["element_identifiers"][1]["name"] == "el2" + + +def test_galactic_job_json_sample_sheet_collection_with_rows(): + """sample_sheet collection passes rows metadata through.""" + created_collections = [] + + def collection_create_func(element_identifiers, collection_type, rows=None, name=None): + created_collections.append( + {"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name} + ) + return {"id": f"collection_{collection_type}"} + + job = { + "input1": { + "class": "Collection", + "collection_type": "sample_sheet", + "elements": [ + {"identifier": "el1", "class": "File", "contents": "element 1"}, + {"identifier": "el2", "class": "File", "contents": "element 2"}, + ], + "rows": {"el1": {"condition": "treatment"}, "el2": {"condition": "control"}}, + } + } + result_job, datasets = galactic_job_json( + job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow" + ) + assert len(created_collections) == 1 + coll = created_collections[0] + assert coll["collection_type"] == "sample_sheet" + assert coll["rows"] == {"el1": {"condition": "treatment"}, "el2": {"condition": "control"}} + + +def test_galactic_job_json_sample_sheet_paired_collection(): + """sample_sheet:paired creates nested collection with paired sub-elements.""" + created_collections = [] + + def collection_create_func(element_identifiers, collection_type, rows=None, name=None): + created_collections.append( + {"element_identifiers": element_identifiers, "collection_type": collection_type, "rows": rows, "name": name} + ) + return {"id": f"collection_{collection_type}"} + + job = { + "input1": { + "class": "Collection", + "collection_type": "sample_sheet:paired", + "elements": [ + { + "identifier": "sample1", + "class": "Collection", + "type": "paired", + "elements": [ + {"identifier": "forward", "class": "File", "contents": "fwd reads"}, + {"identifier": "reverse", "class": "File", "contents": "rev reads"}, + ], + } + ], + } + } + result_job, datasets = galactic_job_json( + job, ".", _mock_upload_func, collection_create_func, tool_or_workflow="workflow" + ) + assert len(created_collections) == 1 + coll = created_collections[0] + assert coll["collection_type"] == "sample_sheet:paired" + assert coll["rows"] is None + assert len(coll["element_identifiers"]) == 1 + el = coll["element_identifiers"][0] + assert el["name"] == "sample1" + assert el["src"] == "new_collection" + assert el["collection_type"] == "paired" + assert len(el["element_identifiers"]) == 2 From fffdb4822d1a90b7e6e394ea295e7da4621141f3 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 25 Feb 2026 11:22:28 -0500 Subject: [PATCH 10/14] Add collection semantics test refs and workflow tests for flat over paired_or_unpaired Update collection_semantics.yml test references and add workflow framework tests for flat list and flat sample_sheet mapped over paired_or_unpaired. Fix mislabeled test #4 doc string. Co-Authored-By: Claude Opus 4.6 --- .../types/collection_semantics.yml | 6 ++- ...emantics_paired_or_unpaired.gxwf-tests.yml | 45 ++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index b4bfbff4e9e..9de9c8406ce 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -882,7 +882,7 @@ tool_runtime: api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_list" workflow_runtime: - framework_test: "collection_semantics_paired_or_unpaired_4" + framework_test: "collection_semantics_paired_or_unpaired_7" workflow_editor: "accepts list -> paired_or_unpaired connection" - doc: | @@ -1129,6 +1129,10 @@ invocation: {inputs: {i: {type: collection, ref: C_AS_UNPAIRED_n}}} output: o tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_paired_or_unpaired_with_sample_sheet" + workflow_runtime: + framework_test: "collection_semantics_paired_or_unpaired_8" workflow_editor: "accepts sample_sheet -> paired_or_unpaired connection" - example: diff --git a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml index cb3a59908ff..c7658405026 100644 --- a/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/collection_semantics_paired_or_unpaired.gxwf-tests.yml @@ -93,8 +93,8 @@ collection_type: list:list - doc: | - MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired collection mapped over - a paired_or_unpaired tool input produces a list output collection. + MAPPING_LIST_PAIRED_OR_UNPAIRED_OVER_PAIRED_OR_UNPAIRED: A list:paired_or_unpaired + collection mapped over a paired_or_unpaired tool input produces a list output collection. job: input1: class: Collection @@ -164,3 +164,44 @@ wf_output: class: Collection collection_type: sample_sheet + +- doc: | + MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED: A flat list mapped over a + paired_or_unpaired tool input via single_datasets produces a list output. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: list + +- doc: | + SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED: A flat sample_sheet mapped over + a paired_or_unpaired tool input via single_datasets produces a sample_sheet output. + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet From 9d49f7f89bf3dd70edd38582226467bbaf9bec10 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Mar 2026 14:37:48 -0400 Subject: [PATCH 11/14] Fix LaTeX rendering errors in collection semantics docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix underscore handling in semantics.py that caused "'_' allowed only in math mode" errors in rendered HTML. Issues: CollectionDefinition used bare escaped underscores outside \text{}, expression_to_latex had double-replacement bugs (unpaired→\text{un\text{paired}}), and sample_sheet was unhandled. Co-Authored-By: Claude Opus 4.6 --- doc/source/dev/collection_semantics.md | 672 +++++++++++------- .../dataset_collections/types/semantics.py | 34 +- 2 files changed, 445 insertions(+), 261 deletions(-) diff --git a/doc/source/dev/collection_semantics.md b/doc/source/dev/collection_semantics.md index 1e3ad18695d..55ef137a190 100644 --- a/doc/source/dev/collection_semantics.md +++ b/doc/source/dev/collection_semantics.md @@ -28,21 +28,22 @@ mapped over. Each element of the implicit collections correspond to their own jo Galaxy very naturally and intuitively parallelizes jobs without extra work from the user and without any knowledge of the tool. + (BASIC_MAPPING_PAIRED)= (BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED)= (BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED)= (BASIC_MAPPING_LIST)= -
Examples -:::{admonition} Example: `BASIC_MAPPING_PAIRED` +:::{admonition} Example: `BASIC_MAPPING_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -50,14 +51,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired},\l ::: -:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED` + + +:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -65,14 +69,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or ::: -:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED` + + +:::{admonition} Example: `BASIC_MAPPING_PAIRED_OR_UNPAIRED_UNPAIRED` :class: note Assuming, -- $ d_u $ is a dataset -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ unpaired }=d_u \right\}\text{>} $ +* $ d_u $ is a dataset +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ unpaired }=d_u \right\}\text{>} $ + then @@ -80,14 +87,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{paired\_or ::: -:::{admonition} Example: `BASIC_MAPPING_LIST` + + +:::{admonition} Example: `BASIC_MAPPING_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -95,24 +105,29 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list},\lef ::: + +

+ + The above description of mapping over inputs works naturally and as expected for nested collections. + (NESTED_LIST_MAPPING)= (BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `NESTED_LIST_MAPPING` +:::{admonition} Example: `NESTED_LIST_MAPPING` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ + then @@ -120,14 +135,17 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\tex ::: -:::{admonition} Example: `BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `BASIC_MAPPING_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -135,24 +153,30 @@ $$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{list}:\tex ::: + +

+ + + For tools with multiple data inputs, the tool can be executed with individual datasets for the non-mapped over input and each tool execution will just be executed with that dataset. The dataset not mapped over serves as the input for each execution. -(BASIC_MAPPING_INCLUDING_SINGLE_DATASET)= +(BASIC_MAPPING_INCLUDING_SINGLE_DATASET)=
Examples -:::{admonition} Example: `BASIC_MAPPING_INCLUDING_SINGLE_DATASET` +:::{admonition} Example: `BASIC_MAPPING_INCLUDING_SINGLE_DATASET` :class: note Assuming, -- $ d_1,...,d_n $, $ d_o $ are datasets -- $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $, $ d_o $ are datasets +* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -160,8 +184,12 @@ $$tool(i=\text{mapOver}(C), i2=d_o) \mapsto \left\{o: \text{collection}<\text{li ::: + +

+ + If a tool consumes two input datasets and produces one output dataset, you can map two collections with identical structure (same element identifiers in the same order) over the respective inputs and the result is an implicit collection with the same structure @@ -178,19 +206,20 @@ of map over operations on tools - the results will all continue to match and wor very naturally - again without extra work by the user and without extra knowledge by the tool author. -(BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE)= +(BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE)=
Examples -:::{admonition} Example: `BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE` +:::{admonition} Example: `BASIC_MAPPING_TWO_INPUTS_WITH_IDENTICAL_STRUCTURE` :class: note Assuming, -- $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C1 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ -- $ C2 $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ +* $ d1_1,...,d1_n $, $ d2_1,...,d2_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }, i2: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C1 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d1_1, ..., \text{ in }=d1_n \right\}\text{>} $ +* $ C2 $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d2_1, ..., \text{ in }=d2_n \right\}\text{>} $ + then @@ -198,33 +227,38 @@ $$tool(i=\text{mapOver}(C1), i2=\text{mapOver}(C2)) \mapsto \left\{o: \text{coll ::: + +

+ + ## Reduction Not all tool executions result in implicit collections and mapping -over inputs. Tool inputs of `type` `data_collection` can consume +over inputs. Tool inputs of ``type`` ``data_collection`` can consume collections directly and do not necessarily result in mapping over. Tools that consume collections and output datasets effectively reduce the dimension of the Galaxy data structure. When used at runtime this is often referred to as a "reduction" in the code. + (COLLECTION_INPUT_PAIRED)= (COLLECTION_INPUT_LIST)= (COLLECTION_INPUT_PAIRED_OR_UNPAIRED)= (COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `COLLECTION_INPUT_PAIRED` +:::{admonition} Example: `COLLECTION_INPUT_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -232,14 +266,17 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ el1 }=d_1, ..., \text{ eln }=d_n \right\}\text{>} $ + then @@ -247,14 +284,17 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -262,14 +302,17 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -277,26 +320,31 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-For nested collections where each rank is a `list` or a `paired` collection, + + +For nested collections where each rank is a ``list`` or a ``paired`` collection, then collection inputs must match every part of the collection type input definition. + (COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS)= (COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST)= (COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= (COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED)= -
Examples -:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` +:::{admonition} Example: `COLLECTION_INPUT_LIST_NOT_CONSUMES_PAIRS` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -304,14 +352,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST` + + +:::{admonition} Example: `COLLECTION_INPUT_PAIRED_NOT_CONSUMES_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -319,14 +370,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -334,14 +388,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` + + +:::{admonition} Example: `COLLECTION_INPUT_LIST_PAIRED_OR_NOT_PAIRED_NOT_CONSUMES_PAIRED_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired:paired,\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired}:\text{paired},\left\{ \text{ forward }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}, \text{ reverse }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -349,24 +406,29 @@ $$tool(i=C)\text{ is invalid}$$ ::: + +

-In addition to explicit collection inputs, tool inputs of `type` `data` -where `multiple="true"` can consume lists directly. This is likewise a + + +In addition to explicit collection inputs, tool inputs of ``type`` ``data`` +where ``multiple="true"`` can consume lists directly. This is likewise a "reduction" and does not result in implicit collection creation. -(LIST_REDUCTION)= +(LIST_REDUCTION)=
Examples -:::{admonition} Example: `LIST_REDUCTION` +:::{admonition} Example: `LIST_REDUCTION` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -374,24 +436,29 @@ $$tool(i=C) == tool(i=[d_1,...,d_n])$$ ::: + +

-Paired collections cannot be reduced this way. `paired` is not meant + + +Paired collections cannot be reduced this way. ``paired`` is not meant to represent a list/array/vector data structure - it is more like a tuple. + (PAIRED_REDUCTION_INVALID)= (PAIRED_OR_UNPAIRED_REDUCTION_INVALID)= -
Examples -:::{admonition} Example: `PAIRED_REDUCTION_INVALID` +:::{admonition} Example: `PAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -399,14 +466,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `PAIRED_OR_UNPAIRED_REDUCTION_INVALID` + + +:::{admonition} Example: `PAIRED_OR_UNPAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -414,25 +484,30 @@ $$tool(i=C)\text{ is invalid}$$ ::: + +

+ + ## Sub-collection Mapping ![](https://planemo.readthedocs.io/en/master/_images/subcollection_mapping_identifiers.svg) -(MAPPING_LIST_PAIRED_OVER_PAIRED)= +(MAPPING_LIST_PAIRED_OVER_PAIRED)=
Examples -:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED` +:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -440,26 +515,31 @@ $$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection} ::: + +

+ + The natural extension of multiple data input parameters consuming list collections as described -above when discussing reductions is that nested lists of lists (`list:list`) can be mapped +above when discussing reductions is that nested lists of lists (``list:list``) can be mapped over a multiple data input parameter. Each nested list will be reduced by this operation but the results will be mapped over. The result will be a list with the same structure as the outer list of the input collection. -(NESTED_LIST_REDUCTION)= +(NESTED_LIST_REDUCTION)=
Examples -:::{admonition} Example: `NESTED_LIST_REDUCTION` +:::{admonition} Example: `NESTED_LIST_REDUCTION` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ + then @@ -467,25 +547,30 @@ $$tool(i=\text{mapOver}(C, '\text{list}')) \mapsto \left\{o: \text{collection}<\ ::: + +

+ + Just as a paired collection won't be reduced by a multiple data input, any sort of nested collection ending in a paired collection cannot be mapped over such an input. So a multiple -data input parameter cannot be mapped over by a list of pairs (`list:paired`) for instance. +data input parameter cannot be mapped over by a list of pairs (``list:paired``) for instance. + (LIST_PAIRED_REDUCTION_INVALID)= (LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID)= -
Examples -:::{admonition} Example: `LIST_PAIRED_REDUCTION_INVALID` +:::{admonition} Example: `LIST_PAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -493,14 +578,17 @@ $$tool(i=\text{mapOver}(C, '\text{paired}'))\text{ is invalid}$$ ::: -:::{admonition} Example: `LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID` + + +:::{admonition} Example: `LIST_PAIRED_OR_UNPAIRED_REDUCTION_INVALID` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -508,32 +596,37 @@ $$tool(i=\text{mapOver}(C, '\text{paired\_or\_unpaired}'))\text{ is invalid}$$ ::: + +

+ + ## paired_or_unpaired Collections -The collection type `paired_or_unpaired` is meant to serve as a stand-in for -an entity that can be either a single dataset or what is effectively a `paired` +The collection type ``paired_or_unpaired`` is meant to serve as a stand-in for +an entity that can be either a single dataset or what is effectively a ``paired`` dataset collection. These collections either have one element with identifier -`unpaired` or two elements with identifiers `forward` and `reverse`. +``unpaired`` or two elements with identifiers ``forward`` and ``reverse``. + +Tools can declare a data_collection input with collection type ``paired_or_unpaired`` +and that input will consume either an explicit ``paired_or_unpaired`` collection +normally or can consume a ``paired`` input. -Tools can declare a data_collection input with collection type `paired_or_unpaired` -and that input will consume either an explicit `paired_or_unpaired` collection -normally or can consume a `paired` input. (PAIRED_OR_UNPAIRED_CONSUMES_PAIRED)= -
Examples -:::{admonition} Example: `PAIRED_OR_UNPAIRED_CONSUMES_PAIRED` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_CONSUMES_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ -- $ C_AS_MIXED = CollectionInstance<\text{paired_or_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ C_AS_MIXED = CollectionInstance<\text{paired\_or\_unpaired}, \left\{\text{forward}: d_f, \text{reverse}: d_r\right\}> $ + then @@ -541,26 +634,32 @@ $$tool(i=C) == tool(i=C_AS_MIXED)$$ ::: + +

-The inverse of this doesn't work intentionally. In some ways a `paired` collection -acts as a `paired_or_unpaired` collection but a `paired_or_unpaired` is not a paired -collection. This makes a lot of sense in terms of tools - a tool consuming a `paired` -dataset expects to find both a `forward` and `reverse` element but these may not exist -in `paired_or_unpaired` collection. + + + +The inverse of this doesn't work intentionally. In some ways a ``paired`` collection +acts as a ``paired_or_unpaired`` collection but a ``paired_or_unpaired`` is not a paired +collection. This makes a lot of sense in terms of tools - a tool consuming a ``paired`` +dataset expects to find both a ``forward`` and ``reverse`` element but these may not exist +in ``paired_or_unpaired`` collection. + (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED)= -
Examples -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED` +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}paired_or_unpaired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{paired\_or\_unpaired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then @@ -568,28 +667,34 @@ $$tool(i=C)\text{ is invalid}$$ ::: + +

-The same logic holds for mapping, lists of paired datasets (`list:paired`) can be mapped over these -`paired_or_unpaired` inputs and mixed lists of pairs (`list:paired_or_unpaired`) cannot -be mapped over a `paired` input. Following the same logic, `list:paired_or_unpaired` cannot -be mapped over a `list` input or multiple data input. + + + +The same logic holds for mapping, lists of paired datasets (``list:paired``) can be mapped over these +``paired_or_unpaired`` inputs and mixed lists of pairs (``list:paired_or_unpaired``) cannot +be mapped over a ``paired`` input. Following the same logic, ``list:paired_or_unpaired`` cannot +be mapped over a ``list`` input or multiple data input. + (MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING)= (PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING)= -
Examples -:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -597,14 +702,17 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING` + + +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -612,14 +720,17 @@ $$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: -:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING` + + +:::{admonition} Example: `PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_LIST_WHEN_MAPPING` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired_or_unpaired,\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ el }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -627,25 +738,30 @@ $$tool(i=\text{mapOver}(C))\text{ is invalid}$$ ::: + +

-This logic extends naturally into higher dimensional collections. A `list:list:paired` -can be mapped over either a `paired_or_unpaired` input to produce a nested list (`list:list`) -or a `list:paired_or_unpaired` input to produce a flat list (`list`). + + +This logic extends naturally into higher dimensional collections. A ``list:list:paired`` +can be mapped over either a ``paired_or_unpaired`` input to produce a nested list (``list:list``) +or a ``list:paired_or_unpaired`` input to produce a flat list (``list``). + (MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list:paired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ -- $ C_AS_MIXED $ is $ \text{CollectionInstance<}list:list:paired_or_unpaired,\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{list}:\text{list}:\text{paired\_or\_unpaired},\left\{ \text{ o1 }=\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\} \right\}\text{>} $ + then @@ -653,25 +769,31 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: + +

-In order for `paired_or_unpaired` collections to also act as a single dataset, + + + +In order for ``paired_or_unpaired`` collections to also act as a single dataset, a flat list can be mapped over a such an input with a special sub collection mapping type of 'single_datasets'. -(MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED)= +(MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED)=
Examples -:::{admonition} Example: `MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -- $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $ + then @@ -679,29 +801,34 @@ $$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{c ::: + +

+ + This treatment of lists without pairing extends to nested structures naturally. -For instance, a list of list of datasets (`list:list`) can be mapped over a -`paired_or_unpaired` input to produce a nested list of lists (`list:list`) +For instance, a list of list of datasets (``list:list``) can be mapped over a +``paired_or_unpaired`` input to produce a nested list of lists (``list:list``) with a structure matching the input. Likewise, the nested list can be mapped over -a `list:paired_or_unpaired` input to produce a flat list with the same structure +a ``list:paired_or_unpaired`` input to produce a flat list with the same structure as the outer list of the input. + (MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED)= (MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ -- $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=dj\right\}> for each \text{dataset} dj in C $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_UNPAIRED_j = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=dj\right\}> for each \text{dataset} dj in C $ + then @@ -709,15 +836,18 @@ $$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{c ::: -:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:list,\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ -- $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired_or_unpaired} via \text{single\_\text{dataset}s} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{list},\left\{ \text{ o1 }=\left\{ \text{ inner }=d_1 \right\}, ..., \text{ on }=\left\{ \text{ inner }=d_n \right\} \right\}\text{>} $ +* $ C_AS_LPU_i = \text{inner} \text{list} of C at position i, treated as \text{list}:\text{paired\_or\_unpaired} via \text{single\_datasets} $ + then @@ -725,8 +855,12 @@ $$tool(i=\text{mapOver}(C, '\text{list}:\text{paired\_or\_unpaired}')) \mapsto \ ::: + +

+ + Due only to implementation time, the special casing of allowing paired_or_unpaired act as both datasets and paired collections only works when it is the deepest collection type. So while list:paired can be consumed by a list:paired_or_unpaired @@ -735,45 +869,50 @@ it should be able to for consistency. We have focused our time on data structure more likely to be used in actual Galaxy analyses given current and guessed future usage. + ## sample_sheet Collections -The collection type `sample_sheet` attaches typed, columnar metadata to each +The collection type ``sample_sheet`` attaches typed, columnar metadata to each element of a dataset collection. For mapping and type matching purposes, -`sample_sheet` behaves identically to `list` - it can be mapped over tool -inputs, matched against `list` collection inputs, and composed with inner types -(`sample_sheet:paired`, `sample_sheet:paired_or_unpaired`). The key asymmetry -is that while a `sample_sheet` output can satisfy a `list` input, a `list` -output cannot satisfy a `sample_sheet` input - sample sheets carry metadata +``sample_sheet`` behaves identically to ``list`` - it can be mapped over tool +inputs, matched against ``list`` collection inputs, and composed with inner types +(``sample_sheet:paired``, ``sample_sheet:paired_or_unpaired``). The key asymmetry +is that while a ``sample_sheet`` output can satisfy a ``list`` input, a ``list`` +output cannot satisfy a ``sample_sheet`` input - sample sheets carry metadata that plain lists do not. + (SAMPLE_SHEET_MAPPING)= (SAMPLE_SHEET_MATCHES_LIST)= -
Examples -:::{admonition} Example: `SAMPLE_SHEET_MAPPING` +:::{admonition} Example: `SAMPLE_SHEET_MAPPING` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ dataset }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then -$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C)) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=d_1)[o],...,\text{in}=tool(i=d_n)[o]\right\}>\right\}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST` + + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_LIST` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -781,41 +920,49 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-Sub-collection mapping works the same as for `list` composites. A -`sample_sheet:paired` can be mapped over a `paired` collection input, -extracting each inner pair and producing a `sample_sheet` implicit output. + + +Sub-collection mapping works the same as for ``list`` composites. A +``sample_sheet:paired`` can be mapped over a ``paired`` collection input, +extracting each inner pair and producing a ``sample_sheet`` implicit output. + (SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED)= (SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED)= -
Examples -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED` +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_PAIRED $ is $ \text{CollectionInstance<}paired,\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C\_PAIRED $ is $ \text{CollectionInstance<}\text{paired},\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\}\text{>} $ + then -$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{paired}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{el1}=tool(i=C\_PAIRED)[o]\right\}>\right\}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED` + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MATCHES_LIST_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -823,45 +970,53 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-The `paired_or_unpaired` integration rules carry over from `list` to -`sample_sheet`. A flat `sample_sheet` can be mapped over a -`paired_or_unpaired` input via `single_datasets` sub-collection mapping, -and `sample_sheet:paired` can be mapped over `paired_or_unpaired` just as -`list:paired` can. + + +The ``paired_or_unpaired`` integration rules carry over from ``list`` to +``sample_sheet``. A flat ``sample_sheet`` can be mapped over a +``paired_or_unpaired`` input via ``single_datasets`` sub-collection mapping, +and ``sample_sheet:paired`` can be mapped over ``paired_or_unpaired`` just as +``list:paired`` can. + (SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED)= (SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED)= (SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED)= -
Examples -:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED` +:::{admonition} Example: `SAMPLE_SHEET_MAPPING_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ -- $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired_or_unpaired},\left\{\text{un\text{paired}}=di\right\}> for i from 1...n $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ C_AS_UNPAIRED_i = CollectionInstance<\text{paired\_or\_unpaired},\left\{\text{unpaired}=di\right\}> for i from 1...n $ + then -$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ +$$tool(i=\text{mapOver}(C, '\text{single\_datasets}')) \mapsto \left\{o: \text{collection}<\text{sample\_sheet},\left\{\text{i1}=tool(i=C_AS_UNPAIRED_1)[o],...,\text{in}=tool(i=C_AS_UNPAIRED_n)[o]\right\}>\right\}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_MAPPING_OVER_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ -- $ C_AS_MIXED $ is $ \text{CollectionInstance<}sample_sheet:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ C_AS_MIXED $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -869,14 +1024,17 @@ $$tool(i=\text{mapOver}(C)) == tool(i=\text{mapOver}(C_AS_MIXED))$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED` + + +:::{admonition} Example: `SAMPLE_SHEET_PAIRED_OR_UNPAIRED_MATCHES_LIST_PAIRED_OR_UNPAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet:paired_or_unpaired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet}:\text{paired\_or\_unpaired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -884,28 +1042,33 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

-The type matching is asymmetric: a `sample_sheet` output can satisfy a `list` + + +The type matching is asymmetric: a ``sample_sheet`` output can satisfy a ``list`` input because sample sheets carry all the structural information lists have (plus -metadata). However, a `list` output cannot satisfy a `sample_sheet` input because -lists lack the `column_definitions` and per-element `columns` metadata that +metadata). However, a ``list`` output cannot satisfy a ``sample_sheet`` input because +lists lack the ``column_definitions`` and per-element ``columns`` metadata that sample sheet consumers expect. + (LIST_NOT_MATCHES_SAMPLE_SHEET)= (LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED)= (SAMPLE_SHEET_MATCHES_SAMPLE_SHEET)= -
Examples -:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET` +:::{admonition} Example: `LIST_NOT_MATCHES_SAMPLE_SHEET` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -913,14 +1076,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED` + + +:::{admonition} Example: `LIST_PAIRED_NOT_MATCHES_SAMPLE_SHEET_PAIRED` :class: note Assuming, -- $ d_f $, $ d_r $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}list:paired,\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ +* $ d_f $, $ d_r $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{list}:\text{paired},\left\{ \text{ el1 }=\left\{ \text{ forward }=d_f, \text{ reverse }=d_r \right\} \right\}\text{>} $ + then @@ -928,14 +1094,17 @@ $$tool(i=C)\text{ is invalid}$$ ::: -:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET` + + +:::{admonition} Example: `SAMPLE_SHEET_MATCHES_SAMPLE_SHEET` :class: note Assuming, -- $ d_1,...,d_n $ are datasets -- $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ -- $ C $ is $ \text{CollectionInstance<}sample_sheet,\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ +* $ d_1,...,d_n $ are datasets +* $ tool \text{ is } (i: \text{ collection }) \Rightarrow \{ o: \text{ dataset } \} $ +* $ C $ is $ \text{CollectionInstance<}\text{sample\_sheet},\left\{ \text{ i1 }=d_1, ..., \text{ in }=d_n \right\}\text{>} $ + then @@ -943,4 +1112,9 @@ $$tool(i=C) \rightarrow \left\{o: \text{dataset}\right\}$$ ::: + +

+ + + diff --git a/lib/galaxy/model/dataset_collections/types/semantics.py b/lib/galaxy/model/dataset_collections/types/semantics.py index 42f947b8b20..5e42b02894e 100644 --- a/lib/galaxy/model/dataset_collections/types/semantics.py +++ b/lib/galaxy/model/dataset_collections/types/semantics.py @@ -72,8 +72,8 @@ class ToolDefinition(BaseModel): outputs: dict[str, str] = Field(alias="out") def as_latex(self) -> str: - inputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.inputs.items()]) - outputs = ", ".join([f"{k}: \\text{{ {v} }}" for (k, v) in self.outputs.items()]) + inputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.inputs.items()]) + outputs = ", ".join([f"{k}: \\text{{ {_escape_underscores(v)} }}" for (k, v) in self.outputs.items()]) return f"tool \\text{{ is }} ({inputs}) \\Rightarrow \\{{ {outputs} \\}}" @@ -104,10 +104,8 @@ class CollectionDefinition(NamedTuple): elements: dict[str, Any] def as_latex(self) -> str: - collection_type = self.collection_type.replace("_", "\\_") - return ( - f"\\text{{CollectionInstance<}}{collection_type},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" - ) + ct = _latex_collection_type(self.collection_type) + return f"\\text{{CollectionInstance<}}{ct},{_assumption_elements_to_latex(self.elements)}\\text{{>}}" class CollectionDeclarations(BaseModel): @@ -120,11 +118,14 @@ Expression = Union[str, DatasetsDeclaration, ToolDeclaration, CollectionDeclarat # --- Structured Then Expression Models --- +def _escape_underscores(text: str) -> str: + return text.replace("_", "\\_") + + def _latex_type_word(word: str) -> str: - if word == "paired_or_unpaired": - return "\\text{paired\\_or\\_unpaired}" - if word == "single_datasets": - return "\\text{single\\_datasets}" + if "_" in word: + escaped = word.replace("_", "\\_") + return "\\text{" + escaped + "}" return "\\text{" + word + "}" @@ -313,6 +314,9 @@ YAMLRootModel = RootModel[list[Union[DocEntry, ExampleEntry]]] WORDS_TO_TEXTIFY = ["list", "forward", "reverse", "mapOver", "collection", "dataset", "inner"] _POU_PLACEHOLDER = "\x00POU\x00" +_UNPAIRED_PLACEHOLDER = "\x00UNPAIRED\x00" +_SINGLE_DATASETS_PLACEHOLDER = "\x00SINGLEDATASETS\x00" +_SAMPLE_SHEET_PLACEHOLDER = "\x00SAMPLESHEET\x00" def expression_to_latex(expression: str, wrap: bool = True): @@ -320,13 +324,19 @@ def expression_to_latex(expression: str, wrap: bool = True): expression = expression.replace("~>", "\\mapsto") expression = expression.replace("{", "\\left\\{") expression = expression.replace("}", "\\right\\}") - expression = expression.replace("single_datasets", "\\text{single\\_datasets}") + # Replace multi-word identifiers with placeholders before WORDS_TO_TEXTIFY + # to prevent double-replacement (e.g. "dataset" inside "single_datasets") + expression = expression.replace("single_datasets", _SINGLE_DATASETS_PLACEHOLDER) + expression = expression.replace("sample_sheet", _SAMPLE_SHEET_PLACEHOLDER) expression = expression.replace("paired_or_unpaired", _POU_PLACEHOLDER) - expression = expression.replace("unpaired", "\\text{unpaired}") + expression = expression.replace("unpaired", _UNPAIRED_PLACEHOLDER) expression = expression.replace("paired", "\\text{paired}") + expression = expression.replace(_UNPAIRED_PLACEHOLDER, "\\text{unpaired}") expression = expression.replace(_POU_PLACEHOLDER, "\\text{paired\\_or\\_unpaired}") for word in WORDS_TO_TEXTIFY: expression = expression.replace(word, "\\text{" + word + "}") + expression = expression.replace(_SINGLE_DATASETS_PLACEHOLDER, "\\text{single\\_datasets}") + expression = expression.replace(_SAMPLE_SHEET_PLACEHOLDER, "\\text{sample\\_sheet}") if wrap: return f"$ {expression} $" else: From 7d918fe9e00f4a19ba2eda9bd54d293f0cd4123d Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 17 Mar 2026 16:38:18 -0400 Subject: [PATCH 12/14] Fix compound :paired_or_unpaired handling in has_subcollections_of_type. has_subcollections_of_type and effective_collection_type only handled paired_or_unpaired as an exact match, missing compound suffix cases like list:paired_or_unpaired. This affected both offline validation and backend runtime (query.py:can_map_over). Now matches TS editor logic for collection_semantics.yml examples like MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED. Also fixes effective_collection_type for non-:paired collections over paired_or_unpaired (e.g. list over paired_or_unpaired now correctly returns full type instead of empty string). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../dataset_collections/type_description.py | 43 +++++++++++++-- .../types/collection_semantics.yml | 4 ++ lib/galaxy_test/api/test_tool_execute.py | 13 +++++ ...ics_list_paired_or_unpaired.gxwf-tests.yml | 31 +++++++++++ .../test_type_descriptions.py | 53 +++++++++++++++++++ 5 files changed, 139 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index af392b4b796..3eba55ea4f0 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -71,11 +71,33 @@ class CollectionTypeDescription: if subcollection_type == "single_datasets": return self.collection_type - # When the subcollection type is paired_or_unpaired but the collection - # actually ends with :paired (a subtype match), strip the actual suffix - # rather than the longer paired_or_unpaired string. - if subcollection_type == "paired_or_unpaired" and self.collection_type.endswith(":paired"): - return self.collection_type[: -len(":paired")] + normalized = _normalize_collection_type(self.collection_type) + normalized_sub = _normalize_collection_type(subcollection_type) + + if subcollection_type == "paired_or_unpaired": + if self.collection_type.endswith(":paired"): + # paired_or_unpaired consumes the :paired suffix + return self.collection_type[: -len(":paired")] + elif normalized.endswith("list"): + # paired_or_unpaired acts like single_datasets for collections + # whose innermost type is list (each element wrapped as unpaired) + return self.collection_type + else: + # strip last rank (paired_or_unpaired consumes it) + return self.collection_type[: self.collection_type.rfind(":")] + + if normalized_sub.endswith(":paired_or_unpaired"): + # Compound :paired_or_unpaired suffix — iterative peel-off matching + # TS effectiveMapOver logic. Strip ranks from both sides, then + # optionally strip one more if :paired was consumed. + current = self.collection_type + current_other = subcollection_type + while ":" in current_other: + current_other = current_other[: current_other.rfind(":")] + current = current[: current.rfind(":")] + if normalized.endswith(":paired"): + current = current[: current.rfind(":")] + return current return self.collection_type[: -(len(subcollection_type) + 1)] @@ -100,6 +122,17 @@ class CollectionTypeDescription: # this can be thought of as a subcollection of anything except a pair # since it would match a pair exactly return collection_type != "paired" + if other_collection_type.endswith(":paired_or_unpaired"): + # Compound :paired_or_unpaired suffix — e.g. list:list can map over + # list:paired_or_unpaired. Strip the :paired_or_unpaired to get the + # required higher ranks, optionally strip :paired from self (since + # paired_or_unpaired consumes paired), then check alignment. + higher_ranks_required = other_collection_type[: other_collection_type.rfind(":")] + if collection_type.endswith(":paired"): + higher_ranks = collection_type[: collection_type.rfind(":")] + else: + higher_ranks = collection_type + return higher_ranks.endswith(higher_ranks_required) and higher_ranks != higher_ranks_required if other_collection_type == "single_datasets": # effectively any collection has unpaired subcollections return True diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 9de9c8406ce..ab3cbc2b585 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -961,6 +961,10 @@ invocation: {inputs: {i: {type: collection, ref: C_AS_LPU_n}}} output: o tests: + tool_runtime: + api_test: "test_tool_execute.py::test_map_over_list_paired_or_unpaired_with_list_of_lists" + workflow_runtime: + framework_test: "collection_semantics_list_paired_or_unpaired_2" workflow_editor: "accepts list:list -> list:paired_or_unpaired connection" - doc: | diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py index bb3c7bb5c11..4b1f7acfff1 100644 --- a/lib/galaxy_test/api/test_tool_execute.py +++ b/lib/galaxy_test/api/test_tool_execute.py @@ -571,6 +571,19 @@ def test_map_over_paired_or_unpaired_with_list_of_lists(target_history: TargetHi assert len(as_dict_0["object"]["elements"]) == 3 +@requires_tool_id("collection_list_paired_or_unpaired") +def test_map_over_list_paired_or_unpaired_with_list_of_lists( + target_history: TargetHistory, required_tool: RequiredTool +): + hdca = target_history.with_example_list_of_lists() + execute = required_tool.execute().with_inputs( + {"f1": {"batch": True, "values": [{"map_over_type": "list", **hdca.src_dict}]}} + ) + execute.assert_has_n_jobs(1).assert_creates_n_implicit_collections(1) + output_collection = execute.assert_creates_implicit_collection(0) + assert output_collection.details["collection_type"] == "list" + + @requires_tool_id("collection_paired_test") def test_simple_subcollection_mapping( target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs diff --git a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml index 9a31554ecac..0b8c3d26751 100644 --- a/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/collection_semantics_list_paired_or_unpaired.gxwf-tests.yml @@ -51,3 +51,34 @@ text: "forward content" - that: has_text text: "reverse content" + +- doc: | + MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED: A list:list collection mapped + over a list:paired_or_unpaired tool input produces a list output collection. + Each inner list is consumed as list:paired_or_unpaired via single_datasets. + job: + input1: + class: Collection + collection_type: list:list + elements: + - identifier: outer1 + class: Collection + type: list + elements: + - identifier: el1 + class: File + contents: "data from outer1 el1" + - identifier: el2 + class: File + contents: "data from outer1 el2" + - identifier: outer2 + class: Collection + type: list + elements: + - identifier: el1 + class: File + contents: "data from outer2 el1" + outputs: + wf_output: + class: Collection + collection_type: list diff --git a/test/unit/data/dataset_collections/test_type_descriptions.py b/test/unit/data/dataset_collections/test_type_descriptions.py index d7cf727f593..155579ac58e 100644 --- a/test/unit/data/dataset_collections/test_type_descriptions.py +++ b/test/unit/data/dataset_collections/test_type_descriptions.py @@ -95,6 +95,59 @@ def test_effective_collection_type_paired_or_unpaired_over_paired(): assert c_t("list:paired_or_unpaired").effective_collection_type("paired_or_unpaired") == "list" +def test_compound_paired_or_unpaired_has_subcollections(): + """Test compound :paired_or_unpaired suffix in has_subcollections_of_type. + + Covers collection_semantics.yml examples: + - MAPPING_LIST_LIST_OVER_LIST_PAIRED_OR_UNPAIRED + - MAPPING_LIST_LIST_PAIRED_OVER_PAIRED_OR_UNPAIRED (via compound path) + """ + # list:list can map over list:paired_or_unpaired + assert c_t("list:list").has_subcollections_of_type("list:paired_or_unpaired") + + # list:list:paired can map over list:paired_or_unpaired + # (paired consumed by paired_or_unpaired, higher ranks list == list) + assert c_t("list:list:paired").has_subcollections_of_type("list:paired_or_unpaired") + + # list:list:list can map over list:paired_or_unpaired + assert c_t("list:list:list").has_subcollections_of_type("list:paired_or_unpaired") + + # list:paired cannot map over list:paired_or_unpaired — same rank, + # after stripping :paired the higher ranks are equal (no remainder) + assert not c_t("list:paired").has_subcollections_of_type("list:paired_or_unpaired") + + # list cannot map over list:paired_or_unpaired — lower rank + assert not c_t("list").has_subcollections_of_type("list:paired_or_unpaired") + + # paired cannot map over list:paired_or_unpaired — higher ranks don't align + assert not c_t("paired").has_subcollections_of_type("list:paired_or_unpaired") + + # paired:paired cannot map over list:paired_or_unpaired — higher ranks + # don't align (paired != list) + assert not c_t("paired:paired").has_subcollections_of_type("list:paired_or_unpaired") + + +def test_compound_paired_or_unpaired_effective_collection_type(): + """Test effective_collection_type with compound :paired_or_unpaired.""" + # list:list over list:paired_or_unpaired → list + assert c_t("list:list").effective_collection_type("list:paired_or_unpaired") == "list" + + # list:list:paired over list:paired_or_unpaired → list + # (peel off list and paired_or_unpaired ranks, then strip :paired) + assert c_t("list:list:paired").effective_collection_type("list:paired_or_unpaired") == "list" + + # list:list:list over list:paired_or_unpaired → list:list + assert c_t("list:list:list").effective_collection_type("list:paired_or_unpaired") == "list:list" + + +def test_effective_collection_type_paired_or_unpaired_non_paired(): + """Test that paired_or_unpaired acts like single_datasets for non-paired collections.""" + # list over paired_or_unpaired → list (full type preserved, like single_datasets) + assert c_t("list").effective_collection_type("paired_or_unpaired") == "list" + # list:list over paired_or_unpaired → list:list + assert c_t("list:list").effective_collection_type("paired_or_unpaired") == "list:list" + + def test_validate(): c_t("list").validate() c_t("list:paired").validate() From 959b6076d6317111d1ba0b45ed41617285baa3ea Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Mar 2026 09:29:27 -0400 Subject: [PATCH 13/14] Fix has_subcollections_of_type endswith to require colon boundary. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The endswith check matched partial type names: e.g. 'list:paired_or_unpaired'.endswith('paired') was True because 'paired' is a suffix of 'paired_or_unpaired'. This incorrectly reported list:paired_or_unpaired as having subcollections of type paired (violates PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING in collection_semantics.yml). Fix by requiring ':' prefix in the endswith check — the exact-match case is already handled by the equality check above. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../dataset_collections/type_description.py | 2 +- .../types/collection_semantics.yml | 2 ++ lib/galaxy_test/api/test_tool_execute.py | 12 ++++++++++ .../test_type_descriptions.py | 23 +++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/dataset_collections/type_description.py b/lib/galaxy/model/dataset_collections/type_description.py index 3eba55ea4f0..7c7f6950a40 100644 --- a/lib/galaxy/model/dataset_collections/type_description.py +++ b/lib/galaxy/model/dataset_collections/type_description.py @@ -116,7 +116,7 @@ class CollectionTypeDescription: other_collection_type = _normalize_collection_type(other_collection_type) if collection_type == other_collection_type: return False - if collection_type.endswith(other_collection_type): + if collection_type.endswith(f":{other_collection_type}"): return True if other_collection_type == "paired_or_unpaired": # this can be thought of as a subcollection of anything except a pair diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index ab3cbc2b585..028e186f9f2 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -795,6 +795,8 @@ i: {type: map_over, collection: C} is_valid: false tests: + tool_runtime: + api_test: "test_tool_execute.py::test_paired_or_unpaired_not_consumed_by_paired_when_mapping" workflow_editor: "rejects list:paired_or_unpaired -> paired connection" - example: diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py index 4b1f7acfff1..0a877ce9dce 100644 --- a/lib/galaxy_test/api/test_tool_execute.py +++ b/lib/galaxy_test/api/test_tool_execute.py @@ -584,6 +584,18 @@ def test_map_over_list_paired_or_unpaired_with_list_of_lists( assert output_collection.details["collection_type"] == "list" +@requires_tool_id("collection_paired_test") +def test_paired_or_unpaired_not_consumed_by_paired_when_mapping( + target_history: TargetHistory, required_tool: RequiredTool +): + """PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING: + list:paired_or_unpaired cannot be mapped over a paired tool input.""" + hdca = target_history.with_list_of_paired_and_unpaired() + required_tool.execute().with_inputs( + {"f1": {"batch": True, "values": [{"map_over_type": "paired", **hdca.src_dict}]}} + ).assert_fails.with_error_containing("Cannot split collection") + + @requires_tool_id("collection_paired_test") def test_simple_subcollection_mapping( target_history: TargetHistory, required_tool: RequiredTool, tool_input_format: DescribeToolInputs diff --git a/test/unit/data/dataset_collections/test_type_descriptions.py b/test/unit/data/dataset_collections/test_type_descriptions.py index 155579ac58e..0f8d1d5ab15 100644 --- a/test/unit/data/dataset_collections/test_type_descriptions.py +++ b/test/unit/data/dataset_collections/test_type_descriptions.py @@ -95,6 +95,29 @@ def test_effective_collection_type_paired_or_unpaired_over_paired(): assert c_t("list:paired_or_unpaired").effective_collection_type("paired_or_unpaired") == "list" +def test_endswith_colon_boundary(): + """Regression: endswith must require colon boundary to avoid partial matches. + + 'list:paired_or_unpaired'.endswith('paired') was True because 'paired' + is a substring at the end of 'paired_or_unpaired'. The fix requires + ':paired' (with colon prefix) so only proper rank boundaries match. + """ + # list:paired_or_unpaired does NOT have subcollections of type paired + # PAIRED_OR_UNPAIRED_NOT_CONSUMED_BY_PAIRED_WHEN_MAPPING + assert not c_t("list:paired_or_unpaired").has_subcollections_of_type("paired") + + # But list:paired DOES have subcollections of type paired (proper boundary) + assert c_t("list:paired").has_subcollections_of_type("paired") + + # And list:list:paired_or_unpaired does NOT have subcollections of type paired + assert not c_t("list:list:paired_or_unpaired").has_subcollections_of_type("paired") + + # Existing cases still work + assert c_t("list:list:paired").has_subcollections_of_type("paired") + assert c_t("list:list:paired").has_subcollections_of_type("list:paired") + assert not c_t("list:list:paired").has_subcollections_of_type("list") + + def test_compound_paired_or_unpaired_has_subcollections(): """Test compound :paired_or_unpaired suffix in has_subcollections_of_type. From 17d1e2286b673819af6233a517944f28fabaa757 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Mar 2026 11:44:36 -0400 Subject: [PATCH 14/14] Fix MAPPING_LIST_PAIRED_OVER_PAIRED api_test ref to correct test location. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../model/dataset_collections/types/collection_semantics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml index 028e186f9f2..650e39e8427 100644 --- a/lib/galaxy/model/dataset_collections/types/collection_semantics.yml +++ b/lib/galaxy/model/dataset_collections/types/collection_semantics.yml @@ -605,7 +605,7 @@ output: o tests: tool_runtime: - api_test: "test_tools.py::TestToolsApi::test_subcollection_mapping" + api_test: "test_tool_execute.py::test_simple_subcollection_mapping" workflow_editor: "accepts list:paired -> paired connection" - doc: |