From 8f9a57d0e5277d0d1edcef1939f1fa00940d8a65 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 25 Feb 2026 01:30:22 -0500 Subject: [PATCH 1/3] Fix workflow implicit mapping of flat collections over paired_or_unpaired The workflow engine failed to map flat list/sample_sheet collections over paired_or_unpaired tool inputs. The API tool execution path worked because it explicitly translates paired_or_unpaired to single_datasets and wraps elements in PromoteCollectionElementToCollectionAdapter. The workflow path was missing both steps. Two fixes in workflow/modules.py: - _find_collections_to_match: translate paired_or_unpaired subcollection type to single_datasets for flat collections (mirrors basic.py:2675) - ToolModule.execute callback: wrap flat DCEs in adapter when mapping type is single_datasets Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/workflow/modules.py | 33 ++++++++++++++- lib/galaxy_test/api/test_tool_execute.py | 12 ++++++ lib/galaxy_test/base/populators.py | 9 +++++ ...lat_over_paired_or_unpaired.gxwf-tests.yml | 40 +++++++++++++++++++ .../flat_over_paired_or_unpaired.gxwf.yml | 13 ++++++ 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf-tests.yml create mode 100644 lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf.yml diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 903717b2d89..4e438eb30b2 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -40,6 +40,7 @@ from galaxy.model import ( ) from galaxy.model.base import ensure_object_added_to_session from galaxy.model.dataset_collections import matching +from galaxy.model.dataset_collections.adapters import PromoteCollectionElementToCollectionAdapter from galaxy.model.dataset_collections.query import HistoryQuery from galaxy.model.dataset_collections.type_description import COLLECTION_TYPE_DESCRIPTION_FACTORY from galaxy.model.dataset_collections.types.sample_sheet_util import validate_column_definitions @@ -638,6 +639,20 @@ class WorkflowModule: subcollection_type_description = history_query.can_map_over(data) or None if subcollection_type_description: + # Translate paired_or_unpaired to concrete mapping type for + # flat collections. Mirrors logic in basic.py:2675-2679 that + # the API tool execution path uses. + _sub_ct = subcollection_type_description.collection_type + _hdca_ct = data.collection.collection_type + if _sub_ct == "paired_or_unpaired" and not _hdca_ct.endswith("paired_or_unpaired"): + if _hdca_ct.endswith("paired"): + subcollection_type_description = dataset_collection_type_descriptions.for_collection_type( + "paired" + ) + else: + subcollection_type_description = dataset_collection_type_descriptions.for_collection_type( + "single_datasets" + ) subcollection_type_list = subcollection_type_description.collection_type.split(":") for collection_type in reversed(subcollection_type_list): if type_list: @@ -2414,9 +2429,25 @@ class ToolModule(WorkflowModule): def callback(input, prefixed_name: str, **kwargs): input_dict = all_inputs_by_name[prefixed_name] - replacement: Union[model.Dataset, NoReplacement] = NO_REPLACEMENT + replacement: Union[model.Dataset, NoReplacement, PromoteCollectionElementToCollectionAdapter] = ( + NO_REPLACEMENT + ) if iteration_elements and prefixed_name in iteration_elements: # noqa: B023 replacement = iteration_elements[prefixed_name] # noqa: B023 + # When mapping flat collections over paired_or_unpaired via + # single_datasets, wrap each element in an adapter so the + # tool sees a paired_or_unpaired collection. + if ( + collection_info # noqa: B023 + and isinstance(replacement, model.DatasetCollectionElement) + and not replacement.child_collection + ): + mapping_type = collection_info.subcollection_mapping_type(prefixed_name) # noqa: B023 + if ( + hasattr(mapping_type, "collection_type") + and mapping_type.collection_type == "single_datasets" + ): + replacement = PromoteCollectionElementToCollectionAdapter(replacement) else: replacement = progress.replacement_for_input(trans, step, input_dict) diff --git a/lib/galaxy_test/api/test_tool_execute.py b/lib/galaxy_test/api/test_tool_execute.py index 59018a37668..f6eb27c31c6 100644 --- a/lib/galaxy_test/api/test_tool_execute.py +++ b/lib/galaxy_test/api/test_tool_execute.py @@ -525,6 +525,18 @@ def test_map_over_paired_or_unpaired_with_list(target_history: TargetHistory, re output_collection.assert_has_dataset_element("foo").with_contents_stripped("text for foo element") +@requires_tool_id("collection_paired_or_unpaired") +def test_map_over_paired_or_unpaired_with_sample_sheet(target_history: TargetHistory, required_tool: RequiredTool): + contents = [("foo", "text for foo element")] + hdca = target_history.with_sample_sheet(contents) + execute = required_tool.execute().with_inputs( + {"f1": {"batch": True, "values": [{"map_over_type": "single_datasets", **hdca.src_dict}]}} + ) + execute.assert_has_n_jobs(1).assert_creates_n_implicit_collections(1) + output_collection = execute.assert_creates_implicit_collection(0) + output_collection.assert_has_dataset_element("foo").with_contents_stripped("text for foo element") + + @requires_tool_id("collection_paired_or_unpaired") def test_map_over_paired_or_unpaired_with_list_of_lists(target_history: TargetHistory, required_tool: RequiredTool): hdca = target_history.with_example_list_of_lists() diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index f70372b5556..4b6d3b7c4cc 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -4742,6 +4742,15 @@ class TargetHistory: ) ) + def with_sample_sheet(self, contents: Optional[ListContentsDescription] = None) -> "HasSrcDict": + if contents is None: + contents = [("foo", "text for foo element")] + create_response = self._dataset_collection_populator.create_sample_sheet( + self._history_id, contents=contents, column_definitions=[], rows={c[0]: [] for c in contents} + ) + api_asserts.assert_status_code_is_ok(create_response) + return HasSrcDict("hdca", create_response.json()) + def with_example_list_of_pairs(self) -> "HasSrcDict": return HasSrcDict("hdca", self._dataset_collection_populator.example_list_of_pairs(self._history_id)) diff --git a/lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf-tests.yml b/lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf-tests.yml new file mode 100644 index 00000000000..94776a87b32 --- /dev/null +++ b/lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf-tests.yml @@ -0,0 +1,40 @@ +- doc: | + A flat list mapped over a paired_or_unpaired tool input via + single_datasets produces a list output. + job: + input1: + class: Collection + collection_type: list + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: list + +- doc: | + A flat sample_sheet mapped over a paired_or_unpaired tool input via + single_datasets produces a sample_sheet output. + job: + input1: + class: Collection + collection_type: sample_sheet + rows: + el1: [] + el2: [] + elements: + - identifier: el1 + class: File + contents: "element 1" + - identifier: el2 + class: File + contents: "element 2" + outputs: + wf_output: + class: Collection + collection_type: sample_sheet diff --git a/lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf.yml b/lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf.yml new file mode 100644 index 00000000000..98252c3723c --- /dev/null +++ b/lib/galaxy_test/workflow/flat_over_paired_or_unpaired.gxwf.yml @@ -0,0 +1,13 @@ +class: GalaxyWorkflow +inputs: + input1: + type: collection + collection_type: paired_or_unpaired +outputs: + wf_output: + outputSource: tool_step/out1 +steps: + tool_step: + tool_id: collection_paired_or_unpaired + in: + f1: input1 From 40e1c3618095e84070026beedf48187770e3d649 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 25 Feb 2026 11:36:18 -0500 Subject: [PATCH 2/3] Allow paired_or_unpaired, record, sample_sheet in collection_type validator Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tool_util_models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util_models/__init__.py b/lib/galaxy/tool_util_models/__init__.py index d492c978bc3..4c3121e5b38 100644 --- a/lib/galaxy/tool_util_models/__init__.py +++ b/lib/galaxy/tool_util_models/__init__.py @@ -218,7 +218,7 @@ def _check_collection_type(v: str) -> str: raise ValueError("Invalid empty collection_type specified.") collection_levels = v.split(":") for collection_level in collection_levels: - if collection_level not in ["list", "paired"]: + if collection_level not in ["list", "paired", "paired_or_unpaired", "record", "sample_sheet"]: raise ValueError(f"Invalid collection_type specified [{v}]") return v From 0188e7094503cda2f6e82ebd66929fdf0ea72678 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 23 Feb 2026 16:01:28 -0500 Subject: [PATCH 3/3] Converge framework workflow test syntax toward Planemo class: syntax Route framework workflow tests through galactic_job_json() (shared with Planemo) instead of custom load_data_dict() dispatch. Detect class: syntax in test YAML and route accordingly, preserving backward compat. - Fix filetype not forwarded for contents uploads in galactic_job_json - FileLiteralTarget reads filetype/dbkey from properties in staging - Add _uses_class_syntax() detection + routing in run_workflow() - Extract type: raw params before stage_inputs(), merge back after - Thread name from YAML through CollectionCreateFunc protocol and staging - Thread name for File inputs through replacement_file() to upload targets - Fix path resolution in stage_inputs() using TestDataResolver fallback - Pass use_path_paste=False from run_workflow() to avoid admin-only file:// URIs - Convert ~20 gxwf-tests.yml files to class: Collection/File syntax with explicit elements Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/tool_util/client/staging.py | 21 ++- lib/galaxy/tool_util/cwl/util.py | 12 +- lib/galaxy_test/base/populators.py | 54 +++++++- .../workflow/directory_index.gxwf-tests.yml | 6 +- .../empty_collection_sort.gxwf-tests.yml | 10 +- .../workflow/filter_null.gxwf-tests.yml | 8 +- ...n_collection_over_execution.gxwf-tests.yml | 6 +- .../integer_into_data_column.gxwf-tests.yml | 6 +- .../map_over_expression.gxwf-tests.yml | 7 +- .../multi_select_mapping.gxwf-tests.yml | 12 +- ...le_integer_into_data_column.gxwf-tests.yml | 6 +- ...tional_inputs_to_build_list.gxwf-tests.yml | 6 +- .../workflow/output_parameter.gxwf-tests.yml | 4 +- ...e_based_on_input_collection.gxwf-tests.yml | 17 +-- .../subcollection_rank_sorting.gxwf-tests.yml | 62 ++++++++- ...lection_rank_sorting_paired.gxwf-tests.yml | 53 ++++++- .../workflow/test_framework_workflows.py | 1 + .../triply_nested_list_mapping.gxwf-tests.yml | 48 ++++++- .../workflow/zip_collection.gxwf-tests.yml | 10 +- test/unit/tool_util/test_cwl_util.py | 130 +++++++++++++++++- 20 files changed, 415 insertions(+), 64 deletions(-) diff --git a/lib/galaxy/tool_util/client/staging.py b/lib/galaxy/tool_util/client/staging.py index dbdfbda8185..a7c64e60f2c 100644 --- a/lib/galaxy/tool_util/client/staging.py +++ b/lib/galaxy/tool_util/client/staging.py @@ -112,7 +112,7 @@ class StagingInterface(metaclass=abc.ABCMeta): decompress=upload_target.properties.get("decompress") or DEFAULT_DECOMPRESS, hashes=upload_target.properties.get("hashes"), ) - name = _file_path_to_name(file_path) + name = upload_target.properties.get("name") or _file_path_to_name(file_path) if file_path is not None: src = _attach_file(fetch_payload, file_path) fetch_payload["targets"][0]["elements"][0].update(src) @@ -134,7 +134,9 @@ class StagingInterface(metaclass=abc.ABCMeta): fetch_payload["targets"][0]["elements"][0]["tags"] = tags fetch_payload["targets"][0]["elements"][0]["name"] = name elif isinstance(upload_target, FileLiteralTarget): - fetch_payload = _fetch_payload(history_id) + file_type = upload_target.properties.get("filetype", None) or DEFAULT_FILE_TYPE + dbkey = upload_target.properties.get("dbkey", None) or DEFAULT_DBKEY + fetch_payload = _fetch_payload(history_id, file_type=file_type, dbkey=dbkey) # For file literals - take them as is - never convert line endings. fetch_payload["targets"][0]["elements"][0].update( { @@ -146,6 +148,9 @@ class StagingInterface(metaclass=abc.ABCMeta): tags = upload_target.properties.get("tags") if tags: fetch_payload["targets"][0]["elements"][0]["tags"] = tags + name = upload_target.properties.get("name") + if name: + fetch_payload["targets"][0]["elements"][0]["name"] = name elif isinstance(upload_target, DirectoryUploadTarget): fetch_payload = _fetch_payload(history_id, file_type=upload_target.file_type) element = fetch_payload["targets"][0]["elements"][0] @@ -191,7 +196,7 @@ class StagingInterface(metaclass=abc.ABCMeta): file_type = upload_target.properties.get("filetype", None) or DEFAULT_FILE_TYPE dbkey = upload_target.properties.get("dbkey", None) or DEFAULT_DBKEY upload_payload = _upload_payload(history_id, file_type=file_type, to_posix_lines=dbkey) - name = _file_path_to_name(file_path) + name = upload_target.properties.get("name") or _file_path_to_name(file_path) upload_payload["inputs"]["files_0|auto_decompress"] = False upload_payload["inputs"]["auto_decompress"] = False if file_path is not None: @@ -216,7 +221,8 @@ class StagingInterface(metaclass=abc.ABCMeta): return self._tools_post(upload_payload) elif isinstance(upload_target, FileLiteralTarget): # For file literals - take them as is - never convert line endings. - payload = _upload_payload(history_id, file_type="auto", auto_decompress=False, to_posix_lines=False) + file_type = upload_target.properties.get("filetype", None) or DEFAULT_FILE_TYPE + payload = _upload_payload(history_id, file_type=file_type, auto_decompress=False, to_posix_lines=False) payload["inputs"]["files_0|url_paste"] = upload_target.contents return self._tools_post(payload) elif isinstance(upload_target, DirectoryUploadTarget): @@ -246,10 +252,13 @@ class StagingInterface(metaclass=abc.ABCMeta): raise ValueError(f"Unsupported type for upload_target: {type(upload_target)}") def create_collection_func( - element_identifiers: List[Dict[str, Any]], collection_type: str, rows: Optional[Dict[str, Any]] = None + element_identifiers: List[Dict[str, Any]], + collection_type: str, + rows: Optional[Dict[str, Any]] = None, + name: Optional[str] = None, ) -> Dict[str, Any]: payload = { - "name": "dataset collection", + "name": name or "dataset collection", "instance_type": "history", "history_id": history_id, "element_identifiers": element_identifiers, diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index aae6561c692..2a455085aa0 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -138,7 +138,11 @@ def path_or_uri_to_uri(path_or_uri: str) -> str: class CollectionCreateFunc(Protocol): def __call__( - self, element_identifiers: List[Dict[str, Any]], collection_type: str, rows: Optional[Dict[str, Any]] = None + self, + element_identifiers: List[Dict[str, Any]], + collection_type: str, + rows: Optional[Dict[str, Any]] = None, + name: Optional[str] = None, ) -> Dict[str, Any]: """Create a collection from these identifiers.""" @@ -253,6 +257,8 @@ def galactic_job_json( kwd["hashes"] = value["hashes"] if value.get("metadata"): kwd["metadata"] = value["metadata"] + if "name" in value: + kwd["name"] = value["name"] if composite_data_raw: composite_data = [] for entry in composite_data_raw: @@ -268,7 +274,7 @@ def galactic_job_json( if file_path is None: contents = value.get("contents") if contents is not None: - return upload_file_literal(contents, **kwd) + return upload_file_literal(contents, filetype=filetype, **kwd) return value @@ -360,6 +366,8 @@ def galactic_job_json( kwds = {} if collection_type.startswith("sample_sheet"): kwds["rows"] = value["rows"] + if "name" in value: + kwds["name"] = value["name"] collection = collection_create_func(elements, collection_type, **kwds) dataset_collections.append(collection) hdca_id = collection["id"] diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 4b6d3b7c4cc..21e33a8d191 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -2646,6 +2646,7 @@ class BaseWorkflowPopulator(BasePopulator): raw_yaml: bool = False, use_cached_job: bool = False, copy_inputs_to_history: bool = False, + job_dir: Optional[str] = None, ): """High-level wrapper around workflow API, etc. to invoke format 2 workflows.""" workflow_populator = self @@ -2679,9 +2680,33 @@ class BaseWorkflowPopulator(BasePopulator): replacement_parameters = test_data_dict.pop("replacement_parameters", {}) if history_id is None: history_id = self.dataset_populator.new_history() - inputs, label_map, has_uploads = load_data_dict( - history_id, test_data_dict, self.dataset_populator, self.dataset_collection_populator - ) + if _uses_class_syntax(test_data_dict): + # Copy because galactic_job_json() mutates the dict in-place + job_copy = dict(test_data_dict) + # Extract raw parameters before staging — galactic_job_json + # doesn't understand type: raw and would misinterpret them + raw_params = {} + for k, v in list(job_copy.items()): + if isinstance(v, dict) and v.get("type") == "raw": + raw_params[k] = v["value"] + del job_copy[k] + staged_job, datasets = stage_inputs( + self.dataset_populator.galaxy_interactor, + history_id, + job_copy, + use_path_paste=False, + job_dir=job_dir, + ) + if datasets: + self.dataset_populator.wait_for_history(history_id, assert_ok=True) + staged_job.update(raw_params) + label_map = staged_job + inputs = staged_job + has_uploads = bool(datasets) + else: + inputs, label_map, has_uploads = load_data_dict( + history_id, test_data_dict, self.dataset_populator, self.dataset_collection_populator + ) workflow_request: dict[str, Any] = dict( history=f"hist_id={history_id}", workflow_id=workflow_id, @@ -3948,6 +3973,16 @@ class DatasetCollectionPopulator(BaseDatasetCollectionPopulator): return create_response +def _uses_class_syntax(test_data_dict: dict) -> bool: + """Detect Planemo-style class: Collection/File syntax in test data.""" + for value in test_data_dict.values(): + if isinstance(value, dict) and "class" in value: + return True + if isinstance(value, list): # CWL list shorthand + return True + return False + + LoadDataDictResponseT = tuple[dict[str, Any], dict[str, Any], bool] @@ -4071,6 +4106,17 @@ def stage_inputs( job_dir: Optional[str] = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """Alternative to load_data_dict that uses production-style workflow inputs.""" + test_data_resolver = TestDataResolver() + + def resolve_data(filename): + result = galaxy_interactor._find_in_test_data_directories(filename) + if result and os.path.exists(result): + return result + try: + return test_data_resolver.get_filename(filename) + except Exception: + return None + kwds = {} if job_dir is not None: kwds["job_dir"] = job_dir @@ -4080,7 +4126,7 @@ def stage_inputs( job=job, use_path_paste=use_path_paste, to_posix_lines=to_posix_lines, - resolve_data=galaxy_interactor._find_in_test_data_directories, + resolve_data=resolve_data, **kwds, ) diff --git a/lib/galaxy_test/workflow/directory_index.gxwf-tests.yml b/lib/galaxy_test/workflow/directory_index.gxwf-tests.yml index e5c74e6658a..62d96ac4953 100644 --- a/lib/galaxy_test/workflow/directory_index.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/directory_index.gxwf-tests.yml @@ -2,9 +2,9 @@ Test that passing directory indexes works job: reference: - type: File - value: 1.fasta - file_type: fasta + class: File + path: 1.fasta + filetype: fasta outputs: output: class: File diff --git a/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml b/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml index d5ac9d9b15b..da1e16da731 100644 --- a/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/empty_collection_sort.gxwf-tests.yml @@ -1,12 +1,16 @@ -- doc: | +- doc: | Test to verify collection operations like the sort tool work fine with empty collections. job: input: + class: Collection collection_type: list elements: - identifier: i1 - content: "0" - filter_file: i1 + class: File + contents: "0" + filter_file: + class: File + contents: "i1" outputs: output: class: Collection diff --git a/lib/galaxy_test/workflow/filter_null.gxwf-tests.yml b/lib/galaxy_test/workflow/filter_null.gxwf-tests.yml index 7572b8c5c62..3a9e81ddaee 100644 --- a/lib/galaxy_test/workflow/filter_null.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/filter_null.gxwf-tests.yml @@ -2,10 +2,12 @@ Test to verify filter null tool keeps non-null datasets. job: input_collection: + class: Collection collection_type: list elements: - identifier: first - content: "abc" + class: File + contents: "abc" when: value: true type: raw @@ -18,10 +20,12 @@ Test to verify filter null tool discards null datasets. job: input_collection: + class: Collection collection_type: list elements: - identifier: first - content: "abc" + class: File + contents: "abc" when: value: false type: raw diff --git a/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml b/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml index 85d986adbb1..4c1354280d8 100644 --- a/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/flatten_collection_over_execution.gxwf-tests.yml @@ -1,11 +1,13 @@ -- doc: | +- doc: | Test to verify collection flatten collection operation mid workflow. job: input_fastqs: + class: Collection collection_type: list elements: - identifier: samp1 - content: "0 mycoolline\n1 mysecondline\n" + class: File + contents: "0 mycoolline\n1 mysecondline\n" outputs: out: class: Collection diff --git a/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml b/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml index eae87aa6f31..bc45bbbe8fe 100644 --- a/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/integer_into_data_column.gxwf-tests.yml @@ -2,9 +2,9 @@ Test to verify text parameter can be connected to data column param job: input: - type: File - value: 2.tabular - file_type: tabular + class: File + path: 2.tabular + filetype: tabular column: value: "2" type: raw diff --git a/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml b/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml index 7a52cb858c2..b6f015ea1fe 100644 --- a/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/map_over_expression.gxwf-tests.yml @@ -2,12 +2,15 @@ Test to verify text parameter can be connected to data column param job: text_input1: + class: Collection collection_type: list elements: - identifier: A - content: A + class: File + contents: A - identifier: B - content: B + class: File + contents: B outputs: out1: class: Collection diff --git a/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml b/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml index b9857cc3064..0380d53a9af 100644 --- a/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/multi_select_mapping.gxwf-tests.yml @@ -5,15 +5,17 @@ string. job: input: - type: collection + class: Collection collection_type: list elements: - identifier: the_example_2 - content: '"ex2"' - ext: 'expression.json' + class: File + contents: '"ex2"' + filetype: 'expression.json' - identifier: the_example_5 - content: '"ex5"' - ext: 'expression.json' + class: File + contents: '"ex5"' + filetype: 'expression.json' outputs: output: class: Collection diff --git a/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml b/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml index 365bf7f6de1..d6a078c7878 100644 --- a/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/multiple_integer_into_data_column.gxwf-tests.yml @@ -2,9 +2,9 @@ Test to verify text parameter can be connected to data column param job: input: - type: File - value: 2.tabular - file_type: tabular + class: File + path: 2.tabular + filetype: tabular column: value: [1, 2] type: raw diff --git a/lib/galaxy_test/workflow/optional_conditional_inputs_to_build_list.gxwf-tests.yml b/lib/galaxy_test/workflow/optional_conditional_inputs_to_build_list.gxwf-tests.yml index f7592cb2722..96574aafa60 100644 --- a/lib/galaxy_test/workflow/optional_conditional_inputs_to_build_list.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/optional_conditional_inputs_to_build_list.gxwf-tests.yml @@ -2,9 +2,9 @@ Test optional unspecified input into conditional step works job: required_dataset: - type: File - value: 2.tabular - file_type: tabular + class: File + path: 2.tabular + filetype: tabular outputs: out: class: Collection diff --git a/lib/galaxy_test/workflow/output_parameter.gxwf-tests.yml b/lib/galaxy_test/workflow/output_parameter.gxwf-tests.yml index d422d9e3d83..4bdf534fd7c 100644 --- a/lib/galaxy_test/workflow/output_parameter.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/output_parameter.gxwf-tests.yml @@ -2,7 +2,7 @@ Test to verify exact output parameter verification works propery. job: text_int: - type: File - content: "43" + class: File + contents: "43" outputs: out_int: 43 diff --git a/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml index 9f76add4f57..109c10c6c3e 100644 --- a/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/rename_based_on_input_collection.gxwf-tests.yml @@ -1,21 +1,22 @@ -- doc: | +- doc: | Test output dataset renaming when the target basename is based on an input collection. job: fasta_input: - value: 1.fasta - type: File + class: File + path: 1.fasta name: fasta1 - file_type: fasta + filetype: fasta fastq_inputs: + class: Collection collection_type: list name: the_dataset_pair elements: - identifier: forward - value: 1.fastq - type: File + class: File + path: 1.fastq - identifier: reverse - value: 1.fastq - type: File + class: File + path: 1.fastq outputs: output: class: File diff --git a/lib/galaxy_test/workflow/subcollection_rank_sorting.gxwf-tests.yml b/lib/galaxy_test/workflow/subcollection_rank_sorting.gxwf-tests.yml index c9a4c74e4e8..df068908f01 100644 --- a/lib/galaxy_test/workflow/subcollection_rank_sorting.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/subcollection_rank_sorting.gxwf-tests.yml @@ -1,11 +1,25 @@ -- doc: | +- doc: | Test to verify a tool that takes in a list or nested list is sent the most specific list possible during workflow execution (the nested list). This should not trigger subcollection mapping. job: input: - type: collection + class: Collection collection_type: list:list + elements: + - identifier: test_level_1 + class: Collection + type: list + elements: + - identifier: data1 + class: File + contents: "TestData123" + - identifier: data2 + class: File + contents: "TestData123" + - identifier: data3 + class: File + contents: "TestData123" outputs: out: asserts: @@ -15,8 +29,26 @@ Once again with mapping, should reduce the result into a flat list. job: input: - type: collection + class: Collection collection_type: list:list:list + elements: + - identifier: test_level_2 + class: Collection + type: list:list + elements: + - identifier: test_level_1 + class: Collection + type: list + elements: + - identifier: data1 + class: File + contents: "TestData123" + - identifier: data2 + class: File + contents: "TestData123" + - identifier: data3 + class: File + contents: "TestData123" outputs: out: class: Collection @@ -34,8 +66,30 @@ two levels should be mapped over. job: input: - type: collection + class: Collection collection_type: list:list:list:list + elements: + - identifier: test_level_3 + class: Collection + type: list:list:list + elements: + - identifier: test_level_2 + class: Collection + type: list:list + elements: + - identifier: test_level_1 + class: Collection + type: list + elements: + - identifier: data1 + class: File + contents: "TestData123" + - identifier: data2 + class: File + contents: "TestData123" + - identifier: data3 + class: File + contents: "TestData123" outputs: out: class: Collection diff --git a/lib/galaxy_test/workflow/subcollection_rank_sorting_paired.gxwf-tests.yml b/lib/galaxy_test/workflow/subcollection_rank_sorting_paired.gxwf-tests.yml index d12a7fb80e2..49f76d9b75f 100644 --- a/lib/galaxy_test/workflow/subcollection_rank_sorting_paired.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/subcollection_rank_sorting_paired.gxwf-tests.yml @@ -1,11 +1,22 @@ -- doc: | +- doc: | Test to verify a tool that takes in a paired or list of paired is sent the most specific list possible during workflow execution (the list of paired datasets). This should not trigger subcollection mapping. job: input: - type: collection + class: Collection collection_type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" outputs: out: asserts: @@ -16,8 +27,23 @@ Once again with mapping, should reduce the result into a flat list. job: input: - type: collection + class: Collection collection_type: list:list:paired + elements: + - identifier: test_level_2 + class: Collection + type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" outputs: out: class: Collection @@ -34,8 +60,27 @@ Once again with double mapping, should reduce the result into a nested list (list:list). job: input: - type: collection + class: Collection collection_type: list:list:list:paired + elements: + - identifier: test_level_3 + class: Collection + type: list:list:paired + elements: + - identifier: test_level_2 + class: Collection + type: list:paired + elements: + - identifier: test_level_1 + class: Collection + type: paired + elements: + - identifier: forward + class: File + contents: "TestData123" + - identifier: reverse + class: File + contents: "TestData123" outputs: out: class: Collection diff --git a/lib/galaxy_test/workflow/test_framework_workflows.py b/lib/galaxy_test/workflow/test_framework_workflows.py index feb46d72bfe..4baeb4bd105 100644 --- a/lib/galaxy_test/workflow/test_framework_workflows.py +++ b/lib/galaxy_test/workflow/test_framework_workflows.py @@ -70,6 +70,7 @@ class TestWorkflow(ApiTestCase): yaml_content, test_data=test_job["job"], history_id=history_id, + job_dir=str(workflow_path.parent), ) if TEST_WORKFLOW_AFTER_RERUN: run_summary = self.workflow_populator.rerun(run_summary) diff --git a/lib/galaxy_test/workflow/triply_nested_list_mapping.gxwf-tests.yml b/lib/galaxy_test/workflow/triply_nested_list_mapping.gxwf-tests.yml index 0c2f03f7284..7e037acfaa6 100644 --- a/lib/galaxy_test/workflow/triply_nested_list_mapping.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/triply_nested_list_mapping.gxwf-tests.yml @@ -1,10 +1,28 @@ -- doc: | +- doc: | Test that a basic list:list:list parameter is consumed directly by the tool and not mapped over. job: input: - type: collection + class: Collection collection_type: list:list:list + elements: + - identifier: test_level_2 + class: Collection + type: list:list + elements: + - identifier: test_level_1 + class: Collection + type: list + elements: + - identifier: data1 + class: File + contents: "TestData123" + - identifier: data2 + class: File + contents: "TestData123" + - identifier: data3 + class: File + contents: "TestData123" outputs: out: asserts: @@ -13,12 +31,34 @@ - that: has_text text: "collection_type" -- doc: | +- doc: | Test that mapping over a list:list:list input parameter. job: input: - type: collection + class: Collection collection_type: list:list:list:list + elements: + - identifier: test_level_3 + class: Collection + type: list:list:list + elements: + - identifier: test_level_2 + class: Collection + type: list:list + elements: + - identifier: test_level_1 + class: Collection + type: list + elements: + - identifier: data1 + class: File + contents: "TestData123" + - identifier: data2 + class: File + contents: "TestData123" + - identifier: data3 + class: File + contents: "TestData123" outputs: out: class: Collection diff --git a/lib/galaxy_test/workflow/zip_collection.gxwf-tests.yml b/lib/galaxy_test/workflow/zip_collection.gxwf-tests.yml index e0e4ffc5ba6..93e740f727a 100644 --- a/lib/galaxy_test/workflow/zip_collection.gxwf-tests.yml +++ b/lib/galaxy_test/workflow/zip_collection.gxwf-tests.yml @@ -1,8 +1,12 @@ -- doc: | +- doc: | Test simple use of __ZIP_COLLECTION__ in a workflow. job: - test_input_1: "samp1\t10.0\nsamp2\t20.0\n" - test_input_2: "samp1\t20.0\nsamp2\t40.0\n" + test_input_1: + class: File + contents: "samp1\t10.0\nsamp2\t20.0\n" + test_input_2: + class: File + contents: "samp1\t20.0\nsamp2\t40.0\n" outputs: out: asserts: diff --git a/test/unit/tool_util/test_cwl_util.py b/test/unit/tool_util/test_cwl_util.py index 24c2ac95cf7..57881e15981 100644 --- a/test/unit/tool_util/test_cwl_util.py +++ b/test/unit/tool_util/test_cwl_util.py @@ -1,6 +1,11 @@ import tempfile -from galaxy.tool_util.cwl.util import output_properties +from galaxy.tool_util.cwl.util import ( + FileLiteralTarget, + galactic_job_json, + output_properties, + UploadTarget, +) def test_output_properties_in_memory(): @@ -23,3 +28,126 @@ def test_output_properties_path(): assert props["nameext"] == ".txt" assert props["size"] == 11 assert props["checksum"] == "sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed" + + +def _mock_upload_func(upload_target: UploadTarget): + """Mock upload that captures the target for inspection.""" + dataset_id = f"dataset_{id(upload_target)}" + return {"outputs": [{"id": dataset_id}]} + + +def _mock_collection_create_func(element_identifiers, collection_type, rows=None, name=None): + return {"id": f"collection_{collection_type}"} + + +def test_galactic_job_json_file_literal_filetype(): + """FileLiteralTarget receives filetype when specified via class: File + contents.""" + captured_targets = [] + + def upload_func(upload_target): + captured_targets.append(upload_target) + return _mock_upload_func(upload_target) + + job = { + "input1": { + "class": "File", + "contents": "some text", + "filetype": "txt", + } + } + result_job, datasets = galactic_job_json( + job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow" + ) + assert len(captured_targets) == 1 + target = captured_targets[0] + assert isinstance(target, FileLiteralTarget) + assert target.contents == "some text" + assert target.properties.get("filetype") == "txt" + + +def test_galactic_job_json_file_literal_no_filetype(): + """FileLiteralTarget receives filetype=None when not specified.""" + captured_targets = [] + + def upload_func(upload_target): + captured_targets.append(upload_target) + return _mock_upload_func(upload_target) + + job = { + "input1": { + "class": "File", + "contents": "some text", + } + } + result_job, datasets = galactic_job_json( + job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow" + ) + assert len(captured_targets) == 1 + target = captured_targets[0] + assert isinstance(target, FileLiteralTarget) + assert target.properties.get("filetype") is None + + +def test_galactic_job_json_file_literal_tags_and_dbkey(): + """FileLiteralTarget receives tags and dbkey.""" + captured_targets = [] + + def upload_func(upload_target): + captured_targets.append(upload_target) + return _mock_upload_func(upload_target) + + job = { + "input1": { + "class": "File", + "contents": "some text", + "filetype": "fastq", + "tags": ["group:sample1"], + "dbkey": "hg38", + } + } + result_job, datasets = galactic_job_json( + job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow" + ) + assert len(captured_targets) == 1 + target = captured_targets[0] + assert isinstance(target, FileLiteralTarget) + assert target.properties["filetype"] == "fastq" + assert target.properties["tags"] == ["group:sample1"] + assert target.properties["dbkey"] == "hg38" + + +def test_galactic_job_json_collection_element_filetype(): + """Collection elements with class: File + contents get filetype forwarded.""" + captured_targets = [] + + def upload_func(upload_target): + captured_targets.append(upload_target) + return _mock_upload_func(upload_target) + + job = { + "reads": { + "class": "Collection", + "collection_type": "paired", + "elements": [ + { + "identifier": "forward", + "class": "File", + "contents": "forward reads", + "filetype": "fastqsanger", + }, + { + "identifier": "reverse", + "class": "File", + "contents": "reverse reads", + "filetype": "fastqsanger", + }, + ], + } + } + result_job, datasets = galactic_job_json( + job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow" + ) + assert len(captured_targets) == 2 + for target in captured_targets: + assert isinstance(target, FileLiteralTarget) + assert target.properties.get("filetype") == "fastqsanger"