Converge framework workflow test syntax toward Planemo class: syntax

Route framework workflow tests through galactic_job_json() (shared with
Planemo) instead of custom load_data_dict() dispatch. Detect class: syntax
in test YAML and route accordingly, preserving backward compat.

- Fix filetype not forwarded for contents uploads in galactic_job_json
- FileLiteralTarget reads filetype/dbkey from properties in staging
- Add _uses_class_syntax() detection + routing in run_workflow()
- Extract type: raw params before stage_inputs(), merge back after
- Thread name from YAML through CollectionCreateFunc protocol and staging
- Thread name for File inputs through replacement_file() to upload targets
- Fix path resolution in stage_inputs() using TestDataResolver fallback
- Pass use_path_paste=False from run_workflow() to avoid admin-only file:// URIs
- Convert ~20 gxwf-tests.yml files to class: Collection/File syntax with explicit elements

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
John Chilton
2026-03-02 11:30:24 +01:00
committed by mvdbeek
co-authored by Claude Opus 4.6
parent 40e1c36180
commit 0188e70945
20 changed files with 415 additions and 64 deletions
+15 -6
View File
@@ -112,7 +112,7 @@ class StagingInterface(metaclass=abc.ABCMeta):
decompress=upload_target.properties.get("decompress") or DEFAULT_DECOMPRESS,
hashes=upload_target.properties.get("hashes"),
)
name = _file_path_to_name(file_path)
name = upload_target.properties.get("name") or _file_path_to_name(file_path)
if file_path is not None:
src = _attach_file(fetch_payload, file_path)
fetch_payload["targets"][0]["elements"][0].update(src)
@@ -134,7 +134,9 @@ class StagingInterface(metaclass=abc.ABCMeta):
fetch_payload["targets"][0]["elements"][0]["tags"] = tags
fetch_payload["targets"][0]["elements"][0]["name"] = name
elif isinstance(upload_target, FileLiteralTarget):
fetch_payload = _fetch_payload(history_id)
file_type = upload_target.properties.get("filetype", None) or DEFAULT_FILE_TYPE
dbkey = upload_target.properties.get("dbkey", None) or DEFAULT_DBKEY
fetch_payload = _fetch_payload(history_id, file_type=file_type, dbkey=dbkey)
# For file literals - take them as is - never convert line endings.
fetch_payload["targets"][0]["elements"][0].update(
{
@@ -146,6 +148,9 @@ class StagingInterface(metaclass=abc.ABCMeta):
tags = upload_target.properties.get("tags")
if tags:
fetch_payload["targets"][0]["elements"][0]["tags"] = tags
name = upload_target.properties.get("name")
if name:
fetch_payload["targets"][0]["elements"][0]["name"] = name
elif isinstance(upload_target, DirectoryUploadTarget):
fetch_payload = _fetch_payload(history_id, file_type=upload_target.file_type)
element = fetch_payload["targets"][0]["elements"][0]
@@ -191,7 +196,7 @@ class StagingInterface(metaclass=abc.ABCMeta):
file_type = upload_target.properties.get("filetype", None) or DEFAULT_FILE_TYPE
dbkey = upload_target.properties.get("dbkey", None) or DEFAULT_DBKEY
upload_payload = _upload_payload(history_id, file_type=file_type, to_posix_lines=dbkey)
name = _file_path_to_name(file_path)
name = upload_target.properties.get("name") or _file_path_to_name(file_path)
upload_payload["inputs"]["files_0|auto_decompress"] = False
upload_payload["inputs"]["auto_decompress"] = False
if file_path is not None:
@@ -216,7 +221,8 @@ class StagingInterface(metaclass=abc.ABCMeta):
return self._tools_post(upload_payload)
elif isinstance(upload_target, FileLiteralTarget):
# For file literals - take them as is - never convert line endings.
payload = _upload_payload(history_id, file_type="auto", auto_decompress=False, to_posix_lines=False)
file_type = upload_target.properties.get("filetype", None) or DEFAULT_FILE_TYPE
payload = _upload_payload(history_id, file_type=file_type, auto_decompress=False, to_posix_lines=False)
payload["inputs"]["files_0|url_paste"] = upload_target.contents
return self._tools_post(payload)
elif isinstance(upload_target, DirectoryUploadTarget):
@@ -246,10 +252,13 @@ class StagingInterface(metaclass=abc.ABCMeta):
raise ValueError(f"Unsupported type for upload_target: {type(upload_target)}")
def create_collection_func(
element_identifiers: List[Dict[str, Any]], collection_type: str, rows: Optional[Dict[str, Any]] = None
element_identifiers: List[Dict[str, Any]],
collection_type: str,
rows: Optional[Dict[str, Any]] = None,
name: Optional[str] = None,
) -> Dict[str, Any]:
payload = {
"name": "dataset collection",
"name": name or "dataset collection",
"instance_type": "history",
"history_id": history_id,
"element_identifiers": element_identifiers,
+10 -2
View File
@@ -138,7 +138,11 @@ def path_or_uri_to_uri(path_or_uri: str) -> str:
class CollectionCreateFunc(Protocol):
def __call__(
self, element_identifiers: List[Dict[str, Any]], collection_type: str, rows: Optional[Dict[str, Any]] = None
self,
element_identifiers: List[Dict[str, Any]],
collection_type: str,
rows: Optional[Dict[str, Any]] = None,
name: Optional[str] = None,
) -> Dict[str, Any]:
"""Create a collection from these identifiers."""
@@ -253,6 +257,8 @@ def galactic_job_json(
kwd["hashes"] = value["hashes"]
if value.get("metadata"):
kwd["metadata"] = value["metadata"]
if "name" in value:
kwd["name"] = value["name"]
if composite_data_raw:
composite_data = []
for entry in composite_data_raw:
@@ -268,7 +274,7 @@ def galactic_job_json(
if file_path is None:
contents = value.get("contents")
if contents is not None:
return upload_file_literal(contents, **kwd)
return upload_file_literal(contents, filetype=filetype, **kwd)
return value
@@ -360,6 +366,8 @@ def galactic_job_json(
kwds = {}
if collection_type.startswith("sample_sheet"):
kwds["rows"] = value["rows"]
if "name" in value:
kwds["name"] = value["name"]
collection = collection_create_func(elements, collection_type, **kwds)
dataset_collections.append(collection)
hdca_id = collection["id"]
+50 -4
View File
@@ -2646,6 +2646,7 @@ class BaseWorkflowPopulator(BasePopulator):
raw_yaml: bool = False,
use_cached_job: bool = False,
copy_inputs_to_history: bool = False,
job_dir: Optional[str] = None,
):
"""High-level wrapper around workflow API, etc. to invoke format 2 workflows."""
workflow_populator = self
@@ -2679,9 +2680,33 @@ class BaseWorkflowPopulator(BasePopulator):
replacement_parameters = test_data_dict.pop("replacement_parameters", {})
if history_id is None:
history_id = self.dataset_populator.new_history()
inputs, label_map, has_uploads = load_data_dict(
history_id, test_data_dict, self.dataset_populator, self.dataset_collection_populator
)
if _uses_class_syntax(test_data_dict):
# Copy because galactic_job_json() mutates the dict in-place
job_copy = dict(test_data_dict)
# Extract raw parameters before staging — galactic_job_json
# doesn't understand type: raw and would misinterpret them
raw_params = {}
for k, v in list(job_copy.items()):
if isinstance(v, dict) and v.get("type") == "raw":
raw_params[k] = v["value"]
del job_copy[k]
staged_job, datasets = stage_inputs(
self.dataset_populator.galaxy_interactor,
history_id,
job_copy,
use_path_paste=False,
job_dir=job_dir,
)
if datasets:
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
staged_job.update(raw_params)
label_map = staged_job
inputs = staged_job
has_uploads = bool(datasets)
else:
inputs, label_map, has_uploads = load_data_dict(
history_id, test_data_dict, self.dataset_populator, self.dataset_collection_populator
)
workflow_request: dict[str, Any] = dict(
history=f"hist_id={history_id}",
workflow_id=workflow_id,
@@ -3948,6 +3973,16 @@ class DatasetCollectionPopulator(BaseDatasetCollectionPopulator):
return create_response
def _uses_class_syntax(test_data_dict: dict) -> bool:
"""Detect Planemo-style class: Collection/File syntax in test data."""
for value in test_data_dict.values():
if isinstance(value, dict) and "class" in value:
return True
if isinstance(value, list): # CWL list shorthand
return True
return False
LoadDataDictResponseT = tuple[dict[str, Any], dict[str, Any], bool]
@@ -4071,6 +4106,17 @@ def stage_inputs(
job_dir: Optional[str] = None,
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
"""Alternative to load_data_dict that uses production-style workflow inputs."""
test_data_resolver = TestDataResolver()
def resolve_data(filename):
result = galaxy_interactor._find_in_test_data_directories(filename)
if result and os.path.exists(result):
return result
try:
return test_data_resolver.get_filename(filename)
except Exception:
return None
kwds = {}
if job_dir is not None:
kwds["job_dir"] = job_dir
@@ -4080,7 +4126,7 @@ def stage_inputs(
job=job,
use_path_paste=use_path_paste,
to_posix_lines=to_posix_lines,
resolve_data=galaxy_interactor._find_in_test_data_directories,
resolve_data=resolve_data,
**kwds,
)
@@ -2,9 +2,9 @@
Test that passing directory indexes works
job:
reference:
type: File
value: 1.fasta
file_type: fasta
class: File
path: 1.fasta
filetype: fasta
outputs:
output:
class: File
@@ -1,12 +1,16 @@
- doc: |
- doc: |
Test to verify collection operations like the sort tool work fine with empty collections.
job:
input:
class: Collection
collection_type: list
elements:
- identifier: i1
content: "0"
filter_file: i1
class: File
contents: "0"
filter_file:
class: File
contents: "i1"
outputs:
output:
class: Collection
@@ -2,10 +2,12 @@
Test to verify filter null tool keeps non-null datasets.
job:
input_collection:
class: Collection
collection_type: list
elements:
- identifier: first
content: "abc"
class: File
contents: "abc"
when:
value: true
type: raw
@@ -18,10 +20,12 @@
Test to verify filter null tool discards null datasets.
job:
input_collection:
class: Collection
collection_type: list
elements:
- identifier: first
content: "abc"
class: File
contents: "abc"
when:
value: false
type: raw
@@ -1,11 +1,13 @@
- doc: |
- doc: |
Test to verify collection flatten collection operation mid workflow.
job:
input_fastqs:
class: Collection
collection_type: list
elements:
- identifier: samp1
content: "0 mycoolline\n1 mysecondline\n"
class: File
contents: "0 mycoolline\n1 mysecondline\n"
outputs:
out:
class: Collection
@@ -2,9 +2,9 @@
Test to verify text parameter can be connected to data column param
job:
input:
type: File
value: 2.tabular
file_type: tabular
class: File
path: 2.tabular
filetype: tabular
column:
value: "2"
type: raw
@@ -2,12 +2,15 @@
Test to verify text parameter can be connected to data column param
job:
text_input1:
class: Collection
collection_type: list
elements:
- identifier: A
content: A
class: File
contents: A
- identifier: B
content: B
class: File
contents: B
outputs:
out1:
class: Collection
@@ -5,15 +5,17 @@
string.
job:
input:
type: collection
class: Collection
collection_type: list
elements:
- identifier: the_example_2
content: '"ex2"'
ext: 'expression.json'
class: File
contents: '"ex2"'
filetype: 'expression.json'
- identifier: the_example_5
content: '"ex5"'
ext: 'expression.json'
class: File
contents: '"ex5"'
filetype: 'expression.json'
outputs:
output:
class: Collection
@@ -2,9 +2,9 @@
Test to verify text parameter can be connected to data column param
job:
input:
type: File
value: 2.tabular
file_type: tabular
class: File
path: 2.tabular
filetype: tabular
column:
value: [1, 2]
type: raw
@@ -2,9 +2,9 @@
Test optional unspecified input into conditional step works
job:
required_dataset:
type: File
value: 2.tabular
file_type: tabular
class: File
path: 2.tabular
filetype: tabular
outputs:
out:
class: Collection
@@ -2,7 +2,7 @@
Test to verify exact output parameter verification works propery.
job:
text_int:
type: File
content: "43"
class: File
contents: "43"
outputs:
out_int: 43
@@ -1,21 +1,22 @@
- doc: |
- doc: |
Test output dataset renaming when the target basename is based on an input collection.
job:
fasta_input:
value: 1.fasta
type: File
class: File
path: 1.fasta
name: fasta1
file_type: fasta
filetype: fasta
fastq_inputs:
class: Collection
collection_type: list
name: the_dataset_pair
elements:
- identifier: forward
value: 1.fastq
type: File
class: File
path: 1.fastq
- identifier: reverse
value: 1.fastq
type: File
class: File
path: 1.fastq
outputs:
output:
class: File
@@ -1,11 +1,25 @@
- doc: |
- doc: |
Test to verify a tool that takes in a list or nested list is sent the most
specific list possible during workflow execution (the nested list). This should
not trigger subcollection mapping.
job:
input:
type: collection
class: Collection
collection_type: list:list
elements:
- identifier: test_level_1
class: Collection
type: list
elements:
- identifier: data1
class: File
contents: "TestData123"
- identifier: data2
class: File
contents: "TestData123"
- identifier: data3
class: File
contents: "TestData123"
outputs:
out:
asserts:
@@ -15,8 +29,26 @@
Once again with mapping, should reduce the result into a flat list.
job:
input:
type: collection
class: Collection
collection_type: list:list:list
elements:
- identifier: test_level_2
class: Collection
type: list:list
elements:
- identifier: test_level_1
class: Collection
type: list
elements:
- identifier: data1
class: File
contents: "TestData123"
- identifier: data2
class: File
contents: "TestData123"
- identifier: data3
class: File
contents: "TestData123"
outputs:
out:
class: Collection
@@ -34,8 +66,30 @@
two levels should be mapped over.
job:
input:
type: collection
class: Collection
collection_type: list:list:list:list
elements:
- identifier: test_level_3
class: Collection
type: list:list:list
elements:
- identifier: test_level_2
class: Collection
type: list:list
elements:
- identifier: test_level_1
class: Collection
type: list
elements:
- identifier: data1
class: File
contents: "TestData123"
- identifier: data2
class: File
contents: "TestData123"
- identifier: data3
class: File
contents: "TestData123"
outputs:
out:
class: Collection
@@ -1,11 +1,22 @@
- doc: |
- doc: |
Test to verify a tool that takes in a paired or list of paired is sent the most
specific list possible during workflow execution (the list of paired datasets). This should
not trigger subcollection mapping.
job:
input:
type: collection
class: Collection
collection_type: list:paired
elements:
- identifier: test_level_1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "TestData123"
- identifier: reverse
class: File
contents: "TestData123"
outputs:
out:
asserts:
@@ -16,8 +27,23 @@
Once again with mapping, should reduce the result into a flat list.
job:
input:
type: collection
class: Collection
collection_type: list:list:paired
elements:
- identifier: test_level_2
class: Collection
type: list:paired
elements:
- identifier: test_level_1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "TestData123"
- identifier: reverse
class: File
contents: "TestData123"
outputs:
out:
class: Collection
@@ -34,8 +60,27 @@
Once again with double mapping, should reduce the result into a nested list (list:list).
job:
input:
type: collection
class: Collection
collection_type: list:list:list:paired
elements:
- identifier: test_level_3
class: Collection
type: list:list:paired
elements:
- identifier: test_level_2
class: Collection
type: list:paired
elements:
- identifier: test_level_1
class: Collection
type: paired
elements:
- identifier: forward
class: File
contents: "TestData123"
- identifier: reverse
class: File
contents: "TestData123"
outputs:
out:
class: Collection
@@ -70,6 +70,7 @@ class TestWorkflow(ApiTestCase):
yaml_content,
test_data=test_job["job"],
history_id=history_id,
job_dir=str(workflow_path.parent),
)
if TEST_WORKFLOW_AFTER_RERUN:
run_summary = self.workflow_populator.rerun(run_summary)
@@ -1,10 +1,28 @@
- doc: |
- doc: |
Test that a basic list:list:list parameter is consumed directly by the tool
and not mapped over.
job:
input:
type: collection
class: Collection
collection_type: list:list:list
elements:
- identifier: test_level_2
class: Collection
type: list:list
elements:
- identifier: test_level_1
class: Collection
type: list
elements:
- identifier: data1
class: File
contents: "TestData123"
- identifier: data2
class: File
contents: "TestData123"
- identifier: data3
class: File
contents: "TestData123"
outputs:
out:
asserts:
@@ -13,12 +31,34 @@
- that: has_text
text: "collection_type<list:list>"
- doc: |
- doc: |
Test that mapping over a list:list:list input parameter.
job:
input:
type: collection
class: Collection
collection_type: list:list:list:list
elements:
- identifier: test_level_3
class: Collection
type: list:list:list
elements:
- identifier: test_level_2
class: Collection
type: list:list
elements:
- identifier: test_level_1
class: Collection
type: list
elements:
- identifier: data1
class: File
contents: "TestData123"
- identifier: data2
class: File
contents: "TestData123"
- identifier: data3
class: File
contents: "TestData123"
outputs:
out:
class: Collection
@@ -1,8 +1,12 @@
- doc: |
- doc: |
Test simple use of __ZIP_COLLECTION__ in a workflow.
job:
test_input_1: "samp1\t10.0\nsamp2\t20.0\n"
test_input_2: "samp1\t20.0\nsamp2\t40.0\n"
test_input_1:
class: File
contents: "samp1\t10.0\nsamp2\t20.0\n"
test_input_2:
class: File
contents: "samp1\t20.0\nsamp2\t40.0\n"
outputs:
out:
asserts:
+129 -1
View File
@@ -1,6 +1,11 @@
import tempfile
from galaxy.tool_util.cwl.util import output_properties
from galaxy.tool_util.cwl.util import (
FileLiteralTarget,
galactic_job_json,
output_properties,
UploadTarget,
)
def test_output_properties_in_memory():
@@ -23,3 +28,126 @@ def test_output_properties_path():
assert props["nameext"] == ".txt"
assert props["size"] == 11
assert props["checksum"] == "sha1$2aae6c35c94fcfb415dbe95f408b9ce91ee846ed"
def _mock_upload_func(upload_target: UploadTarget):
"""Mock upload that captures the target for inspection."""
dataset_id = f"dataset_{id(upload_target)}"
return {"outputs": [{"id": dataset_id}]}
def _mock_collection_create_func(element_identifiers, collection_type, rows=None, name=None):
return {"id": f"collection_{collection_type}"}
def test_galactic_job_json_file_literal_filetype():
"""FileLiteralTarget receives filetype when specified via class: File + contents."""
captured_targets = []
def upload_func(upload_target):
captured_targets.append(upload_target)
return _mock_upload_func(upload_target)
job = {
"input1": {
"class": "File",
"contents": "some text",
"filetype": "txt",
}
}
result_job, datasets = galactic_job_json(
job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow"
)
assert len(captured_targets) == 1
target = captured_targets[0]
assert isinstance(target, FileLiteralTarget)
assert target.contents == "some text"
assert target.properties.get("filetype") == "txt"
def test_galactic_job_json_file_literal_no_filetype():
"""FileLiteralTarget receives filetype=None when not specified."""
captured_targets = []
def upload_func(upload_target):
captured_targets.append(upload_target)
return _mock_upload_func(upload_target)
job = {
"input1": {
"class": "File",
"contents": "some text",
}
}
result_job, datasets = galactic_job_json(
job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow"
)
assert len(captured_targets) == 1
target = captured_targets[0]
assert isinstance(target, FileLiteralTarget)
assert target.properties.get("filetype") is None
def test_galactic_job_json_file_literal_tags_and_dbkey():
"""FileLiteralTarget receives tags and dbkey."""
captured_targets = []
def upload_func(upload_target):
captured_targets.append(upload_target)
return _mock_upload_func(upload_target)
job = {
"input1": {
"class": "File",
"contents": "some text",
"filetype": "fastq",
"tags": ["group:sample1"],
"dbkey": "hg38",
}
}
result_job, datasets = galactic_job_json(
job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow"
)
assert len(captured_targets) == 1
target = captured_targets[0]
assert isinstance(target, FileLiteralTarget)
assert target.properties["filetype"] == "fastq"
assert target.properties["tags"] == ["group:sample1"]
assert target.properties["dbkey"] == "hg38"
def test_galactic_job_json_collection_element_filetype():
"""Collection elements with class: File + contents get filetype forwarded."""
captured_targets = []
def upload_func(upload_target):
captured_targets.append(upload_target)
return _mock_upload_func(upload_target)
job = {
"reads": {
"class": "Collection",
"collection_type": "paired",
"elements": [
{
"identifier": "forward",
"class": "File",
"contents": "forward reads",
"filetype": "fastqsanger",
},
{
"identifier": "reverse",
"class": "File",
"contents": "reverse reads",
"filetype": "fastqsanger",
},
],
}
}
result_job, datasets = galactic_job_json(
job, ".", upload_func, _mock_collection_create_func, tool_or_workflow="workflow"
)
assert len(captured_targets) == 2
for target in captured_targets:
assert isinstance(target, FileLiteralTarget)
assert target.properties.get("filetype") == "fastqsanger"