Merge pull request #23084 from guerler/toolrequests.002

Align async tool requests with sync execution

lib/galaxy/managers/hdas.py

@@ -184,6 +184,7 @@ class HDAManager(
             file_sources=self.app.file_sources,
             sa_session=session,
             user_context=user_context,
+            datatypes_registry=self.app.datatypes_registry,
         )
         if request.source == DatasetSourceType.hda:
             dataset_instance: HistoryDatasetAssociation | LibraryDatasetDatasetAssociation = self.get_accessible(

lib/galaxy/model/deferred.py

@@ -9,8 +9,10 @@ from typing import (
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import DetachedInstanceError
 
+from galaxy.datatypes.registry import Registry
 from galaxy.datatypes.sniff import (
     convert_function,
+    guess_ext,
     stream_url_to_file,
 )
 from galaxy.exceptions import ObjectAttributeInvalidException
@@ -76,6 +78,7 @@ class DatasetInstanceMaterializer:
     def __init__(
         self,
         attached: bool,
+        datatypes_registry: Registry,
         object_store_populator: ObjectStorePopulator | None = None,
         transient_path_mapper: TransientPathMapper | None = None,
         file_sources: ConfiguredFileSources | None = None,
@@ -91,6 +94,9 @@ class DatasetInstanceMaterializer:
         ``user_context`` is forwarded to file source operations so that access
         controls (``requires_roles`` / ``requires_groups``) are enforced when
         materializing from ``gxfiles://`` URIs.
+
+        ``datatypes_registry`` enables content sniffing for deferred datasets whose
+        extension is ``"auto"``.
         """
         self._attached = attached
         self._transient_path_mapper = transient_path_mapper
@@ -98,6 +104,7 @@ class DatasetInstanceMaterializer:
         self._file_sources = file_sources
         self._sa_session = sa_session
         self._user_context = user_context
+        self._datatypes_registry = datatypes_registry
         self._previously_materialized: dict[int, HistoryDatasetAssociation] = {}
 
     def ensure_materialized(
@@ -189,6 +196,7 @@ class DatasetInstanceMaterializer:
             if not replacement_dataset:
                 try:
                     path = self._stream_source(target_source, dataset_instance.datatype, materialized_dataset)
+                    self._sniff_deferred_extension(path, dataset_instance)
                     object_store.update_from_file(materialized_dataset, file_name=path)
                     materialized_dataset.set_size()
                 except Exception as e:
@@ -201,6 +209,7 @@ class DatasetInstanceMaterializer:
             # TODO: take into account transform and ensure we are and are not modifying the file as appropriate.
             try:
                 path = self._stream_source(target_source, dataset_instance.datatype, materialized_dataset)
+                self._sniff_deferred_extension(path, dataset_instance)
                 shutil.move(path, transient_paths.external_filename)
                 materialized_dataset.external_filename = transient_paths.external_filename
             except Exception as e:
@@ -256,6 +265,23 @@ class DatasetInstanceMaterializer:
         self._previously_materialized[dataset_instance.id] = materialized_dataset_instance
         return materialized_dataset_instance
 
+    def _sniff_deferred_extension(
+        self,
+        path: str,
+        dataset_instance: HistoryDatasetAssociation | LibraryDatasetDatasetAssociation,
+    ) -> None:
+        """Resolve ``extension="auto"`` once the deferred source is streamed to ``path``.
+
+        A deferred fetch/upload with no explicit ``ext`` is registered as ``auto`` because
+        the content is not available to sniff at request time. ``path`` is the first point
+        the bytes exist locally, so we sniff here while materializing.
+        """
+        if dataset_instance.extension != "auto":
+            return
+        sniffed = guess_ext(path, self._datatypes_registry.sniff_order)
+        if sniffed and sniffed != "auto":
+            dataset_instance.extension = sniffed
+
     def _stream_source(self, target_source: DatasetSource, datatype, dataset: Dataset) -> str:
         source_uri = target_source.source_uri
         if source_uri is None:
@@ -387,6 +413,7 @@ def _materialize_collection_element(
 
 def materializer_factory(
     attached: bool,
+    datatypes_registry: Registry,
     object_store: ObjectStore | None = None,
     object_store_populator: ObjectStorePopulator | None = None,
     transient_path_mapper: TransientPathMapper | None = None,
@@ -401,6 +428,7 @@ def materializer_factory(
         transient_path_mapper = SimpleTransientPathMapper(transient_directory)
     return DatasetInstanceMaterializer(
         attached,
+        datatypes_registry,
         object_store_populator=object_store_populator,
         transient_path_mapper=transient_path_mapper,
         file_sources=file_sources,

lib/galaxy/tool_util/parameters/case.py

@@ -96,18 +96,23 @@ def legacy_from_string(parameter: ToolParameterT, value: Any | None, warnings: l
     """
     result_value: Any = value
     if isinstance(value, str):
-        if isinstance(parameter, (IntegerParameterModel,)):
-            if WARN_ON_UNTYPED_XML_STRINGS:
-                warnings.append(
-                    f"Implicitly converted {parameter.name} to an integer from a string value, please use 'value_json' to define this test input parameter value instead."
-                )
-            result_value = int(value)
-        elif isinstance(parameter, (FloatParameterModel,)):
-            if WARN_ON_UNTYPED_XML_STRINGS:
-                warnings.append(
-                    f"Implicitly converted {parameter.name} to a floating point number from a string value, please use 'value_json' to define this test input parameter value instead."
-                )
-            result_value = float(value)
+        if isinstance(parameter, (IntegerParameterModel, FloatParameterModel)):
+            # ``value=""`` on a numeric param is the legacy "not set" convention; emit None
+            # rather than raising on int("")/float("").
+            if value == "":
+                result_value = None
+            elif isinstance(parameter, IntegerParameterModel):
+                if WARN_ON_UNTYPED_XML_STRINGS:
+                    warnings.append(
+                        f"Implicitly converted {parameter.name} to an integer from a string value, please use 'value_json' to define this test input parameter value instead."
+                    )
+                result_value = int(value)
+            else:
+                if WARN_ON_UNTYPED_XML_STRINGS:
+                    warnings.append(
+                        f"Implicitly converted {parameter.name} to a floating point number from a string value, please use 'value_json' to define this test input parameter value instead."
+                    )
+                result_value = float(value)
         elif isinstance(parameter, (BooleanParameterModel,)):
             if WARN_ON_UNTYPED_XML_STRINGS:
                 warnings.append(

lib/galaxy/tool_util/parameters/convert.py

@@ -14,6 +14,7 @@ from typing import (
 
 from galaxy.tool_util_models.parameters import (
     BooleanParameterModel,
+    ColorParameterModel,
     ConditionalParameterModel,
     ConditionalWhen,
     create_job_runtime_model,
@@ -491,7 +492,7 @@ def _fill_default_for(tool_state: dict[str, Any], parameter: ToolParameterT) ->
             # see test_tools.py -> expression_null_handling_boolean or test cases for gx_boolean_optional.xml
             tool_state[parameter_name] = parameter.value or False
 
-    if isinstance(parameter, (IntegerParameterModel, FloatParameterModel, HiddenParameterModel)):
+    if isinstance(parameter, (IntegerParameterModel, FloatParameterModel, HiddenParameterModel, ColorParameterModel)):
         if parameter_name not in tool_state:
             tool_state[parameter_name] = parameter.value
     elif isinstance(parameter, GenomeBuildParameterModel):

lib/galaxy/tool_util/parameters/factory.py

@@ -198,11 +198,16 @@ def _from_input_source_galaxy(input_source: InputSource, profile: float) -> Tool
             )
         elif param_type == "color":
             optional = input_source.parse_optional()
+            color_value: str | None = get_color_value(input_source)
+            # A color default must be a valid color or None. The legacy ``value=""``
+            # on an optional color means "unset".
+            if optional and color_value == "":
+                color_value = None
             return ColorParameterModel(
                 type="color",
                 name=input_source.parse_name(),
                 optional=optional,
-                value=get_color_value(input_source),
+                value=color_value,
                 **_common_param_kwargs(input_source),
             )
         elif param_type == "rules":

lib/galaxy/tool_util_models/parameters.py

@@ -327,6 +327,11 @@ def _json_schema_extra_for_validators(validators: Sequence[VT]) -> dict[str, Any
             # Python re.match anchors at start; JSON Schema pattern does not
             if not pattern.startswith("^"):
                 pattern = "^" + pattern
+            # Python ``re`` (used by pydantic's ``pattern``) doesn't recognise POSIX
+            # character classes; defer such patterns to ``statically_validate`` which uses
+            # the ``regex`` module (seurat_plot).
+            if "[:" in pattern and ":]" in pattern:
+                continue
             extra["pattern"] = pattern
             break
     for v in validators:

lib/galaxy/tools/evaluation.py

@@ -309,6 +309,7 @@ class ToolEvaluator:
             transient_directory=transient_directory,
             file_sources=self.app.file_sources,
             user_context=user_context,
+            datatypes_registry=self.app.datatypes_registry,
         )
         for key, value in deferred_objects.items():
             if isinstance(value, model.DatasetInstance):

packages/tool_util/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "packaging",
     "pydantic>=2.7.4",
     "PyYAML",
+    "regex",
     "requests",
     "sortedcontainers",
     "typing-extensions",

packages/tool_util_models/pyproject.toml

@@ -15,6 +15,7 @@ authors = [
 dependencies = [
     "pydantic>=2.7.4",
     "pydantic-extra-types",
+    "regex",
     "typing-extensions",
 ]
 classifiers = [

test/functional/tools/async_posix_regex_validator.xml

@@ -0,0 +1,52 @@
+<tool id="async_posix_regex_validator" name="async_posix_regex_validator" version="1.0.0">
+    <description>async regression: POSIX character-class regex validators are skipped by pydantic and deferred to statically_validate</description>
+    <!--
+      Models seurat_plot test 2. The tool declares a text param with a regex
+      validator that uses POSIX character classes:
+
+          <validator type="regex">^[\w[:punct:]]+$</validator>
+
+      Sync validates via the ``regex`` module, which recognises POSIX classes and
+      accepts values like ``X16cell`` and ``midblast.9``.
+
+      Pydantic's ``pattern`` extra compiles through Python's ``re``, which does
+      NOT recognise POSIX classes and instead treats ``[:punct:]`` as a literal
+      character class of ``:``, ``p``, ``u``, ``n``, ``c``, ``t``. The resulting
+      pattern rejects perfectly legitimate values and the async request build
+      fails at pydantic time.
+
+      The framework fix: skip emitting the ``pattern`` extra when the validator
+      contains a POSIX character class, and defer validation to
+      ``statically_validate`` (which uses the ``regex`` module). Both sync and
+      async now accept the same values on this tool.
+    -->
+    <command><![CDATA[
+echo '$cell1' > '$out_file1'
+    ]]></command>
+    <inputs>
+        <param name="cell1" type="text" value="cell 1">
+            <validator type="regex" message="letters, numbers, or punctuation">^[\w[:punct:]]+$</validator>
+        </param>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="cell1" value="X16cell" />
+            <output name="out_file1">
+                <assert_contents>
+                    <has_text text="X16cell" />
+                </assert_contents>
+            </output>
+        </test>
+        <test>
+            <param name="cell1" value="midblast.9" />
+            <output name="out_file1">
+                <assert_contents>
+                    <has_text text="midblast.9" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/parameters/gx_color_optional_no_default.xml

@@ -0,0 +1,13 @@
+<tool id="gx_color_optional_no_default" name="gx_color_optional_no_default" version="1.0.0">
+    <!-- The legacy convention for "no default color" is value="" on an optional color
+         param (see e.g. tools-iuc arriba color1). ``_fill_default_for`` must not assign
+         the empty string as the default, or the job-internal color validator rejects
+         with "Invalid color value ''". Fill None (unset) instead. -->
+    <command>echo "${parameter}" > $out_file1</command>
+    <inputs>
+        <param name="parameter" type="color" value="" optional="true" />
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+</tool>

test/functional/tools/sample_tool_conf.xml

@@ -124,6 +124,7 @@
   <tool file="async_ambiguous_unqualified_name.xml" />
   <tool file="async_nested_conditional_shared_discriminator.xml" />
   <tool file="async_nested_conditional_shared_discriminator_bare.xml" />
+  <tool file="async_posix_regex_validator.xml" />
   <tool file="output_empty_work_dir.xml" />
   <tool file="output_filter_with_input.xml" />
   <tool file="output_filter_with_input_optional.xml" />

test/unit/data/test_dataset_materialization.py

@@ -36,7 +36,7 @@ def test_undeferred_hdas_untouched(tmpdir):
     hda = _create_hda(sa_session, app.object_store, history, hda_fh, include_metadata_file=False)
     sa_session.commit()
 
-    materializer = materializer_factory(True, object_store=app.object_store)
+    materializer = materializer_factory(True, object_store=app.object_store, datatypes_registry=app.datatypes_registry)
     assert materializer.ensure_materialized(hda) == hda
 
 
@@ -49,7 +49,11 @@ def test_deferred_hdas_basic_attached():
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -72,7 +76,11 @@ def test_hash_validate():
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -89,7 +97,11 @@ def test_hash_invalid():
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -138,7 +150,11 @@ def test_hash_validate_source_of_download():
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -157,7 +173,11 @@ def test_hash_invalid_source_of_download():
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -174,7 +194,11 @@ def test_deferred_hdas_basic_attached_store_by_uuid():
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -196,7 +220,11 @@ def test_deferred_hdas_basic_detached(tmpdir):
     _assert_2_bed_metadata(deferred_hda)
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -218,7 +246,11 @@ def test_deferred_datasets_with_legacy_transforms_respect_transform(tmpdir):
     assert deferred_hda.dataset.state == "deferred"
     assert deferred_hda.dataset.sources[0].transform is None
     assert deferred_hda.dataset.sources[0].requested_transform == [{"action": "spaces_to_tabs"}]
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -244,7 +276,11 @@ def test_deferred_datasets_with_requested_transforms_respect_transform(tmpdir):
         {"action": "datatype_groom"},
         {"action": "spaces_to_tabs"},
     ]
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -270,7 +306,11 @@ def test_deferred_datasets_do_not_apply_unspecified_transforms_legacy(tmpdir):
     assert deferred_hda.dataset.state == "deferred"
     assert deferred_hda.dataset.sources[0].transform is None
     assert deferred_hda.dataset.sources[0].requested_transform == []
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -293,7 +333,11 @@ def test_deferred_datasets_do_not_apply_unspecified_transforms(tmpdir):
     assert deferred_hda.dataset.state == "deferred"
     assert deferred_hda.dataset.sources[0].transform is None
     assert deferred_hda.dataset.sources[0].requested_transform == [{"action": "datatype_groom"}]
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -317,7 +361,11 @@ def test_deferred_hdas_basic_detached_from_detached_hda(tmpdir):
 
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -341,7 +389,10 @@ def test_deferred_hdas_basic_attached_from_detached_hda():
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
     materializer = materializer_factory(
-        True, object_store=fixture_context.app.object_store, sa_session=fixture_context.sa_session()
+        True,
+        object_store=fixture_context.app.object_store,
+        sa_session=fixture_context.sa_session(),
+        datatypes_registry=fixture_context.app.datatypes_registry,
     )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
@@ -368,7 +419,11 @@ def test_deferred_ldda_basic_attached():
     assert deferred_ldda.dataset is not None
     assert deferred_ldda.dataset.state == "deferred"
 
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_ldda)
     assert materialized_hda.history is None
     materialized_dataset = materialized_hda.dataset
@@ -397,7 +452,12 @@ def test_deferred_hdas_basic_attached_file_sources(tmpdir):
     assert deferred_hda
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store, file_sources=file_sources)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        file_sources=file_sources,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -419,7 +479,11 @@ def test_deferred_hdas_with_deferred_metadata():
     assert deferred_hda
     assert deferred_hda.dataset is not None
     assert deferred_hda.dataset.state == "deferred"
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     materialized_hda = materializer.ensure_materialized(deferred_hda)
     materialized_dataset = materialized_hda.dataset
     assert materialized_dataset is not None
@@ -434,9 +498,41 @@ def test_deferred_hdas_with_deferred_metadata():
     _assert_2_bed_metadata(materialized_hda)
 
 
+def test_deferred_hda_with_auto_extension_gets_sniffed_from_content():
+    # Scenario: a deferred fetch/upload with no explicit ``ext`` is stored as
+    # extension="auto" because the bytes aren't available to sniff at request time
+    # (data_fetch.py: ``requested_ext = item.get("ext", "auto")``). When the dataset is
+    # later materialized for a tool run, the real type is sniffed from the downloaded
+    # content. Requires a datatypes_registry.
+    fixture_context = setup_fixture_context_with_history()
+    store_dict = deferred_hda_model_store_dict()
+    store_dict["datasets"][0]["extension"] = "auto"
+    perform_import_from_store_dict(fixture_context, store_dict)
+    deferred_hda = fixture_context.history.datasets[0]
+    assert deferred_hda.extension == "auto"
+    assert deferred_hda.dataset is not None
+    assert deferred_hda.dataset.state == "deferred"
+
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
+    materialized_hda = materializer.ensure_materialized(deferred_hda)
+
+    assert materialized_hda.dataset is not None
+    assert materialized_hda.dataset.state == "ok"
+    # The fixture source is a .bed file; content sniffing must upgrade "auto" to "bed".
+    assert materialized_hda.extension == "bed"
+
+
 def test_materialize_attached_hdcas_unimplemented(tmpdir):
     fixture_context = setup_fixture_context_with_history()
-    materializer = materializer_factory(True, object_store=fixture_context.app.object_store)
+    materializer = materializer_factory(
+        True,
+        object_store=fixture_context.app.object_store,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     hdca = _test_hdca(tmpdir, fixture_context)
     exception_found = False
     try:
@@ -448,7 +544,11 @@ def test_materialize_attached_hdcas_unimplemented(tmpdir):
 
 def test_materialize_unattached_undeferred_hdcas_noop(tmpdir):
     fixture_context = setup_fixture_context_with_history()
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     input_hdca = _test_hdca(tmpdir, fixture_context, include_element_deferred=False)
     materialized_hdca = materialize_collection_instance(input_hdca, materializer)
     assert input_hdca == materialized_hdca  # doesn't have deferred data so just assert it is input.
@@ -456,7 +556,11 @@ def test_materialize_unattached_undeferred_hdcas_noop(tmpdir):
 
 def test_materialize_unattached_deferred_hdcas(tmpdir):
     fixture_context = setup_fixture_context_with_history()
-    materializer = materializer_factory(False, transient_directory=tmpdir)
+    materializer = materializer_factory(
+        False,
+        transient_directory=tmpdir,
+        datatypes_registry=fixture_context.app.datatypes_registry,
+    )
     deferred_hdca = _test_hdca(tmpdir, fixture_context)
     assert deferred_hdca.has_deferred_data
     assert len(deferred_hdca.collection.elements) == 2

test/unit/tool_util/test_parameter_convert.py

@@ -337,6 +337,13 @@ def test_fill_defaults():
     with_defaults = fill_state_for({}, "parameters/gx_genomebuild_optional")
     assert with_defaults["parameter"] is None
 
+    # ``<param type="color" value="" optional="true">`` is the legacy "no default color"
+    # convention (see tools-iuc arriba color1). The empty string default would trip the
+    # color validator on the job-internal model, so ``_fill_default_for`` must emit None
+    # instead. Regression for the async CI ``Invalid color value ''`` failures.
+    with_defaults = fill_state_for({}, "parameters/gx_color_optional_no_default")
+    assert with_defaults["parameter"] is None
+
     with_defaults = fill_state_for({}, "parameters/gx_select")
     assert with_defaults["parameter"] == "--ex1"
 

test/unit/tool_util/test_parameter_test_cases.py

@@ -628,6 +628,41 @@ def test_legacy_boolean_test_values_are_coerced_to_booleans():
     assert flag_value_for("-") is False
 
 
+def test_legacy_numeric_empty_test_values_are_none():
+    # An optional numeric param given value="" in a test is the legacy "not set" convention.
+    # The test-case builder must emit None rather than raising on int("")/float(""). Regression
+    # for async failures on macs2 (macs2_predictd tsize), ucsc_blat (blat), vegan
+    # (vegan_rarefaction sample_size), and vsearch (dereplication topn).
+    tool_template = """
+<tool id="numeric_legacy_empty" name="numeric_legacy_empty" version="1.0.0" profile="23.02">
+    <command>echo</command>
+    <inputs>
+        <param name="int_param" type="integer" value="" optional="true" />
+        <param name="float_param" type="float" value="" optional="true" />
+    </inputs>
+    <outputs />
+    <tests>
+        <test><param name="int_param" value="{int_value}" /><param name="float_param" value="{float_value}" /></test>
+    </tests>
+</tool>
+        """
+
+    def state_for(int_value: str, float_value: str):
+        tool_source = raw_xml_tool_source(tool_template.format(int_value=int_value, float_value=float_value))
+        parsed_tool = parse_tool(tool_source)
+        test_case = tool_source.parse_tests_to_dict()["tests"][0]
+        return case_state(test_case, parsed_tool.inputs, tool_source.parse_profile()).tool_state.input_state
+
+    # value="" -> None (was: raised int("")/float(""))
+    empty = state_for("", "")
+    assert empty["int_param"] is None
+    assert empty["float_param"] is None
+    # non-empty values still coerce normally
+    typed = state_for("5", "2.5")
+    assert typed["int_param"] == 5
+    assert typed["float_param"] == 2.5
+
+
 def test_legacy_unqualified_conditional_discriminator_in_section_is_resolved():
     # A conditional inside a section may have its name elided in the test, with the
     # discriminator given directly under the section (e.g. <section name="adv">