mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
narrow YAML tool parameter schema for UserToolSource
Introduce lib/galaxy/tool_util_models/yaml_parameters.py with a narrow YamlGalaxyToolParameter union (extra=forbid) used for UserToolSource.inputs and ToolSourceBase.inputs. v1 supports boolean, integer, float, text, select, color, data, data_collection, conditional, repeat, section. XML-only fields (truevalue, falsevalue, argument, is_dynamic, hidden, parameter_type) and unsupported types (hidden, drill_down, data_column, genomebuild, group_tag, baseurl, rules, directory) now reject at parse time. to_internal() maps each YAML model onto the existing internal parameter metamodel so no downstream pipeline needs to change shape. Production path keeps re-parsing via YamlToolSource (plan option a). UserToolSource envelope also gets extra=forbid - stray top-level keys like argument: at the tool level fail fast. Regenerated ToolSourceSchema.json is 32KB (was 77KB); Monaco no longer autocompletes the leaked fields. runtimeify() gains yaml_origin flag; evaluation.py passes it based on tool_source.parse_class(). assert_yaml_v1_parameters() hard-fails on any out-of-v1 type inside a YAML-origin bundle, recursing through whens/repeat/ section. XML path untouched. doc/source/admin/user_defined_tools.md documents the supported set and rejected types. 34 new tests in test_yaml_parameters.py cover red cases, green round-trips, structural groups, the runtimeify guard, and a snapshot blacklist on the published schema. test_parsing.py still green (83 tests). No compat shim for existing beta DB rows that contain XML-only fields - those will now fail validation on load and need to be recreated. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
John Chilton
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
51e94ba946
commit
ec5cfe6cce
@@ -81,7 +81,7 @@ function getToolBadges(tool: UnprivilegedToolResponse) {
|
||||
return [
|
||||
{
|
||||
id: "version",
|
||||
label: tool.representation.version,
|
||||
label: tool.representation.version ?? "",
|
||||
title: "Version of this custom tool",
|
||||
},
|
||||
];
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -93,6 +93,66 @@ See https://training.galaxyproject.org/training-material/topics/admin/tutorials/
|
||||
and see https://github.com/galaxyproject/galaxy/blob/dev/test/integration/embedded_pulsar_job_conf.yml#L29 for a simple example that uses embedded pulsar to isolate mounts and disables network access.
|
||||
While the feature is in beta we recommend that only trusted users are allowed to use this feature.
|
||||
|
||||
## Supported input parameter types
|
||||
|
||||
The YAML authoring layer exposes a deliberately narrow subset of Galaxy's
|
||||
parameter model. The JSON schema published to the editor (`ToolSourceSchema.json`)
|
||||
is generated from this subset and rejects any unknown fields via
|
||||
`extra="forbid"`, so unsupported attributes fail fast at parse time.
|
||||
|
||||
Supported leaf parameter types:
|
||||
|
||||
| Type | Supported fields (beyond `name`, `label`, `help`, `optional`) |
|
||||
| ----------------- | ---------------------------------------------------------------------- |
|
||||
| `boolean` | `value` |
|
||||
| `integer` | `value`, `min`, `max`, `validators` (`in_range` only) |
|
||||
| `float` | `value`, `min`, `max`, `validators` (`in_range` only) |
|
||||
| `text` | `value`, `area`, `validators` (`length`, `regex`, `empty_field`) |
|
||||
| `select` | `options` (static, non-empty), `multiple`, `validators` (`no_options`) |
|
||||
| `color` | `value` |
|
||||
| `data` | `format`, `multiple`, `min`, `max` |
|
||||
| `data_collection` | `collection_type`, `format` |
|
||||
|
||||
Supported structural groups: `conditional`, `repeat`, `section`. These recurse
|
||||
into the supported leaf set.
|
||||
|
||||
Example using several supported types:
|
||||
|
||||
```yaml
|
||||
class: GalaxyUserTool
|
||||
id: example_tool
|
||||
version: "0.1"
|
||||
name: Example
|
||||
container: busybox
|
||||
shell_command: echo "$(inputs.greeting) $(inputs.count)"
|
||||
inputs:
|
||||
- name: greeting
|
||||
type: select
|
||||
options:
|
||||
- { label: Hi, value: hi, selected: true }
|
||||
- { label: Hello, value: hello, selected: false }
|
||||
- name: count
|
||||
type: integer
|
||||
value: 1
|
||||
min: 1
|
||||
max: 10
|
||||
- name: extras
|
||||
type: repeat
|
||||
min: 0
|
||||
parameters:
|
||||
- name: input_file
|
||||
type: data
|
||||
format: txt
|
||||
outputs: []
|
||||
```
|
||||
|
||||
Types that exist in the XML tool vocabulary but are **not** supported in YAML
|
||||
user-defined tools and will be rejected at parse time: `hidden`, `drill_down`,
|
||||
`data_column`, `genomebuild`, `group_tag`, `baseurl`, `rules`, `directory`.
|
||||
XML-only fields such as `truevalue`, `falsevalue`, `argument`, `is_dynamic`,
|
||||
`hidden` (as a field), and `parameter_type` are likewise rejected on any
|
||||
parameter.
|
||||
|
||||
## Limitations
|
||||
|
||||
The user-defined tool language is still evolving, and additional safety audits are ongoing.
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import (
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
Sequence,
|
||||
)
|
||||
|
||||
from galaxy.tool_util_models.parameters import (
|
||||
@@ -537,12 +538,58 @@ DatasetToRuntimeJson = Callable[[DataJobInternalT], DataInternalJson]
|
||||
CollectionToRuntimeJson = Callable[[DataCollectionRequestInternal, Optional[str]], Any]
|
||||
|
||||
|
||||
# Parameter models the narrow YAML authoring layer is allowed to produce.
|
||||
# Mirrors the v1 supported set in `lib/galaxy/tool_util_models/yaml_parameters.py`.
|
||||
# Anything outside this set in a YAML-origin tool indicates a bug in the
|
||||
# authoring → internal mapping and should hard-fail before we try to build
|
||||
# runtime state.
|
||||
YAML_V1_SUPPORTED_PARAMETER_MODELS: frozenset = frozenset(
|
||||
{
|
||||
BooleanParameterModel,
|
||||
IntegerParameterModel,
|
||||
FloatParameterModel,
|
||||
TextParameterModel,
|
||||
SelectParameterModel,
|
||||
DataParameterModel,
|
||||
DataCollectionParameterModel,
|
||||
ConditionalParameterModel,
|
||||
RepeatParameterModel,
|
||||
SectionParameterModel,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def assert_yaml_v1_parameters(parameters: Sequence[ToolParameterT]) -> None:
|
||||
"""Raise if any parameter (including nested ones) is outside the YAML v1 set.
|
||||
|
||||
Defense-in-depth: the narrow ``YamlGalaxyToolParameter`` cannot produce a
|
||||
disallowed type, so this guard should never fire for a legitimately
|
||||
constructed YAML tool. It exists to surface mapping bugs instead of letting
|
||||
an unsupported type silently pass through ``runtimeify``'s default-case
|
||||
``VISITOR_NO_REPLACEMENT``.
|
||||
"""
|
||||
for parameter in parameters:
|
||||
if type(parameter) not in YAML_V1_SUPPORTED_PARAMETER_MODELS:
|
||||
raise AssertionError(
|
||||
f"YAML-origin tool produced unsupported parameter type "
|
||||
f"{type(parameter).__name__} for parameter {getattr(parameter, 'name', '?')!r}"
|
||||
)
|
||||
if isinstance(parameter, ConditionalParameterModel):
|
||||
for when in parameter.whens:
|
||||
assert_yaml_v1_parameters(when.parameters)
|
||||
elif isinstance(parameter, (RepeatParameterModel, SectionParameterModel)):
|
||||
assert_yaml_v1_parameters(parameter.parameters)
|
||||
|
||||
|
||||
def runtimeify(
|
||||
internal_state: JobInternalToolState,
|
||||
input_models: ToolParameterBundle,
|
||||
adapt_dataset: DatasetToRuntimeJson,
|
||||
adapt_collection: CollectionToRuntimeJson,
|
||||
yaml_origin: bool = False,
|
||||
) -> JobRuntimeToolState:
|
||||
if yaml_origin:
|
||||
assert_yaml_v1_parameters(list(input_models.parameters))
|
||||
|
||||
def adapt_dict(value: dict):
|
||||
assert isinstance(value, dict), str(value)
|
||||
|
||||
@@ -26,6 +26,10 @@ from typing_extensions import (
|
||||
)
|
||||
|
||||
from galaxy.tool_util_models.parameter_validators import AnyValidatorModel
|
||||
from galaxy.tool_util_models.testing_types import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
)
|
||||
from galaxy.tool_util_models.tool_source import (
|
||||
BaseJsonTestCollectionDefCollectionElementDict,
|
||||
Citation,
|
||||
@@ -66,13 +70,6 @@ NOT_IMPLEMENTED_MESSAGE = "Galaxy tool format does not yet support this tool fea
|
||||
INPUT_CLASS_T = Literal["galaxy", "cwl"]
|
||||
|
||||
|
||||
class AssertionDict(TypedDict):
|
||||
tag: str
|
||||
attributes: Dict[str, Any]
|
||||
children: "AssertionList"
|
||||
|
||||
|
||||
AssertionList = Optional[List[AssertionDict]]
|
||||
XmlInt = Union[str, int]
|
||||
|
||||
|
||||
@@ -117,25 +114,6 @@ class ToolSourceTestInput(TypedDict):
|
||||
attributes: ToolSourceTestInputAttributes
|
||||
|
||||
|
||||
class DirectCredentialValue(TypedDict):
|
||||
"""Represents a credential value (variable or secret) provided directly."""
|
||||
|
||||
name: str
|
||||
value: str
|
||||
|
||||
|
||||
class _DirectCredentialRequired(TypedDict):
|
||||
name: str # Name of the credentials group
|
||||
variables: List[DirectCredentialValue]
|
||||
secrets: List[DirectCredentialValue]
|
||||
|
||||
|
||||
class DirectCredential(_DirectCredentialRequired, total=False):
|
||||
"""Represents a credential group with variables and secrets provided directly."""
|
||||
|
||||
version: str # Version of the credential definition (defaults to "1.0")
|
||||
|
||||
|
||||
ToolSourceTestInputs = List[ToolSourceTestInput]
|
||||
ToolSourceTestOutputs = List[ToolSourceTestOutput]
|
||||
TestSourceTestOutputColllection = Any
|
||||
|
||||
@@ -31,6 +31,10 @@ from galaxy.tool_util.parser.util import (
|
||||
DEFAULT_SORT,
|
||||
)
|
||||
from galaxy.tool_util_models.parameter_validators import AnyValidatorModel
|
||||
from galaxy.tool_util_models.testing_types import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
)
|
||||
from galaxy.tool_util_models.tool_source import (
|
||||
Citation,
|
||||
DrillDownOptionsDict,
|
||||
@@ -53,8 +57,6 @@ from galaxy.util import (
|
||||
xml_to_string,
|
||||
)
|
||||
from .interface import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
DrillDownDynamicOptions,
|
||||
DynamicOptions,
|
||||
InputSource,
|
||||
|
||||
@@ -32,6 +32,11 @@ from galaxy.tool_util_models.parameters import (
|
||||
ToolParameterBundleModel,
|
||||
ToolParameterT,
|
||||
)
|
||||
from galaxy.tool_util_models.testing_types import (
|
||||
AssertionDict,
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
)
|
||||
from galaxy.tool_util_models.tool_source import (
|
||||
HelpContent,
|
||||
JsonTestCollectionDefDict,
|
||||
@@ -40,9 +45,6 @@ from galaxy.tool_util_models.tool_source import (
|
||||
)
|
||||
from galaxy.util import listify
|
||||
from .interface import (
|
||||
AssertionDict,
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
InputSource,
|
||||
PageSource,
|
||||
PagesSource,
|
||||
@@ -273,7 +275,7 @@ class YamlToolSource(ToolSource):
|
||||
tests: List[ToolSourceTest] = []
|
||||
rval: ToolSourceTests = dict(tests=tests)
|
||||
|
||||
raw_tests = deepcopy(self.root_dict.get("tests", []))
|
||||
raw_tests = deepcopy(self.root_dict.get("tests") or [])
|
||||
for i, test_dict in enumerate(raw_tests):
|
||||
inputs = test_dict.get("inputs", {})
|
||||
state = TestCaseJsonToolState(inputs)
|
||||
@@ -530,7 +532,11 @@ class YamlInputSource(InputSource):
|
||||
def parse_extensions(self):
|
||||
extensions = self.input_dict.get("extensions")
|
||||
if not extensions:
|
||||
extensions = self.get("format", "data").split(",")
|
||||
format_raw = self.get("format", "data")
|
||||
if isinstance(format_raw, str):
|
||||
extensions = format_raw.split(",")
|
||||
else:
|
||||
extensions = format_raw
|
||||
return [ext.strip().lower() for ext in extensions]
|
||||
|
||||
def parse_nested_inputs_source(self):
|
||||
|
||||
@@ -15,11 +15,13 @@ from typing_extensions import (
|
||||
)
|
||||
|
||||
from galaxy.tool_util.parser.interface import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
TestSourceTestOutputColllection,
|
||||
ToolSourceTestOutputs,
|
||||
)
|
||||
from galaxy.tool_util_models.testing_types import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
)
|
||||
|
||||
# legacy inputs for working with POST /api/tools
|
||||
# + inputs that have been processed with parse.py and expanded out
|
||||
|
||||
@@ -46,8 +46,6 @@ from galaxy.tool_util.parameters import (
|
||||
ToolParameterBundle,
|
||||
)
|
||||
from galaxy.tool_util.parser.interface import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
TestCollectionDef,
|
||||
TestCollectionOutputDef,
|
||||
TestSourceTestOutputColllection,
|
||||
@@ -55,6 +53,10 @@ from galaxy.tool_util.parser.interface import (
|
||||
XmlTestCollectionDefDict,
|
||||
)
|
||||
from galaxy.tool_util.verify.test_data import TestDataResolver
|
||||
from galaxy.tool_util_models.testing_types import (
|
||||
AssertionList,
|
||||
DirectCredential,
|
||||
)
|
||||
from galaxy.tool_util_models.tool_source import (
|
||||
JsonTestCollectionDefDict,
|
||||
JsonTestDatasetDefDict,
|
||||
|
||||
@@ -28,7 +28,6 @@ from typing_extensions import (
|
||||
TypedDict,
|
||||
)
|
||||
|
||||
from galaxy.tool_util_models.parameters import GalaxyToolParameterModel
|
||||
from ._base import ToolSourceBaseModel
|
||||
from .assertions import assertions
|
||||
from .parameters import ToolParameterT
|
||||
@@ -46,6 +45,7 @@ from .tool_source import (
|
||||
XrefDict,
|
||||
YamlTemplateConfigFile,
|
||||
)
|
||||
from .yaml_parameters import YamlGalaxyToolParameter
|
||||
|
||||
|
||||
def normalize_dict(values, keys: List[str]):
|
||||
@@ -56,44 +56,24 @@ def normalize_dict(values, keys: List[str]):
|
||||
values[key] = [{"name": k, **v} for k, v in items.items()]
|
||||
|
||||
|
||||
class ToolSourceBase(ToolSourceBaseModel):
|
||||
id: Optional[str] = None
|
||||
name: Optional[str] = None
|
||||
version: Optional[str] = "1.0"
|
||||
profile: Optional[float] = None
|
||||
description: Optional[str] = None
|
||||
container: Optional[str] = None
|
||||
requirements: Optional[List[Union[JavascriptRequirement, ResourceRequirement, ContainerRequirement]]] = []
|
||||
inputs: List[GalaxyToolParameterModel] = []
|
||||
outputs: List[IncomingToolOutput] = []
|
||||
citations: Optional[List[Citation]] = None
|
||||
license: Optional[str] = None
|
||||
edam_operations: Optional[List[str]] = None
|
||||
edam_topics: Optional[List[str]] = None
|
||||
xrefs: Optional[List[XrefDict]] = None
|
||||
help: Optional[HelpContent] = None
|
||||
class _DynamicToolSourceBase(ToolSourceBaseModel):
|
||||
# extra="forbid" rejects unknown top-level keys (e.g. a stray `argument:` at
|
||||
# the tool level), matching the strict-narrow stance on `inputs`.
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
field_title_generator=lambda field_name, field_info: field_name.lower(),
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def normalize_items(cls, values):
|
||||
if isinstance(values, dict):
|
||||
normalize_dict(values, ["inputs", "outputs"])
|
||||
return values
|
||||
|
||||
|
||||
# repeated fields to get consistent order, ugh, FIXME obviously
|
||||
class UserToolSource(ToolSourceBaseModel):
|
||||
class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")]
|
||||
id: Annotated[
|
||||
str,
|
||||
Optional[str],
|
||||
Field(
|
||||
description="Unique identifier for the tool. Should be all lower-case and should not include whitespace.",
|
||||
examples=["my-cool-tool"],
|
||||
min_length=3,
|
||||
max_length=255,
|
||||
),
|
||||
]
|
||||
version: Annotated[str, Field(description="Version for the tool.", examples=["0.1.0"])]
|
||||
] = None
|
||||
version: Annotated[Optional[str], Field(description="Version for the tool.", examples=["0.1.0"])] = None
|
||||
name: Annotated[
|
||||
str,
|
||||
Field(
|
||||
@@ -109,9 +89,6 @@ class UserToolSource(ToolSourceBaseModel):
|
||||
configfiles: Annotated[
|
||||
Optional[List[YamlTemplateConfigFile]], Field(description="A list of config files for this tool.")
|
||||
] = None
|
||||
container: Annotated[
|
||||
str, Field(description="Container image to use for this tool.", examples=["quay.io/biocontainers/python:3.13"])
|
||||
]
|
||||
requirements: Annotated[
|
||||
Optional[List[Union[JavascriptRequirement, ResourceRequirement, ContainerRequirement]]],
|
||||
Field(
|
||||
@@ -126,7 +103,7 @@ class UserToolSource(ToolSourceBaseModel):
|
||||
examples=["head -n '$(inputs.n_lines)' '$(inputs.data_input.path)'"],
|
||||
),
|
||||
]
|
||||
inputs: List[GalaxyToolParameterModel] = []
|
||||
inputs: List[YamlGalaxyToolParameter] = []
|
||||
outputs: List[IncomingToolOutput] = []
|
||||
citations: Optional[List[Citation]] = None
|
||||
license: Annotated[
|
||||
@@ -139,7 +116,9 @@ class UserToolSource(ToolSourceBaseModel):
|
||||
edam_operations: Optional[List[str]] = None
|
||||
edam_topics: Optional[List[str]] = None
|
||||
xrefs: Optional[List[XrefDict]] = None
|
||||
profile: Optional[float] = None
|
||||
help: Annotated[Optional[HelpContent], Field(description="Help text shown below the tool interface.")] = None
|
||||
tests: Optional[List["YamlToolTest"]] = None
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
@@ -149,12 +128,25 @@ class UserToolSource(ToolSourceBaseModel):
|
||||
return values
|
||||
|
||||
|
||||
class AdminToolSource(ToolSourceBase):
|
||||
class UserToolSource(_DynamicToolSourceBase):
|
||||
class_: Annotated[Literal["GalaxyUserTool"], Field(alias="class")]
|
||||
container: Annotated[
|
||||
str, Field(description="Container image to use for this tool.", examples=["quay.io/biocontainers/python:3.13"])
|
||||
]
|
||||
|
||||
|
||||
class YamlToolSource(_DynamicToolSourceBase):
|
||||
class_: Annotated[Literal["GalaxyTool"], Field(alias="class")]
|
||||
command: str
|
||||
container: Annotated[
|
||||
Optional[str],
|
||||
Field(
|
||||
description="Container image to use for this tool.",
|
||||
examples=["quay.io/biocontainers/python:3.13"],
|
||||
),
|
||||
] = None
|
||||
|
||||
|
||||
DynamicToolSources = Annotated[Union[UserToolSource, AdminToolSource], Field(discriminator="class_")]
|
||||
DynamicToolSources = Annotated[Union[UserToolSource, YamlToolSource], Field(discriminator="class_")]
|
||||
|
||||
|
||||
class ParsedTool(ToolSourceBaseModel):
|
||||
@@ -243,6 +235,78 @@ TestOutputLiteral = Union[bool, int, float, str]
|
||||
|
||||
TestOutputAssertions = Union[TestCollectionOutputAssertions, TestDataOutputAssertions, TestOutputLiteral]
|
||||
|
||||
|
||||
TestInputValue = Union[bool, int, float, str, List[Any], Dict[str, Any]]
|
||||
|
||||
|
||||
class YamlTestCredentialValue(StrictModel):
|
||||
name: Annotated[str, Field(description="Name of the credential variable or secret.")]
|
||||
value: Annotated[str, Field(description="Value of the credential variable or secret.")]
|
||||
|
||||
|
||||
class YamlTestCredential(StrictModel):
|
||||
name: Annotated[str, Field(description="Name of the credentials group.")]
|
||||
variables: Annotated[
|
||||
List[YamlTestCredentialValue],
|
||||
Field(description="Variables exposed to the tool environment."),
|
||||
] = []
|
||||
secrets: Annotated[
|
||||
List[YamlTestCredentialValue],
|
||||
Field(description="Secrets exposed to the tool environment."),
|
||||
] = []
|
||||
version: Annotated[
|
||||
Optional[str],
|
||||
Field(description="Version of the credential definition."),
|
||||
] = None
|
||||
|
||||
|
||||
class YamlToolTest(BaseModel):
|
||||
"""In-tool test case as authored in YAML tool fixtures."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
doc: Annotated[Optional[str], Field(description="Human-readable description of this test case.")] = None
|
||||
inputs: Annotated[
|
||||
Optional[Dict[str, TestInputValue]],
|
||||
Field(description="Mapping of input parameter names to test values."),
|
||||
] = None
|
||||
outputs: Annotated[
|
||||
Dict[str, TestOutputAssertions],
|
||||
Field(description="Mapping of output names to expected values or assertions."),
|
||||
] = {}
|
||||
assert_stdout: Annotated[
|
||||
Optional[assertions],
|
||||
Field(description="Assertions to apply against the tool's standard output."),
|
||||
] = None
|
||||
assert_stderr: Annotated[
|
||||
Optional[assertions],
|
||||
Field(description="Assertions to apply against the tool's standard error."),
|
||||
] = None
|
||||
command: Annotated[
|
||||
Optional[assertions],
|
||||
Field(description="Assertions to apply against the executed command line."),
|
||||
] = None
|
||||
expect_exit_code: Annotated[
|
||||
Optional[int],
|
||||
Field(description="Expected process exit code."),
|
||||
] = None
|
||||
expect_failure: Annotated[
|
||||
Optional[bool],
|
||||
Field(description="If true, the tool is expected to produce an error."),
|
||||
] = None
|
||||
expect_test_failure: Annotated[
|
||||
Optional[bool],
|
||||
Field(description="If true, the test itself is expected to fail."),
|
||||
] = None
|
||||
credentials: Annotated[
|
||||
Optional[List[YamlTestCredential]],
|
||||
Field(description="Credentials to inject for this test case."),
|
||||
] = None
|
||||
|
||||
|
||||
UserToolSource.model_rebuild()
|
||||
YamlToolSource.model_rebuild()
|
||||
|
||||
JobDict = Dict[str, Any]
|
||||
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ class base_has_line_matching_model_relaxed(AssertionModel):
|
||||
class has_line_matching_model(base_has_line_matching_model):
|
||||
r"""Asserts the specified output contains a line matching the
|
||||
regular expression specified by the argument expression. If n is given
|
||||
the assertion checks for exactly n occurences."""
|
||||
the assertion checks for exactly n occurrences."""
|
||||
|
||||
that: Literal["has_line_matching"] = "has_line_matching"
|
||||
|
||||
@@ -327,7 +327,7 @@ class has_line_matching_model_nested(AssertionModel):
|
||||
class has_line_matching_model_relaxed(base_has_line_matching_model_relaxed):
|
||||
r"""Asserts the specified output contains a line matching the
|
||||
regular expression specified by the argument expression. If n is given
|
||||
the assertion checks for exactly n occurences."""
|
||||
the assertion checks for exactly n occurrences."""
|
||||
|
||||
that: Literal["has_line_matching"] = "has_line_matching"
|
||||
|
||||
@@ -700,8 +700,8 @@ class base_has_text_matching_model_relaxed(AssertionModel):
|
||||
class has_text_matching_model(base_has_text_matching_model):
|
||||
r"""Asserts the specified output contains text matching the
|
||||
regular expression specified by the argument expression.
|
||||
If n is given the assertion checks for exacly n (nonoverlapping)
|
||||
occurences."""
|
||||
If n is given the assertion checks for exactly n (nonoverlapping)
|
||||
occurrences."""
|
||||
|
||||
that: Literal["has_text_matching"] = "has_text_matching"
|
||||
|
||||
@@ -715,8 +715,8 @@ class has_text_matching_model_nested(AssertionModel):
|
||||
class has_text_matching_model_relaxed(base_has_text_matching_model_relaxed):
|
||||
r"""Asserts the specified output contains text matching the
|
||||
regular expression specified by the argument expression.
|
||||
If n is given the assertion checks for exacly n (nonoverlapping)
|
||||
occurences."""
|
||||
If n is given the assertion checks for exactly n (nonoverlapping)
|
||||
occurrences."""
|
||||
|
||||
that: Literal["has_text_matching"] = "has_text_matching"
|
||||
|
||||
@@ -893,7 +893,7 @@ class has_n_columns_model(base_has_n_columns_model):
|
||||
|
||||
For instance, ``<has_n_columns n="3"/>``. The assertion tests only the first line.
|
||||
Number of columns can optionally also be specified with ``delta``. Alternatively the
|
||||
range of expected occurences can be specified by ``min`` and/or ``max``.
|
||||
range of expected occurrences can be specified by ``min`` and/or ``max``.
|
||||
|
||||
Optionally a column separator (``sep``, default is `` ``) `and comment character(s)
|
||||
can be specified (``comment``, default is empty string). The first non-comment
|
||||
@@ -914,7 +914,7 @@ class has_n_columns_model_relaxed(base_has_n_columns_model_relaxed):
|
||||
|
||||
For instance, ``<has_n_columns n="3"/>``. The assertion tests only the first line.
|
||||
Number of columns can optionally also be specified with ``delta``. Alternatively the
|
||||
range of expected occurences can be specified by ``min`` and/or ``max``.
|
||||
range of expected occurrences can be specified by ``min`` and/or ``max``.
|
||||
|
||||
Optionally a column separator (``sep``, default is `` ``) `and comment character(s)
|
||||
can be specified (``comment``, default is empty string). The first non-comment
|
||||
@@ -1585,7 +1585,7 @@ class has_n_elements_with_path_model(base_has_n_elements_with_path_model):
|
||||
```
|
||||
|
||||
Alternatively to ``n`` and ``delta`` also the ``min`` and ``max`` attributes
|
||||
can be used to specify the range of the expected number of occurences.
|
||||
can be used to specify the range of the expected number of occurrences.
|
||||
With ``negate`` the result of the assertion can be inverted."""
|
||||
|
||||
that: Literal["has_n_elements_with_path"] = "has_n_elements_with_path"
|
||||
@@ -1608,7 +1608,7 @@ class has_n_elements_with_path_model_relaxed(base_has_n_elements_with_path_model
|
||||
```
|
||||
|
||||
Alternatively to ``n`` and ``delta`` also the ``min`` and ``max`` attributes
|
||||
can be used to specify the range of the expected number of occurences.
|
||||
can be used to specify the range of the expected number of occurrences.
|
||||
With ``negate`` the result of the assertion can be inverted."""
|
||||
|
||||
that: Literal["has_n_elements_with_path"] = "has_n_elements_with_path"
|
||||
@@ -1786,7 +1786,7 @@ class base_xml_element_model_relaxed(AssertionModel):
|
||||
class xml_element_model(base_xml_element_model):
|
||||
r"""Assert if the XML file contains element(s) or tag(s) with the specified
|
||||
[XPath-like ``path``](https://lxml.de/xpathxslt.html). If ``n`` and ``delta``
|
||||
or ``min`` and ``max`` are given also the number of occurences is checked.
|
||||
or ``min`` and ``max`` are given also the number of occurrences is checked.
|
||||
|
||||
```xml
|
||||
<assert_contents>
|
||||
@@ -1796,7 +1796,7 @@ class xml_element_model(base_xml_element_model):
|
||||
</assert_contents>
|
||||
```
|
||||
|
||||
With ``negate="true"`` the outcome of the assertions wrt the precence and number
|
||||
With ``negate="true"`` the outcome of the assertions wrt the presence and number
|
||||
of ``path`` can be negated. If there are any sub assertions then check them against
|
||||
|
||||
- the content of the attribute ``attribute``
|
||||
@@ -1811,7 +1811,7 @@ class xml_element_model(base_xml_element_model):
|
||||
```
|
||||
|
||||
Sub-assertions are not subject to the ``negate`` attribute of ``xml_element``.
|
||||
If ``all`` is ``true`` then the sub assertions are checked for all occurences.
|
||||
If ``all`` is ``true`` then the sub assertions are checked for all occurrences.
|
||||
|
||||
Note that all other XML assertions can be expressed by this assertion (Galaxy
|
||||
also implements the other assertions by calling this one)."""
|
||||
@@ -1828,7 +1828,7 @@ class xml_element_model_nested(AssertionModel):
|
||||
class xml_element_model_relaxed(base_xml_element_model_relaxed):
|
||||
r"""Assert if the XML file contains element(s) or tag(s) with the specified
|
||||
[XPath-like ``path``](https://lxml.de/xpathxslt.html). If ``n`` and ``delta``
|
||||
or ``min`` and ``max`` are given also the number of occurences is checked.
|
||||
or ``min`` and ``max`` are given also the number of occurrences is checked.
|
||||
|
||||
```xml
|
||||
<assert_contents>
|
||||
@@ -1838,7 +1838,7 @@ class xml_element_model_relaxed(base_xml_element_model_relaxed):
|
||||
</assert_contents>
|
||||
```
|
||||
|
||||
With ``negate="true"`` the outcome of the assertions wrt the precence and number
|
||||
With ``negate="true"`` the outcome of the assertions wrt the presence and number
|
||||
of ``path`` can be negated. If there are any sub assertions then check them against
|
||||
|
||||
- the content of the attribute ``attribute``
|
||||
@@ -1853,7 +1853,7 @@ class xml_element_model_relaxed(base_xml_element_model_relaxed):
|
||||
```
|
||||
|
||||
Sub-assertions are not subject to the ``negate`` attribute of ``xml_element``.
|
||||
If ``all`` is ``true`` then the sub assertions are checked for all occurences.
|
||||
If ``all`` is ``true`` then the sub assertions are checked for all occurrences.
|
||||
|
||||
Note that all other XML assertions can be expressed by this assertion (Galaxy
|
||||
also implements the other assertions by calling this one)."""
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Shared TypedDicts for tool test definitions and assertion structures.
|
||||
|
||||
These live in ``tool_util_models`` so both ``tool_util_models`` and
|
||||
``tool_util.parser`` can depend on them without creating a circular import.
|
||||
"""
|
||||
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
)
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
||||
class AssertionDict(TypedDict):
|
||||
tag: str
|
||||
attributes: Dict[str, Any]
|
||||
children: "AssertionList"
|
||||
|
||||
|
||||
AssertionList = Optional[List[AssertionDict]]
|
||||
|
||||
|
||||
class DirectCredentialValue(TypedDict):
|
||||
"""Represents a credential value (variable or secret) provided directly."""
|
||||
|
||||
name: str
|
||||
value: str
|
||||
|
||||
|
||||
class _DirectCredentialRequired(TypedDict):
|
||||
name: str
|
||||
variables: List[DirectCredentialValue]
|
||||
secrets: List[DirectCredentialValue]
|
||||
|
||||
|
||||
class DirectCredential(_DirectCredentialRequired, total=False):
|
||||
"""Represents a credential group with variables and secrets provided directly."""
|
||||
|
||||
version: str
|
||||
@@ -0,0 +1,324 @@
|
||||
"""Narrow YAML-facing tool parameter models.
|
||||
|
||||
`UserToolSource` and `YamlToolSource` use these for their `inputs` field instead
|
||||
of the full internal Galaxy XML metamodel union. The YAML layer is purely an
|
||||
authoring/publication surface: it validates what users may write in YAML tools and
|
||||
rejects XML-only fields and unsupported parameter types via ``extra="forbid"``.
|
||||
|
||||
Every model exposes ``to_internal()`` returning the matching internal
|
||||
``GalaxyParameterT`` instance so callers that need the internal metamodel (e.g.
|
||||
round-trip tests) can construct it directly without re-parsing through
|
||||
``YamlToolSource``. The primary production path still builds internal models via
|
||||
``input_models_for_tool_source`` from the raw validated dict, so ``to_internal()``
|
||||
is not load-bearing for execution today.
|
||||
"""
|
||||
|
||||
from typing import (
|
||||
List,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
field_validator,
|
||||
RootModel,
|
||||
)
|
||||
from typing_extensions import (
|
||||
Annotated,
|
||||
Literal,
|
||||
)
|
||||
|
||||
from .parameter_validators import (
|
||||
EmptyFieldParameterValidatorModel,
|
||||
InRangeParameterValidatorModel,
|
||||
LengthParameterValidatorModel,
|
||||
NoOptionsParameterValidatorModel,
|
||||
RegexParameterValidatorModel,
|
||||
)
|
||||
from .parameters import (
|
||||
BooleanParameterModel,
|
||||
ColorParameterModel,
|
||||
cond_test_parameter_default_value,
|
||||
ConditionalParameterModel,
|
||||
ConditionalWhen,
|
||||
DataCollectionParameterModel,
|
||||
DataParameterModel,
|
||||
FloatParameterModel,
|
||||
GalaxyParameterT,
|
||||
IntegerParameterModel,
|
||||
LabelValue,
|
||||
RepeatParameterModel,
|
||||
SectionParameterModel,
|
||||
SelectParameterModel,
|
||||
TextParameterModel,
|
||||
)
|
||||
|
||||
|
||||
class YamlLabelValue(BaseModel):
|
||||
"""YAML-friendly option model — ``selected`` defaults to ``False``."""
|
||||
|
||||
label: str
|
||||
value: str
|
||||
selected: bool = False
|
||||
|
||||
def to_internal(self) -> LabelValue:
|
||||
return LabelValue(label=self.label, value=self.value, selected=self.selected)
|
||||
|
||||
|
||||
# Narrow validator unions — drops XML-only validators like Expression.
|
||||
YamlTextValidators = Union[
|
||||
LengthParameterValidatorModel,
|
||||
RegexParameterValidatorModel,
|
||||
EmptyFieldParameterValidatorModel,
|
||||
]
|
||||
YamlNumberValidators = Union[InRangeParameterValidatorModel,]
|
||||
YamlSelectValidators = Union[NoOptionsParameterValidatorModel,]
|
||||
|
||||
|
||||
class _YamlParamBase(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
name: str
|
||||
label: Optional[str] = None
|
||||
help: Optional[str] = None
|
||||
optional: bool = False
|
||||
|
||||
|
||||
def _common_internal_kwargs(yaml_param: "_YamlParamBase") -> dict:
|
||||
kwargs: dict = {"name": yaml_param.name, "optional": yaml_param.optional}
|
||||
if yaml_param.label is not None:
|
||||
kwargs["label"] = yaml_param.label
|
||||
if yaml_param.help is not None:
|
||||
kwargs["help"] = yaml_param.help
|
||||
return kwargs
|
||||
|
||||
|
||||
class YamlBooleanParameter(_YamlParamBase):
|
||||
type: Literal["boolean"]
|
||||
value: Optional[bool] = False
|
||||
|
||||
def to_internal(self) -> BooleanParameterModel:
|
||||
return BooleanParameterModel(type="boolean", value=self.value, **_common_internal_kwargs(self))
|
||||
|
||||
|
||||
class YamlIntegerParameter(_YamlParamBase):
|
||||
type: Literal["integer"]
|
||||
value: Optional[int] = None
|
||||
min: Optional[int] = None
|
||||
max: Optional[int] = None
|
||||
validators: List[YamlNumberValidators] = []
|
||||
|
||||
def to_internal(self) -> IntegerParameterModel:
|
||||
return IntegerParameterModel(
|
||||
type="integer",
|
||||
value=self.value,
|
||||
min=self.min,
|
||||
max=self.max,
|
||||
validators=list(self.validators),
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlFloatParameter(_YamlParamBase):
|
||||
type: Literal["float"]
|
||||
value: Optional[float] = None
|
||||
min: Optional[float] = None
|
||||
max: Optional[float] = None
|
||||
validators: List[YamlNumberValidators] = []
|
||||
|
||||
def to_internal(self) -> FloatParameterModel:
|
||||
return FloatParameterModel(
|
||||
type="float",
|
||||
value=self.value,
|
||||
min=self.min,
|
||||
max=self.max,
|
||||
validators=list(self.validators),
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlTextParameter(_YamlParamBase):
|
||||
type: Literal["text"]
|
||||
value: Optional[str] = Field(default=None, alias="value")
|
||||
area: bool = False
|
||||
validators: List[YamlTextValidators] = []
|
||||
|
||||
def to_internal(self) -> TextParameterModel:
|
||||
return TextParameterModel(
|
||||
type="text",
|
||||
default_value=self.value,
|
||||
area=self.area,
|
||||
validators=list(self.validators),
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlSelectParameter(_YamlParamBase):
|
||||
type: Literal["select"]
|
||||
options: Annotated[List[YamlLabelValue], Field(min_length=1)]
|
||||
multiple: bool = False
|
||||
validators: List[YamlSelectValidators] = []
|
||||
|
||||
def to_internal(self) -> SelectParameterModel:
|
||||
return SelectParameterModel(
|
||||
type="select",
|
||||
options=[o.to_internal() for o in self.options],
|
||||
multiple=self.multiple,
|
||||
validators=list(self.validators),
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlColorParameter(_YamlParamBase):
|
||||
type: Literal["color"]
|
||||
value: Optional[str] = None
|
||||
|
||||
def to_internal(self) -> ColorParameterModel:
|
||||
return ColorParameterModel(type="color", value=self.value, **_common_internal_kwargs(self))
|
||||
|
||||
|
||||
def _split_format(v):
|
||||
# Accept the XML-style comma-separated string form (`format: "txt,tabular"`)
|
||||
# as well as a list. Internal `DataParameterModel.extensions` is always a list.
|
||||
if isinstance(v, str):
|
||||
return [ext.strip().lower() for ext in v.split(",") if ext.strip()]
|
||||
return v
|
||||
|
||||
|
||||
class YamlDataParameter(_YamlParamBase):
|
||||
type: Literal["data"]
|
||||
format: List[str] = ["data"]
|
||||
multiple: bool = False
|
||||
min: Optional[int] = None
|
||||
max: Optional[int] = None
|
||||
|
||||
@field_validator("format", mode="before")
|
||||
@classmethod
|
||||
def _coerce_format(cls, v):
|
||||
return _split_format(v)
|
||||
|
||||
def to_internal(self) -> DataParameterModel:
|
||||
return DataParameterModel(
|
||||
type="data",
|
||||
extensions=list(self.format),
|
||||
multiple=self.multiple,
|
||||
min=self.min,
|
||||
max=self.max,
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlDataCollectionParameter(_YamlParamBase):
|
||||
type: Literal["data_collection"]
|
||||
collection_type: Optional[str] = None
|
||||
format: List[str] = ["data"]
|
||||
|
||||
@field_validator("format", mode="before")
|
||||
@classmethod
|
||||
def _coerce_format(cls, v):
|
||||
return _split_format(v)
|
||||
|
||||
def to_internal(self) -> DataCollectionParameterModel:
|
||||
return DataCollectionParameterModel(
|
||||
type="data_collection",
|
||||
collection_type=self.collection_type,
|
||||
extensions=list(self.format),
|
||||
value=None,
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
YamlConditionalTestParameter = Annotated[Union[YamlBooleanParameter, YamlSelectParameter], Field(discriminator="type")]
|
||||
|
||||
|
||||
class YamlConditionalWhen(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
discriminator: Union[bool, str]
|
||||
parameters: List["YamlGalaxyToolParameter"] = []
|
||||
|
||||
|
||||
class YamlConditionalParameter(_YamlParamBase):
|
||||
type: Literal["conditional"]
|
||||
test_parameter: YamlConditionalTestParameter
|
||||
whens: Annotated[List[YamlConditionalWhen], Field(min_length=1)]
|
||||
|
||||
def to_internal(self) -> ConditionalParameterModel:
|
||||
internal_test = self.test_parameter.to_internal()
|
||||
default_value = cond_test_parameter_default_value(internal_test)
|
||||
internal_whens: List[ConditionalWhen] = []
|
||||
for when in self.whens:
|
||||
internal_params = [p.root.to_internal() for p in when.parameters]
|
||||
internal_whens.append(
|
||||
ConditionalWhen(
|
||||
discriminator=when.discriminator,
|
||||
parameters=internal_params,
|
||||
is_default_when=when.discriminator == default_value,
|
||||
)
|
||||
)
|
||||
return ConditionalParameterModel(
|
||||
type="conditional",
|
||||
test_parameter=internal_test,
|
||||
whens=internal_whens,
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlRepeatParameter(_YamlParamBase):
|
||||
type: Literal["repeat"]
|
||||
parameters: List["YamlGalaxyToolParameter"] = []
|
||||
min: Optional[int] = None
|
||||
max: Optional[int] = None
|
||||
|
||||
def to_internal(self) -> RepeatParameterModel:
|
||||
return RepeatParameterModel(
|
||||
type="repeat",
|
||||
parameters=[p.root.to_internal() for p in self.parameters],
|
||||
min=self.min,
|
||||
max=self.max,
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
class YamlSectionParameter(_YamlParamBase):
|
||||
type: Literal["section"]
|
||||
parameters: List["YamlGalaxyToolParameter"] = []
|
||||
|
||||
def to_internal(self) -> SectionParameterModel:
|
||||
return SectionParameterModel(
|
||||
type="section",
|
||||
parameters=[p.root.to_internal() for p in self.parameters],
|
||||
**_common_internal_kwargs(self),
|
||||
)
|
||||
|
||||
|
||||
YamlGalaxyParameterT = Union[
|
||||
YamlBooleanParameter,
|
||||
YamlIntegerParameter,
|
||||
YamlFloatParameter,
|
||||
YamlTextParameter,
|
||||
YamlSelectParameter,
|
||||
YamlColorParameter,
|
||||
YamlDataParameter,
|
||||
YamlDataCollectionParameter,
|
||||
YamlConditionalParameter,
|
||||
YamlRepeatParameter,
|
||||
YamlSectionParameter,
|
||||
]
|
||||
|
||||
|
||||
class YamlGalaxyToolParameter(RootModel):
|
||||
root: Annotated[YamlGalaxyParameterT, Field(discriminator="type")]
|
||||
|
||||
def to_internal(self) -> GalaxyParameterT:
|
||||
return self.root.to_internal()
|
||||
|
||||
|
||||
YamlConditionalWhen.model_rebuild()
|
||||
YamlConditionalParameter.model_rebuild()
|
||||
YamlRepeatParameter.model_rebuild()
|
||||
YamlSectionParameter.model_rebuild()
|
||||
YamlGalaxyToolParameter.model_rebuild()
|
||||
@@ -1099,7 +1099,14 @@ class UserToolEvaluator(ToolEvaluator):
|
||||
if self.tool.parameters is None:
|
||||
raise RequestParameterInvalidException(f"Tool {self.tool.id} has no parameters defined")
|
||||
parameter_bundle = ToolParameterBundleModel(parameters=self.tool.parameters)
|
||||
job_runtime_state = runtimeify(validated_tool_state, parameter_bundle, adapt_datasets, adapt_collections)
|
||||
yaml_origin = self.tool.tool_source.parse_class() in ("GalaxyUserTool", "GalaxyTool")
|
||||
job_runtime_state = runtimeify(
|
||||
validated_tool_state,
|
||||
parameter_bundle,
|
||||
adapt_datasets,
|
||||
adapt_collections,
|
||||
yaml_origin=yaml_origin,
|
||||
)
|
||||
cwl_style_inputs = job_runtime_state.input_state
|
||||
else:
|
||||
from galaxy.workflow.modules import to_cwl
|
||||
|
||||
@@ -43,17 +43,17 @@ MINIMAL_TOOL = {
|
||||
"name": "Minimal Tool",
|
||||
"class": "GalaxyTool",
|
||||
"version": "1.0.0",
|
||||
"command": "echo 'Hello World' > $output1",
|
||||
"shell_command": "echo 'Hello World' > 'output.txt'",
|
||||
"inputs": [],
|
||||
"outputs": {"output1": {"format": "txt", "type": "data"}},
|
||||
"outputs": {"output1": {"format": "txt", "type": "data", "from_work_dir": "output.txt"}},
|
||||
}
|
||||
MINIMAL_TOOL_NO_ID = {
|
||||
"name": "Minimal Tool",
|
||||
"class": "GalaxyTool",
|
||||
"version": "1.0.0",
|
||||
"command": "echo 'Hello World 2' > $output1",
|
||||
"shell_command": "echo 'Hello World 2' > 'output.txt'",
|
||||
"inputs": [],
|
||||
"outputs": {"output1": {"format": "txt", "type": "data"}},
|
||||
"outputs": {"output1": {"format": "txt", "type": "data", "from_work_dir": "output.txt"}},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -2,12 +2,11 @@ class: GalaxyUserTool
|
||||
id: gx_boolean_user
|
||||
version: "1.0.0"
|
||||
name: gx_boolean_user
|
||||
container: busybox
|
||||
shell_command: echo '$(inputs.parameter)' >> output.txt
|
||||
inputs:
|
||||
- name: parameter
|
||||
type: boolean
|
||||
truevalue: mytrue
|
||||
falsevalue: myfalse
|
||||
outputs:
|
||||
- name: output
|
||||
type: data
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
class: GalaxyTool
|
||||
class: GalaxyUserTool
|
||||
id: gx_data_collection_sample_sheet_y
|
||||
name: gx_data_collection_sample_sheet_y
|
||||
version: "1.0.0"
|
||||
|
||||
@@ -14,7 +14,12 @@ class TestDynamicToolManager(BaseToolBoxTestCase):
|
||||
def test_create_tool(self):
|
||||
tool_version = "0.1"
|
||||
payload = DynamicToolCreatePayload(
|
||||
representation={"class": "GalaxyTool", "version": tool_version, "command": "echo 42"}
|
||||
representation={
|
||||
"class": "GalaxyTool",
|
||||
"name": "Test Tool",
|
||||
"version": tool_version,
|
||||
"shell_command": "echo 42",
|
||||
}
|
||||
)
|
||||
dynamic_tool = self.dynamic_tool_manager.create_tool(payload)
|
||||
assert dynamic_tool.active
|
||||
@@ -23,6 +28,8 @@ class TestDynamicToolManager(BaseToolBoxTestCase):
|
||||
assert dynamic_tool.tool_version == tool_version
|
||||
|
||||
def test_create_tool_no_version(self):
|
||||
payload = DynamicToolCreatePayload(representation={"class": "GalaxyTool", "command": "echo 42"})
|
||||
payload = DynamicToolCreatePayload(
|
||||
representation={"class": "GalaxyTool", "name": "Test Tool", "shell_command": "echo 42"}
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
self.dynamic_tool_manager.create_tool(payload)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Validate every YAML tool fixture in ``test/functional/tools`` against the
|
||||
narrow authoring models (``UserToolSource`` / ``YamlToolSource``).
|
||||
|
||||
The existing ``test_validate_framework_test_tools`` in
|
||||
``test_parameter_test_cases.py`` only exercises the XML-era
|
||||
``YamlToolSource`` dict parser path; it does not touch the narrow pydantic
|
||||
authoring schema in ``lib/galaxy/tool_util_models/yaml_parameters.py``.
|
||||
This file closes that gap so a stray XML-only field (e.g. ``truevalue``) or
|
||||
a deferred parameter type added to any fixture fails a cheap unit test
|
||||
instead of only an API integration.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from galaxy.tool_util.unittest_utils import functional_test_tool_directory
|
||||
from galaxy.tool_util_models import DynamicToolSources
|
||||
|
||||
# Fixtures that predate the narrow authoring schema and use conventions
|
||||
# (``checked``, ``display``, ``blocks``, ``when``, unquoted version floats,
|
||||
# dict-style collection outputs) that cannot be represented via
|
||||
# ``UserToolSource`` / ``YamlToolSource``.
|
||||
_LEGACY_FIXTURES = {
|
||||
"simple_constructs.yml",
|
||||
"collection_creates_pair_y.yml",
|
||||
}
|
||||
|
||||
_dynamic_tool_source_adapter: TypeAdapter = TypeAdapter(DynamicToolSources)
|
||||
|
||||
|
||||
def _collect_yaml_tool_fixtures() -> List[str]:
|
||||
root = functional_test_tool_directory()
|
||||
directories = [root, os.path.join(root, "parameters")]
|
||||
paths: List[str] = []
|
||||
for directory in directories:
|
||||
if not os.path.isdir(directory):
|
||||
continue
|
||||
for name in sorted(os.listdir(directory)):
|
||||
if name in _LEGACY_FIXTURES:
|
||||
continue
|
||||
if not name.endswith(".yml"):
|
||||
continue
|
||||
full = os.path.join(directory, name)
|
||||
if os.path.isdir(full):
|
||||
continue
|
||||
paths.append(full)
|
||||
return paths
|
||||
|
||||
|
||||
_YAML_FIXTURES = _collect_yaml_tool_fixtures()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_path", _YAML_FIXTURES, ids=lambda p: os.path.relpath(p, functional_test_tool_directory())
|
||||
)
|
||||
def test_yaml_fixture_validates_against_authoring_schema(tool_path: str):
|
||||
with open(tool_path) as fh:
|
||||
raw = yaml.safe_load(fh)
|
||||
assert isinstance(raw, dict), f"{tool_path}: expected top-level mapping"
|
||||
assert "class" in raw, f"{tool_path}: missing `class:` key"
|
||||
# Validates against the UserToolSource / YamlToolSource union,
|
||||
# discriminated on ``class``.
|
||||
_dynamic_tool_source_adapter.validate_python(raw)
|
||||
|
||||
|
||||
def test_fixture_collection_is_non_empty():
|
||||
assert _YAML_FIXTURES, "no YAML tool fixtures discovered under test/functional/tools"
|
||||
@@ -0,0 +1,355 @@
|
||||
"""Tests for the narrow YAML tool parameter models.
|
||||
|
||||
Covers:
|
||||
- reject cases for XML-only fields and unsupported parameter types,
|
||||
- green round-trip from YAML authoring models through ``to_internal()`` into the
|
||||
existing internal metamodel, including the ``create_job_runtime_model`` path
|
||||
that backs ``/api/unprivileged_tools/runtime_model``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from galaxy.tool_util.parameters.convert import assert_yaml_v1_parameters
|
||||
from galaxy.tool_util_models import UserToolSource
|
||||
from galaxy.tool_util_models.parameters import (
|
||||
BooleanParameterModel,
|
||||
ConditionalParameterModel,
|
||||
create_job_runtime_model,
|
||||
DataParameterModel,
|
||||
HiddenParameterModel,
|
||||
RepeatParameterModel,
|
||||
SectionParameterModel,
|
||||
SelectParameterModel,
|
||||
ToolParameterBundleModel,
|
||||
)
|
||||
from galaxy.tool_util_models.yaml_parameters import YamlGalaxyToolParameter
|
||||
|
||||
|
||||
def _validate(input_dict):
|
||||
return YamlGalaxyToolParameter.model_validate(input_dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Red cases: XML-only fields rejected on otherwise-supported types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_field",
|
||||
[
|
||||
{"truevalue": "yes"},
|
||||
{"falsevalue": "no"},
|
||||
{"argument": "--foo"},
|
||||
{"is_dynamic": True},
|
||||
{"hidden": True},
|
||||
{"parameter_type": "gx_boolean"},
|
||||
],
|
||||
)
|
||||
def test_boolean_rejects_xml_only_fields(extra_field):
|
||||
with pytest.raises(ValidationError):
|
||||
_validate({"name": "b", "type": "boolean", **extra_field})
|
||||
|
||||
|
||||
def test_text_rejects_expression_validator():
|
||||
with pytest.raises(ValidationError):
|
||||
_validate(
|
||||
{
|
||||
"name": "t",
|
||||
"type": "text",
|
||||
"validators": [{"type": "expression", "expression": "value=='ok'"}],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_select_rejects_empty_options():
|
||||
with pytest.raises(ValidationError):
|
||||
_validate({"name": "s", "type": "select", "options": []})
|
||||
|
||||
|
||||
def test_select_rejects_dynamic_options():
|
||||
# No `dynamic_options` field on YamlSelectParameter → extra forbid.
|
||||
with pytest.raises(ValidationError):
|
||||
_validate(
|
||||
{
|
||||
"name": "s",
|
||||
"type": "select",
|
||||
"options": [{"label": "A", "value": "a", "selected": True}],
|
||||
"dynamic_options": "some_fn()",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Red cases: whole parameter types rejected
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_type",
|
||||
[
|
||||
"hidden",
|
||||
"drill_down",
|
||||
"data_column",
|
||||
"genomebuild",
|
||||
"group_tag",
|
||||
"baseurl",
|
||||
"rules",
|
||||
"directory",
|
||||
],
|
||||
)
|
||||
def test_unsupported_parameter_types_rejected(bad_type):
|
||||
with pytest.raises(ValidationError):
|
||||
_validate({"name": "x", "type": bad_type})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Green cases: leaf types round-trip through to_internal()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_boolean_roundtrip():
|
||||
p = _validate({"name": "b", "type": "boolean", "value": True})
|
||||
internal = p.to_internal()
|
||||
assert isinstance(internal, BooleanParameterModel)
|
||||
assert internal.value is True
|
||||
# YAML layer did not populate Cheetah-only fields on the internal model.
|
||||
assert internal.truevalue is None
|
||||
assert internal.falsevalue is None
|
||||
|
||||
|
||||
def test_integer_with_inrange_validator():
|
||||
p = _validate(
|
||||
{
|
||||
"name": "n",
|
||||
"type": "integer",
|
||||
"value": 5,
|
||||
"min": 0,
|
||||
"max": 10,
|
||||
"validators": [{"type": "in_range", "min": 0, "max": 10}],
|
||||
}
|
||||
)
|
||||
internal = p.to_internal()
|
||||
assert internal.min == 0
|
||||
assert internal.max == 10
|
||||
assert len(internal.validators) == 1
|
||||
|
||||
|
||||
def test_select_static_options():
|
||||
p = _validate(
|
||||
{
|
||||
"name": "s",
|
||||
"type": "select",
|
||||
"multiple": False,
|
||||
"options": [
|
||||
{"label": "A", "value": "a", "selected": True},
|
||||
{"label": "B", "value": "b", "selected": False},
|
||||
],
|
||||
}
|
||||
)
|
||||
internal = p.to_internal()
|
||||
assert isinstance(internal, SelectParameterModel)
|
||||
assert internal.default_value == "a"
|
||||
|
||||
|
||||
def test_data_accepts_format_string():
|
||||
# Matches the XML `format="txt"` vocabulary and the PR 19434 example.
|
||||
p = _validate({"name": "input1", "type": "data", "format": "txt"})
|
||||
internal = p.to_internal()
|
||||
assert isinstance(internal, DataParameterModel)
|
||||
assert internal.extensions == ["txt"]
|
||||
|
||||
|
||||
def test_data_accepts_format_list():
|
||||
p = _validate({"name": "input1", "type": "data", "format": ["txt", "tabular"]})
|
||||
assert p.to_internal().extensions == ["txt", "tabular"]
|
||||
|
||||
|
||||
def test_data_accepts_format_comma_string():
|
||||
p = _validate({"name": "input1", "type": "data", "format": "txt,tabular"})
|
||||
assert p.to_internal().extensions == ["txt", "tabular"]
|
||||
|
||||
|
||||
def test_data_rejects_extensions_key():
|
||||
# `extensions` is the internal metamodel name; the YAML authoring surface
|
||||
# exposes `format` only.
|
||||
with pytest.raises(ValidationError):
|
||||
_validate({"name": "input1", "type": "data", "extensions": ["txt"]})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Green cases: structural groups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_conditional_rejects_empty_whens():
|
||||
with pytest.raises(ValidationError):
|
||||
_validate(
|
||||
{
|
||||
"name": "cond",
|
||||
"type": "conditional",
|
||||
"test_parameter": {
|
||||
"name": "mode",
|
||||
"type": "select",
|
||||
"options": [{"label": "A", "value": "a", "selected": True}],
|
||||
},
|
||||
"whens": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_conditional_with_select_test_parameter():
|
||||
p = _validate(
|
||||
{
|
||||
"name": "cond",
|
||||
"type": "conditional",
|
||||
"test_parameter": {
|
||||
"name": "mode",
|
||||
"type": "select",
|
||||
"options": [
|
||||
{"label": "A", "value": "a", "selected": True},
|
||||
{"label": "B", "value": "b", "selected": False},
|
||||
],
|
||||
},
|
||||
"whens": [
|
||||
{
|
||||
"discriminator": "a",
|
||||
"parameters": [{"name": "x", "type": "text", "value": "hi"}],
|
||||
},
|
||||
{"discriminator": "b", "parameters": []},
|
||||
],
|
||||
}
|
||||
)
|
||||
internal = p.to_internal()
|
||||
assert isinstance(internal, ConditionalParameterModel)
|
||||
default_flags = {w.discriminator: w.is_default_when for w in internal.whens}
|
||||
assert default_flags == {"a": True, "b": False}
|
||||
|
||||
|
||||
def test_repeat_of_data():
|
||||
p = _validate(
|
||||
{
|
||||
"name": "rep",
|
||||
"type": "repeat",
|
||||
"min": 1,
|
||||
"max": 3,
|
||||
"parameters": [{"name": "input1", "type": "data", "format": "txt"}],
|
||||
}
|
||||
)
|
||||
internal = p.to_internal()
|
||||
assert isinstance(internal, RepeatParameterModel)
|
||||
assert internal.min == 1
|
||||
assert internal.max == 3
|
||||
assert isinstance(internal.parameters[0], DataParameterModel)
|
||||
|
||||
|
||||
def test_section_recurses():
|
||||
p = _validate(
|
||||
{
|
||||
"name": "sec",
|
||||
"type": "section",
|
||||
"parameters": [
|
||||
{"name": "a", "type": "integer", "value": 1},
|
||||
{"name": "b", "type": "boolean", "value": False},
|
||||
],
|
||||
}
|
||||
)
|
||||
internal = p.to_internal()
|
||||
assert isinstance(internal, SectionParameterModel)
|
||||
assert [p.name for p in internal.parameters] == ["a", "b"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Green: UserToolSource end-to-end for the PR 19434 example shape
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
CAT_USER_DEFINED = {
|
||||
"class": "GalaxyUserTool",
|
||||
"id": "cat_user_defined",
|
||||
"version": "0.1",
|
||||
"name": "cat_user_defined",
|
||||
"description": "concatenates a file",
|
||||
"container": "busybox",
|
||||
"shell_command": "cat '$(inputs.input1.path)' > output.txt",
|
||||
"inputs": [{"name": "input1", "type": "data", "format": "txt"}],
|
||||
"outputs": [],
|
||||
}
|
||||
|
||||
|
||||
def test_user_tool_source_rejects_unknown_top_level_key():
|
||||
bad = {**CAT_USER_DEFINED, "argument": "--nope"}
|
||||
with pytest.raises(ValidationError):
|
||||
UserToolSource.model_validate(bad)
|
||||
|
||||
|
||||
def test_user_tool_source_validates_pr19434_example():
|
||||
tool = UserToolSource.model_validate(CAT_USER_DEFINED)
|
||||
assert tool.inputs[0].root.type == "data"
|
||||
assert tool.inputs[0].root.format == ["txt"]
|
||||
|
||||
|
||||
def test_runtime_model_pipeline_from_yaml_internal():
|
||||
tool = UserToolSource.model_validate(CAT_USER_DEFINED)
|
||||
bundle = ToolParameterBundleModel(parameters=[i.to_internal() for i in tool.inputs])
|
||||
model = create_job_runtime_model(bundle)
|
||||
schema = model.model_json_schema()
|
||||
assert "input1" in schema["properties"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Snapshot: published ToolSourceSchema.json is free of XML-only leaks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_BLACKLIST_SUBSTRINGS = (
|
||||
"truevalue",
|
||||
"falsevalue",
|
||||
"argument",
|
||||
"is_dynamic",
|
||||
"parameter_type",
|
||||
"hierarchy",
|
||||
"data_ref",
|
||||
"gx_hidden",
|
||||
"gx_drill_down",
|
||||
"gx_genomebuild",
|
||||
"gx_group_tag",
|
||||
"gx_baseurl",
|
||||
"gx_rules",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 6: runtimeify enforces the v1 parameter allowlist for YAML-origin tools
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_assert_yaml_v1_parameters_accepts_supported_set():
|
||||
tool = UserToolSource.model_validate(CAT_USER_DEFINED)
|
||||
parameters = [i.to_internal() for i in tool.inputs]
|
||||
# should not raise
|
||||
assert_yaml_v1_parameters(parameters)
|
||||
|
||||
|
||||
def test_assert_yaml_v1_parameters_rejects_deferred_type():
|
||||
hidden = HiddenParameterModel(type="hidden", name="h", value=None)
|
||||
with pytest.raises(AssertionError):
|
||||
assert_yaml_v1_parameters([hidden])
|
||||
|
||||
|
||||
def test_assert_yaml_v1_parameters_walks_nested_groups():
|
||||
hidden = HiddenParameterModel(type="hidden", name="h", value=None)
|
||||
repeat = RepeatParameterModel(type="repeat", name="r", parameters=[hidden], min=None, max=None)
|
||||
with pytest.raises(AssertionError):
|
||||
assert_yaml_v1_parameters([repeat])
|
||||
|
||||
|
||||
def test_published_tool_source_schema_has_no_xml_only_leaks():
|
||||
raw = UserToolSource.model_json_schema()
|
||||
# Collect all property names across every $defs entry and the top level.
|
||||
all_property_names: set = set()
|
||||
for defn in raw.get("$defs", {}).values():
|
||||
all_property_names.update(defn.get("properties", {}).keys())
|
||||
all_property_names.update(raw.get("properties", {}).keys())
|
||||
leaks = [bad for bad in _BLACKLIST_SUBSTRINGS if bad in all_property_names]
|
||||
assert not leaks, f"XML-only fields leaked into published schema: {leaks}"
|
||||
Reference in New Issue
Block a user