diff --git a/test/evals/evaluators.py b/test/evals/evaluators.py index de301aaed3e..9501c9728e2 100644 --- a/test/evals/evaluators.py +++ b/test/evals/evaluators.py @@ -1,13 +1,22 @@ """Custom pydantic-evals evaluators for Galaxy agents.""" from dataclasses import dataclass -from typing import Any +from typing import ( + Any, + Generic, + TypeVar, +) from pydantic_evals.evaluators import ( Evaluator, EvaluatorContext, ) +# Routing datasets vary in their case-input shape: a plain query string, or a dict for the +# multi-turn cases. HandoffMatch ignores the input entirely (it only compares output to +# expected), so it is generic over the input type and adapts to whichever dataset it scores. +HandoffInputsT = TypeVar("HandoffInputsT") + def _output_text(output: Any) -> str: """Pull a string out of a task output that may be a str or {"content": str, ...} dict.""" @@ -17,13 +26,10 @@ def _output_text(output: Any) -> str: @dataclass -class HandoffMatch(Evaluator[Any, str, dict]): - """Score 1.0 if router's chosen agent_type matches expected, 0.0 otherwise. +class HandoffMatch(Evaluator[HandoffInputsT, str, dict], Generic[HandoffInputsT]): + """Score 1.0 if router's chosen agent_type matches expected, 0.0 otherwise.""" - Input type is ``Any`` so this works with both string-query datasets (routing) and - dict-input multi-turn datasets (routing_depth, routing_clarification_followup).""" - - def evaluate(self, ctx: EvaluatorContext[Any, str, dict]) -> float: + def evaluate(self, ctx: EvaluatorContext[HandoffInputsT, str, dict]) -> float: return 1.0 if ctx.output == ctx.expected_output else 0.0 diff --git a/test/evals/specs.py b/test/evals/specs.py index 3faaec7111e..1c8d269aade 100644 --- a/test/evals/specs.py +++ b/test/evals/specs.py @@ -10,7 +10,9 @@ from collections.abc import ( from dataclasses import dataclass from typing import ( Any, + Generic, Optional, + TypeVar, ) from pydantic_ai.models import Model @@ -47,17 +49,18 @@ from .tasks import ( make_tool_recommendation_task, ) +# A case input is either a plain query string or, for the multi-turn datasets +# (routing_depth, routing_clarification_followup), a dict. Parameterizing BuiltDataset on it +# keeps the dataset and its task linked: a dict-input dataset must pair with a dict-input task. +CaseInputsT = TypeVar("CaseInputsT") + @dataclass -class BuiltDataset: - """A dataset configured with evaluators and a task ready to evaluate. +class BuiltDataset(Generic[CaseInputsT]): + """A dataset configured with evaluators and a task ready to evaluate.""" - Input type is ``Any``: most datasets take a string query, but the multi-turn ones - (routing_depth, routing_clarification_followup) take a dict case input. - """ - - dataset: Dataset[Any, Any, dict[str, Any]] - task: Callable[..., Awaitable[Any]] + dataset: Dataset[CaseInputsT, Any, dict[str, Any]] + task: Callable[[CaseInputsT], Awaitable[Any]] primary_score: str # name of the headline scorer for the summary table