From 806d7c22ec5e3e321198adc5c0ec5bcaff96f2c3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 20 Jun 2026 16:28:58 +0200 Subject: [PATCH] Strip the provider prefix when building the eval judge model --judge-model values are provider-qualified (e.g. `openai:gpt-5`), but the OpenAI API expects the bare model id. build_judge_model passed the qualified name straight to OpenAIChatModel, so any judge model failed. Strip the `provider:` prefix the same way the agents under test already do. --- test/evals/judge.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/evals/judge.py b/test/evals/judge.py index 412a1511636..b343f59b9cf 100644 --- a/test/evals/judge.py +++ b/test/evals/judge.py @@ -14,5 +14,8 @@ def build_judge_model( api_key: str, ) -> OpenAIChatModel: """Build an OpenAI-compatible pydantic-ai model for use as an LLMJudge.""" + # Strip the `provider:` prefix (e.g. `openai:gpt-5-mini`) the same way the + # agents under test do; the bare model name is what the API expects. + bare_name = model_name.split(":", 1)[1] if ":" in model_name else model_name provider = OpenAIProvider(base_url=base_url, api_key=api_key) - return OpenAIChatModel(model_name, provider=provider) + return OpenAIChatModel(bare_name, provider=provider)