From f06601d02f84e6dfe5d7c0ec4423028d5ae780fb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Mar 2026 11:20:47 -0400 Subject: [PATCH 1/5] Add static YAML agent backend for deterministic testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StaticAgentRegistry subclasses AgentRegistry, returns StaticAgent instances that match YAML rules instead of calling LLMs. Swapped at DI container level via static_agents_config — no mocks needed. New files: - lib/galaxy/agents/static_backend.py - test/unit/app/test_static_agent_backend.py (29 tests) - test/integration/static_agents.yml - test/integration/test_agents_static.py Co-Authored-By: Claude Opus 4.6 --- lib/galaxy/agents/factory.py | 29 ++ lib/galaxy/agents/static_backend.py | 126 +++++++ lib/galaxy/app.py | 8 +- lib/galaxy/config/schemas/config_schema.yml | 3 + lib/galaxy/managers/configuration.py | 8 + lib/galaxy_test/api/test_agents.py | 133 +++++++ lib/galaxy_test/base/data/static_agents.yml | 67 ++++ lib/galaxy_test/base/populators.py | 13 + lib/galaxy_test/driver/driver_util.py | 7 + test/integration/test_agents.py | 231 +----------- test/unit/app/test_static_agent_backend.py | 395 ++++++++++++++++++++ 11 files changed, 789 insertions(+), 231 deletions(-) create mode 100644 lib/galaxy/agents/factory.py create mode 100644 lib/galaxy/agents/static_backend.py create mode 100644 lib/galaxy_test/api/test_agents.py create mode 100644 lib/galaxy_test/base/data/static_agents.yml create mode 100644 test/unit/app/test_static_agent_backend.py diff --git a/lib/galaxy/agents/factory.py b/lib/galaxy/agents/factory.py new file mode 100644 index 00000000000..d4fc34c930f --- /dev/null +++ b/lib/galaxy/agents/factory.py @@ -0,0 +1,29 @@ +"""Factory for building the appropriate AgentRegistry from config.""" + +import logging +from typing import TYPE_CHECKING + +from .registry import ( + AgentRegistry, + build_default_registry, +) +from .static_backend import StaticAgentRegistry + +if TYPE_CHECKING: + from galaxy.config import GalaxyAppConfiguration + +log = logging.getLogger(__name__) + + +def build_registry(config: "GalaxyAppConfiguration") -> AgentRegistry: + """Build an AgentRegistry based on config. + + Uses StaticAgentRegistry when inference_services.static_responses is set, + otherwise builds the default registry with real agents. + """ + inference_config = getattr(config, "inference_services", None) or {} + static_responses = inference_config.get("static_responses") if isinstance(inference_config, dict) else None + if static_responses: + log.info(f"Static agent backend loaded: {static_responses}") + return StaticAgentRegistry(static_responses) + return build_default_registry(config) diff --git a/lib/galaxy/agents/static_backend.py b/lib/galaxy/agents/static_backend.py new file mode 100644 index 00000000000..7546d17f33a --- /dev/null +++ b/lib/galaxy/agents/static_backend.py @@ -0,0 +1,126 @@ +"""Static agent backend for deterministic testing without LLM calls. + +Provides StaticAgent and StaticAgentRegistry that return canned responses +from YAML rules. Swap at the DI container level — no mocks, no pydantic-ai. +""" + +import re +from typing import ( + Any, + Optional, +) + +import yaml + +from .base import ( + ActionSuggestion, + AgentResponse, + BaseGalaxyAgent, + GalaxyAgentDependencies, +) +from .registry import AgentRegistry + + +class StaticAgent(BaseGalaxyAgent): + """Agent that returns canned responses from YAML rules. + + Subclasses BaseGalaxyAgent but skips pydantic-ai Agent creation entirely. + Only process() is meaningful — all other BaseGalaxyAgent methods are stubs. + """ + + agent_type = "static" # overridden per-instance + + def __init__( + self, + agent_type_str: str, + rules: list[dict[str, Any]], + fallback: dict[str, Any], + defaults: dict[str, Any], + ): + # Intentionally skip super().__init__() — no pydantic-ai Agent needed. + self.agent_type = agent_type_str + self._rules = rules + self._fallback = fallback + self._defaults = defaults + + def _create_agent(self): + raise NotImplementedError("StaticAgent does not use pydantic-ai") + + def get_system_prompt(self) -> str: + return "" + + async def process(self, query: str, context: Optional[dict[str, Any]] = None) -> AgentResponse: + for rule in self._rules: + if self._rule_matches(rule.get("match", {}), query, context): + return self._make_response(rule["response"]) + return self._make_response(self._fallback) + + def _rule_matches(self, match: dict[str, Any], query: str, context: Optional[dict[str, Any]]) -> bool: + if "agent_type" in match and match["agent_type"] != self.agent_type: + return False + if "query" in match and not re.search(match["query"], query): + return False + if "context" in match: + if not context: + return False + for field, pattern in match["context"].items(): + if field not in context or not re.search(pattern, str(context[field])): + return False + return True + + def _make_response(self, resp: dict[str, Any]) -> AgentResponse: + raw_suggestions = resp.get("suggestions", []) + suggestions = [ActionSuggestion(**s) for s in raw_suggestions] + return AgentResponse( + content=resp.get("content", self._fallback.get("content", "")), + confidence=resp.get("confidence", self._defaults.get("confidence", "medium")), + agent_type=resp.get("agent_type", self.agent_type), + suggestions=suggestions, + metadata={**resp.get("metadata", {}), "static_backend": True}, + reasoning=resp.get("reasoning"), + ) + + +class StaticAgentRegistry(AgentRegistry): + """Registry that returns StaticAgent instances from YAML config. + + Subclasses AgentRegistry so it's type-compatible with the DI container. + """ + + def __init__(self, config_path: str): + super().__init__() + with open(config_path) as f: + self._config: dict[str, Any] = yaml.safe_load(f) or {} + self._rules: list[dict[str, Any]] = self._config.get("rules", []) + self._fallback: dict[str, Any] = self._config.get("fallback", {}) + self._defaults: dict[str, Any] = self._config.get("defaults", {}) + + # Collect known agent_types from rules + self._known_types: set[str] = set() + for rule in self._rules: + match = rule.get("match", {}) + if "agent_type" in match: + self._known_types.add(match["agent_type"]) + + def get_agent(self, agent_type: str, deps: GalaxyAgentDependencies) -> StaticAgent: + """Return a StaticAgent that matches rules for this agent_type.""" + applicable = [r for r in self._rules if r.get("match", {}).get("agent_type", agent_type) == agent_type] + return StaticAgent(agent_type, applicable, self._fallback, self._defaults) + + def is_registered(self, agent_type: str) -> bool: + return agent_type in self._known_types or bool(self._fallback) + + def list_agents(self) -> list[str]: + return sorted(self._known_types) + + def get_agent_info(self, agent_type: str) -> dict[str, Any]: + return { + "agent_type": agent_type, + "class_name": "StaticAgent", + "module": "galaxy.agents.static_backend", + "metadata": {"static_backend": True}, + "description": "Static test agent", + } + + def list_agent_info(self) -> list[dict[str, Any]]: + return [self.get_agent_info(t) for t in sorted(self._known_types)] diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index a31bf45b048..05843b558ca 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -25,10 +25,8 @@ from galaxy import ( jobs, tools, ) -from galaxy.agents.registry import ( - AgentRegistry, - build_default_registry, -) +from galaxy.agents.factory import build_registry as build_agent_registry +from galaxy.agents.registry import AgentRegistry from galaxy.carbon_emissions import get_carbon_intensity_entry from galaxy.celery.base_task import ( GalaxyTaskBeforeStart, @@ -775,7 +773,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication, InstallationT self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self)) # AI agent registry and service - agent_registry = build_default_registry(self.config) + agent_registry = build_agent_registry(self.config) self._register_singleton(AgentRegistry, agent_registry) self._register_singleton(AgentService, AgentService(self.config, JobQueryManager(self), agent_registry)) diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index f0a4ac4d1dd..d6d3e027f64 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -4091,6 +4091,9 @@ mapping: Agents and plugins inherit from 'default' configuration, which itself falls back to global ai_model/ai_api_key settings. All agents are enabled by default. Example: inference_services: { default: { model: gpt-4o-mini, temperature: 0.7 }, custom_tool: { enabled: false }, jupyterlite: { model: gpt-4o } } + Set static_responses to a YAML file path to replace all LLM calls with + deterministic responses for testing: + inference_services: { static_responses: test/integration/static_agents.yml } enable_tool_recommendations: type: bool diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 93aae1ce7a2..192c7d56725 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -20,6 +20,13 @@ from galaxy.structured_app import StructuredApp log = logging.getLogger(__name__) +def _get_registry_type(config) -> str: + inference_config = getattr(config, "inference_services", None) or {} + if isinstance(inference_config, dict) and inference_config.get("static_responses"): + return "static" + return "default" + + class ConfigurationManager: """Interface/service object for interacting with configuration and related data.""" @@ -232,6 +239,7 @@ class ConfigSerializer(base.ModelSerializer): "llm_api_configured": lambda item, key, **context: bool( item.ai_api_key or item.ai_api_base_url or getattr(item, "inference_services", None) ), + "llm_registry_type": lambda item, key, **context: _get_registry_type(item), "install_tool_dependencies": _use_config, "install_repository_dependencies": _use_config, "install_resolver_dependencies": _use_config, diff --git a/lib/galaxy_test/api/test_agents.py b/lib/galaxy_test/api/test_agents.py new file mode 100644 index 00000000000..599a52bf99f --- /dev/null +++ b/lib/galaxy_test/api/test_agents.py @@ -0,0 +1,133 @@ +"""API tests for Galaxy AI agents. + +Tests run against any Galaxy with agents configured (static or real LLM). +Assertions adapt based on llm_registry_type: strong for "static", weak for "default". + +## Running (auto-started server with static backend): + ./run_tests.sh -api lib/galaxy_test/api/test_agents.py + +## Running against an external Galaxy with a real LLM: + ./run_tests.sh -api lib/galaxy_test/api/test_agents.py \ + --external_url http://localhost:8080/ \ + --external_user_key YOUR_API_KEY +""" + +from galaxy_test.base.populators import ( + DatasetPopulator, + skip_without_agents, +) +from ._framework import ApiTestCase + + +class TestAgentsApi(ApiTestCase): + dataset_populator: DatasetPopulator + + def setUp(self): + super().setUp() + self.dataset_populator = DatasetPopulator(self.galaxy_interactor) + config = self._get("configuration").json() + self._registry_type = config.get("llm_registry_type", "default") + + @property + def _is_static(self) -> bool: + return self._registry_type == "static" + + def _query_agent(self, query: str, agent_type: str = "auto") -> dict: + response = self._post( + "ai/agents/query", + data={"query": query, "agent_type": agent_type}, + json=True, + ) + self._assert_status_code_is_ok(response) + return response.json() + + @skip_without_agents + def test_configuration_reports_agents(self): + """GET /api/configuration reports llm_api_configured.""" + config = self._get("configuration").json() + assert config.get("llm_api_configured") is True + if self._is_static: + assert config.get("llm_registry_type") == "static" + + @skip_without_agents + def test_list_agents(self): + """GET /api/ai/agents returns agent list.""" + response = self._get("ai/agents") + self._assert_status_code_is_ok(response) + data = response.json() + assert "agents" in data + agent_types = [a["agent_type"] for a in data["agents"]] + assert len(agent_types) > 0 + if self._is_static: + assert "router" in agent_types + assert "error_analysis" in agent_types + + @skip_without_agents + def test_list_agents_includes_custom_tool(self): + """GET /api/ai/agents includes custom_tool agent.""" + response = self._get("ai/agents") + self._assert_status_code_is_ok(response) + agent_types = [a["agent_type"] for a in response.json()["agents"]] + assert "custom_tool" in agent_types + + @skip_without_agents + def test_chat_greeting(self): + """Query with greeting returns appropriate response.""" + data = self._query_agent("Hello!") + content = data["response"]["content"] + assert len(content) > 0 + if self._is_static: + assert "Hello" in content + + @skip_without_agents + def test_chat_domain_query(self): + """Query about RNA-seq returns domain-specific response.""" + data = self._query_agent("How do I analyze RNA-seq data?") + content = data["response"]["content"] + assert len(content) > 0 + if self._is_static: + assert "HISAT2" in content + + @skip_without_agents + def test_chat_fallback(self): + """Generic query gets a response.""" + data = self._query_agent("Tell me about Galaxy") + content = data["response"]["content"] + assert len(content) > 0 + if self._is_static: + assert "Galaxy" in content + + @skip_without_agents + def test_response_metadata(self): + """Agent responses include metadata.""" + data = self._query_agent("Hello!") + metadata = data["response"].get("metadata", {}) + assert isinstance(metadata, dict) + if self._is_static: + assert metadata.get("static_backend") is True + + @skip_without_agents + def test_custom_tool_agent(self): + """custom_tool agent returns tool metadata.""" + data = self._query_agent("Create a tool that counts lines", agent_type="custom_tool") + resp = data["response"] + assert "metadata" in resp + assert "tool_id" in resp["metadata"] + if self._is_static: + assert resp["metadata"]["tool_id"] == "line-counter" + assert "wc -l" in resp["metadata"]["tool_yaml"] + suggestions = resp.get("suggestions", []) + assert len(suggestions) >= 1 + save = [s for s in suggestions if s["action_type"] == "save_tool"] + assert len(save) == 1 + assert save[0]["parameters"]["tool_id"] == "line-counter" + assert save[0]["parameters"]["tool_yaml"] + + @skip_without_agents + def test_error_analysis_agent(self): + """error_analysis agent returns error-related response.""" + data = self._query_agent("My job failed with exit code 1", agent_type="error_analysis") + content = data["response"]["content"] + assert len(content) > 0 + if self._is_static: + assert "error" in content.lower() or "configuration" in content.lower() diff --git a/lib/galaxy_test/base/data/static_agents.yml b/lib/galaxy_test/base/data/static_agents.yml new file mode 100644 index 00000000000..eadb71e1357 --- /dev/null +++ b/lib/galaxy_test/base/data/static_agents.yml @@ -0,0 +1,67 @@ +defaults: + confidence: medium + +rules: + - match: + agent_type: router + query: "(?i)hello|hi|hey" + response: + content: "Hello! I'm Galaxy's AI assistant. How can I help?" + confidence: high + agent_type: router + + - match: + agent_type: router + query: "(?i)rna.?seq" + response: + content: "Galaxy has several RNA-seq tools including HISAT2 and STAR." + confidence: high + agent_type: router + + - match: + agent_type: custom_tool + response: + content: "I've created a Line Counter tool that counts lines in a file." + confidence: high + agent_type: custom_tool + metadata: + tool_id: line-counter + tool_yaml: | + class: GalaxyUserTool + id: line-counter + name: Line Counter + version: "1.0.0" + description: Counts lines in a file + container: ubuntu:latest + shell_command: "wc -l < $(inputs.input_file.path) > output.txt" + suggestions: + - action_type: save_tool + description: "Save the Line Counter tool to your Galaxy instance" + parameters: + tool_id: line-counter + tool_yaml: | + class: GalaxyUserTool + id: line-counter + name: Line Counter + version: "1.0.0" + confidence: high + priority: 1 + + - match: + agent_type: error_analysis + response: + content: "This error appears to be a tool configuration issue." + confidence: high + agent_type: error_analysis + + - match: + agent_type: router + response: + content: "I can help with Galaxy workflows, tools, and data analysis." + confidence: medium + agent_type: router + +fallback: + content: "Static backend: no matching rule." + confidence: low + agent_type: unknown diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 6e5776f11ae..724557788ae 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -226,6 +226,19 @@ def skip_without_tool(tool_id: str): return method_wrapper +def skip_without_agents(method): + @wraps(method) + def wrapped_method(api_test_case, *args, **kwd): + interactor = api_test_case.anonymous_galaxy_interactor + resp = interactor.get("configuration") + api_asserts.assert_status_code_is_ok(resp) + if not resp.json().get("llm_api_configured", False): + raise unittest.SkipTest("Agents not available") + return method(api_test_case, *args, **kwd) + + return wrapped_method + + def skip_without_asgi(method): @wraps(method) def wrapped_method(api_test_case: HasAnonymousGalaxyInteractor, *args, **kwd): diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index 37907b887de..9aeb37cf0d6 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -293,6 +293,13 @@ backends: # TODO: read from Galaxy's config API. os.environ["GALAXY_TEST_TOOL_DEPENDENCY_DIR"] = tool_dependency_dir or os.path.join(tmpdir, "dependencies") + # Static agent backend for deterministic AI agent testing + static_agents_path = os.path.realpath( + os.path.join(os.path.dirname(__file__), "..", "base", "data", "static_agents.yml") + ) + if os.path.exists(static_agents_path): + config["inference_services"] = {"static_responses": static_agents_path} + return config diff --git a/test/integration/test_agents.py b/test/integration/test_agents.py index 3868bc09337..3c94d7196a9 100644 --- a/test/integration/test_agents.py +++ b/test/integration/test_agents.py @@ -1,43 +1,19 @@ -"""Test Galaxy AI agents API and functionality. +"""Test Galaxy AI agents API with live LLM. -This module contains two test suites: -1. Mocked tests - Deterministic tests with mocked LLM responses (always run) -2. Live LLM tests - Integration tests requiring configured LLM (optional, marked with @pytest.mark.requires_llm) +Requires a configured LLM — skipped unless GALAXY_TEST_ENABLE_LIVE_LLM=1. +For deterministic tests without LLM, see test_agents_static.py. -## Running the tests: - -### API tests (Galaxy test instance auto-configured): - # Run mocked API tests (Galaxy test framework handles setup): - pytest test/integration/test_agents.py::TestAgentsApiMocked -v - - # Run live LLM API tests: - GALAXY_TEST_ENABLE_LIVE_LLM=1 pytest test/integration/test_agents.py::TestAgentsApiLiveLLM -v - -### Configuration for live API tests (TestAgentsApiLiveLLM): +## Running: export GALAXY_TEST_AI_API_KEY="your-api-key" export GALAXY_TEST_AI_MODEL="llama-4-scout" export GALAXY_TEST_AI_API_BASE_URL="http://localhost:4000/v1/" export GALAXY_TEST_ENABLE_LIVE_LLM=1 - -### Configuration for live unit tests (TestAgentUnitLiveLLM): - export GALAXY_AI_API_KEY="your-api-key" - export GALAXY_AI_MODEL="llama-4-scout" - export GALAXY_AI_API_BASE_URL="http://localhost:4000/v1/" - export GALAXY_TEST_ENABLE_LIVE_LLM=1 + pytest test/integration/test_agents.py -v """ import logging import os -from unittest.mock import ( - AsyncMock, - MagicMock, - patch, -) -from galaxy.agents import GalaxyAgentDependencies -from galaxy.agents.error_analysis import ErrorAnalysisResult -from galaxy.agents.registry import build_default_registry -from galaxy.tool_util_models import UserToolSource from galaxy.util.unittest_utils import pytestmark_live_llm from galaxy_test.base.populators import ( DatasetPopulator, @@ -70,203 +46,6 @@ class AgentIntegrationTestCase(IntegrationTestCase): config["ai_model"] = ai_model -_registry = build_default_registry() - - -def _create_deps_with_mock_model(self, trans, user): - """Replacement for AgentService.create_dependencies that injects a mock model_factory.""" - toolbox = trans.app.toolbox if hasattr(trans, "app") and hasattr(trans.app, "toolbox") else None - return GalaxyAgentDependencies( - trans=trans, - user=user, - config=self.config, - job_manager=self.job_manager, - toolbox=toolbox, - get_agent=_registry.get_agent, - model_factory=lambda: MagicMock(), - ) - - -class TestAgentsApiMocked(AgentIntegrationTestCase): - """Test the Galaxy AI agents API with mocked LLM responses. - - These tests use mocked LLM responses for deterministic testing. - They always run in CI and don't require any LLM configuration. - """ - - def setUp(self): - super().setUp() - self.dataset_populator = DatasetPopulator(self.galaxy_interactor) - self.workflow_populator = WorkflowPopulator(self.galaxy_interactor) - - def test_list_agents(self): - """Test listing available agents (no LLM needed).""" - response = self._get("ai/agents") - self._assert_status_code_is_ok(response) - data = response.json() - assert "agents" in data - agents = data["agents"] - assert len(agents) > 0 - # Check that core agents are registered - agent_types = [a["agent_type"] for a in agents] - assert "router" in agent_types - assert "custom_tool" in agent_types - assert "error_analysis" in agent_types - - @patch("galaxy.managers.agents.AgentService.create_dependencies", _create_deps_with_mock_model) - @patch("galaxy.agents.router.Agent") - def test_query_agent_auto_routing_mocked(self, mock_router_agent_class): - """Test automatic agent routing with mocked LLM. - - With the new router architecture, the router uses output functions - and returns the final response directly (either answering or handing - off to specialists internally). - """ - # Set up mock router agent - mock_router_agent = AsyncMock() - mock_router_agent_class.return_value = mock_router_agent - - # Mock router response - now returns string directly - async def mock_router_run(query, *args, **kwargs): - result = MagicMock() - if "BWA" in query or "tool" in query.lower(): - # Simulate what custom_tool handoff would return - result.output = "I've created a BWA-MEM tool for paired-end reads. The tool definition includes inputs for reference and read files." - else: - # Direct response from router - result.output = "I'm Galaxy's AI assistant. How can I help you today?" - return result - - mock_router_agent.run = mock_router_run - - response = self._post( - "ai/agents/query", - data={ - "query": "Create a BWA-MEM tool for paired-end reads", - "agent_type": "auto", - }, - json=True, - ) - self._assert_status_code_is_ok(response) - data = response.json() - # Router now returns content in the response object - assert "response" in data - assert "content" in data["response"] - assert "BWA" in data["response"]["content"] or len(data["response"]["content"]) > 0 - - @patch("galaxy.managers.agents.AgentService.create_dependencies", _create_deps_with_mock_model) - @patch("galaxy.agents.custom_tool.Agent") - def test_query_custom_tool_agent_mocked(self, mock_agent_class): - """Test the custom tool agent with mocked LLM.""" - mock_agent = AsyncMock() - mock_agent_class.return_value = mock_agent - - # Mock tool creation with UserToolSource - mock_tool = UserToolSource( - **{ - "class": "GalaxyUserTool", - "id": "line-counter", - "name": "Line Counter", - "version": "1.0.0", - "description": "Counts lines in a file", - "container": "ubuntu:latest", - "shell_command": "wc -l < $(inputs.input_file.path) > output.txt", - "inputs": [], - "outputs": [], - } - ) - - async def mock_run(*args, **kwargs): - result = MagicMock() - result.output = mock_tool - return result - - mock_agent.run = mock_run - - response = self._post( - "ai/agents/query", - data={ - "query": "Create a simple tool that counts lines in a file", - "agent_type": "custom_tool", - }, - json=True, - ) - self._assert_status_code_is_ok(response) - data = response.json() - # Response structure is {'response': {..., 'metadata': {...}}, 'routing_info': ..., ...} - assert "response" in data - assert "metadata" in data["response"] - assert data["response"]["metadata"]["tool_id"] == "line-counter" - assert "wc -l" in data["response"]["metadata"]["tool_yaml"] - - # Verify suggestions are present and valid - suggestions = data["response"].get("suggestions", []) - assert len(suggestions) >= 1, "Custom tool should return at least one suggestion" - # Should have a SAVE_TOOL suggestion with required parameters - save_suggestions = [s for s in suggestions if s["action_type"] == "save_tool"] - assert len(save_suggestions) == 1, "Should have exactly one SAVE_TOOL suggestion" - assert save_suggestions[0]["parameters"].get("tool_yaml"), "SAVE_TOOL must have tool_yaml" - assert save_suggestions[0]["parameters"].get("tool_id"), "SAVE_TOOL must have tool_id" - - @patch("galaxy.managers.agents.AgentService.create_dependencies", _create_deps_with_mock_model) - @patch("galaxy.agents.error_analysis.Agent") - def test_query_error_analysis_agent_mocked(self, mock_agent_class): - """Test the error analysis agent with mocked LLM.""" - mock_agent = AsyncMock() - mock_agent_class.return_value = mock_agent - - # Mock error analysis - async def mock_run(*args, **kwargs): - result = MagicMock() - mock_analysis = ErrorAnalysisResult( - error_category="tool_configuration", - error_severity="medium", - likely_cause="The command 'samtools' was not found in PATH", - solution_steps=[ - "Install samtools using conda", - "Check tool dependencies", - "Verify container configuration", - ], - confidence="high", - ) - result.output = mock_analysis - return result - - mock_agent.run = mock_run - - # Don't pass a job_id - the agent handles missing job context gracefully - # and we're testing the mocked LLM response, not job lookup - response = self._post( - "ai/agents/query", - data={ - "query": "Why did my job fail? stderr shows: command not found: samtools", - "agent_type": "error_analysis", - }, - json=True, - ) - self._assert_status_code_is_ok(response) - data = response.json() - # Response structure is {'response': {..., 'content': ...}, ...} - assert "response" in data - assert "content" in data["response"] - # Should mention the error type or solution - content = data["response"]["content"].lower() - assert "command" in content or "samtools" in content or "not found" in content - - # Suggestions are optional - only returned for actionable items like CONTACT_SUPPORT - # In this mock, requires_admin=False, so no suggestions expected - suggestions = data["response"].get("suggestions", []) - for suggestion in suggestions: - assert "action_type" in suggestion - assert "description" in suggestion - assert "confidence" in suggestion - - -# ============================================================================ -# LIVE LLM TEST SUITE - Requires configured LLM -# ============================================================================ - - @pytestmark_live_llm class TestAgentsApiLiveLLM(AgentIntegrationTestCase): """Test Galaxy AI agents API with real LLM. diff --git a/test/unit/app/test_static_agent_backend.py b/test/unit/app/test_static_agent_backend.py new file mode 100644 index 00000000000..dbd8b26bd93 --- /dev/null +++ b/test/unit/app/test_static_agent_backend.py @@ -0,0 +1,395 @@ +"""Unit tests for the static agent backend (YAML rule matching).""" + +import os +import tempfile + +import pytest +import yaml + +from galaxy.agents.base import ( + AgentResponse, + BaseGalaxyAgent, + ConfidenceLevel, +) +from galaxy.agents.static_backend import ( + StaticAgent, + StaticAgentRegistry, +) + + +@pytest.fixture +def sample_config(): + return { + "defaults": {"confidence": "medium"}, + "rules": [ + { + "match": {"agent_type": "router", "query": "(?i)hello|hi"}, + "response": { + "content": "Hello! How can I help?", + "confidence": "high", + "agent_type": "router", + }, + }, + { + "match": {"agent_type": "router", "query": "(?i)rna.?seq"}, + "response": { + "content": "Galaxy has RNA-seq tools including HISAT2.", + "confidence": "high", + "agent_type": "router", + }, + }, + { + "match": {"agent_type": "error_analysis"}, + "response": { + "content": "This error is a tool configuration issue.", + "confidence": "high", + "agent_type": "error_analysis", + }, + }, + { + "match": {"agent_type": "router"}, + "response": { + "content": "I can help with Galaxy workflows and tools.", + "confidence": "medium", + "agent_type": "router", + }, + }, + ], + "fallback": { + "content": "Static backend: no matching rule.", + "confidence": "low", + "agent_type": "unknown", + }, + } + + +@pytest.fixture +def yaml_config_path(sample_config): + with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: + yaml.dump(sample_config, f) + path = f.name + yield path + os.unlink(path) + + +@pytest.fixture +def static_registry(yaml_config_path): + return StaticAgentRegistry(yaml_config_path) + + +# --- StaticAgent tests --- + + +class TestStaticAgent: + def _make_agent(self, agent_type, rules, fallback=None, defaults=None): + fallback = fallback or {"content": "fallback", "confidence": "low", "agent_type": "unknown"} + defaults = defaults or {"confidence": "medium"} + return StaticAgent(agent_type, rules, fallback, defaults) + + @pytest.mark.asyncio + async def test_exact_agent_type_match(self): + rules = [ + { + "match": {"agent_type": "router"}, + "response": {"content": "Router response", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("anything") + assert response.content == "Router response" + assert response.confidence == ConfidenceLevel.HIGH + + @pytest.mark.asyncio + async def test_query_regex_match(self): + rules = [ + { + "match": {"query": "(?i)hello|hi"}, + "response": {"content": "Greeting!", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("Hello there") + assert response.content == "Greeting!" + + @pytest.mark.asyncio + async def test_query_regex_case_insensitive(self): + rules = [ + { + "match": {"query": "(?i)hello"}, + "response": {"content": "Hi!", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("HELLO WORLD") + assert response.content == "Hi!" + + @pytest.mark.asyncio + async def test_combined_match(self): + """Both agent_type and query must match (AND semantics).""" + rules = [ + { + "match": {"agent_type": "router", "query": "(?i)hello"}, + "response": {"content": "Router hello", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + # Both match + response = await agent.process("hello") + assert response.content == "Router hello" + + @pytest.mark.asyncio + async def test_combined_match_query_mismatch(self): + """agent_type matches but query doesn't — should fall through.""" + rules = [ + { + "match": {"agent_type": "router", "query": "(?i)hello"}, + "response": {"content": "Router hello", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("goodbye") + assert response.content == "fallback" + + @pytest.mark.asyncio + async def test_fallthrough_to_catchall(self): + """Specific rule fails, generic catchall matches.""" + rules = [ + { + "match": {"agent_type": "router", "query": "(?i)hello"}, + "response": {"content": "Hello!", "confidence": "high", "agent_type": "router"}, + }, + { + "match": {"agent_type": "router"}, + "response": {"content": "Generic router", "confidence": "medium", "agent_type": "router"}, + }, + ] + agent = self._make_agent("router", rules) + response = await agent.process("something else") + assert response.content == "Generic router" + assert response.confidence == ConfidenceLevel.MEDIUM + + @pytest.mark.asyncio + async def test_fallback_when_nothing_matches(self): + rules = [ + { + "match": {"agent_type": "error_analysis"}, + "response": {"content": "Error help", "confidence": "high", "agent_type": "error_analysis"}, + } + ] + fallback = {"content": "No match found.", "confidence": "low", "agent_type": "unknown"} + agent = self._make_agent("router", rules, fallback=fallback) + response = await agent.process("anything") + assert response.content == "No match found." + assert response.confidence == ConfidenceLevel.LOW + + @pytest.mark.asyncio + async def test_static_backend_metadata_flag(self): + """Every response should have static_backend: True in metadata.""" + rules = [ + { + "match": {"agent_type": "router"}, + "response": {"content": "Hello", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("test") + assert response.metadata.get("static_backend") is True + + @pytest.mark.asyncio + async def test_fallback_also_has_metadata_flag(self): + agent = self._make_agent("router", []) + response = await agent.process("test") + assert response.metadata.get("static_backend") is True + + @pytest.mark.asyncio + async def test_defaults_applied(self): + """confidence from defaults block used when rule omits it.""" + rules = [ + { + "match": {"agent_type": "router"}, + "response": {"content": "No confidence set", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules, defaults={"confidence": "medium"}) + response = await agent.process("test") + assert response.confidence == ConfidenceLevel.MEDIUM + + @pytest.mark.asyncio + async def test_response_is_agent_response(self): + rules = [ + { + "match": {"agent_type": "router"}, + "response": {"content": "Hello", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("test") + assert isinstance(response, AgentResponse) + + @pytest.mark.asyncio + async def test_first_matching_rule_wins(self): + rules = [ + { + "match": {"agent_type": "router"}, + "response": {"content": "First", "confidence": "high", "agent_type": "router"}, + }, + { + "match": {"agent_type": "router"}, + "response": {"content": "Second", "confidence": "medium", "agent_type": "router"}, + }, + ] + agent = self._make_agent("router", rules) + response = await agent.process("test") + assert response.content == "First" + + def test_static_agent_is_base_galaxy_agent(self): + agent = self._make_agent("router", []) + assert isinstance(agent, BaseGalaxyAgent) + + def test_create_agent_raises(self): + agent = self._make_agent("router", []) + with pytest.raises(NotImplementedError): + agent._create_agent() + + def test_get_system_prompt_empty(self): + agent = self._make_agent("router", []) + assert agent.get_system_prompt() == "" + + @pytest.mark.asyncio + async def test_context_rule_fails_when_no_context(self): + """Rule requiring context should NOT match when context is None.""" + rules = [ + { + "match": {"agent_type": "router", "context": {"page_content": "methods"}}, + "response": {"content": "Context match", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("test", context=None) + assert response.content == "fallback" + + @pytest.mark.asyncio + async def test_context_rule_matches_when_context_present(self): + rules = [ + { + "match": {"agent_type": "router", "context": {"page_content": "(?i)methods"}}, + "response": {"content": "Context match", "confidence": "high", "agent_type": "router"}, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("test", context={"page_content": "## Methods\nSome content"}) + assert response.content == "Context match" + + @pytest.mark.asyncio + async def test_suggestions_preserved(self): + rules = [ + { + "match": {"agent_type": "router"}, + "response": { + "content": "Hello", + "confidence": "high", + "agent_type": "router", + "suggestions": [ + { + "action_type": "save_tool", + "description": "Save tool", + "parameters": {"tool_id": "test", "tool_yaml": "test: true"}, + "confidence": "high", + "priority": 1, + } + ], + }, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("test") + assert len(response.suggestions) == 1 + assert response.suggestions[0].action_type.value == "save_tool" + assert response.suggestions[0].parameters["tool_id"] == "test" + + @pytest.mark.asyncio + async def test_reasoning_preserved(self): + rules = [ + { + "match": {"agent_type": "router"}, + "response": { + "content": "Hello", + "confidence": "high", + "agent_type": "router", + "reasoning": "Because reasons", + }, + } + ] + agent = self._make_agent("router", rules) + response = await agent.process("test") + assert response.reasoning == "Because reasons" + + +# --- StaticAgentRegistry tests --- + + +class TestStaticAgentRegistry: + def test_list_agents(self, static_registry): + agents = static_registry.list_agents() + assert "router" in agents + assert "error_analysis" in agents + + def test_get_agent_returns_static_agent(self, static_registry): + # deps not used by StaticAgent, pass None + agent = static_registry.get_agent("router", None) + assert isinstance(agent, StaticAgent) + assert isinstance(agent, BaseGalaxyAgent) + + def test_is_registered_known_type(self, static_registry): + assert static_registry.is_registered("router") is True + assert static_registry.is_registered("error_analysis") is True + + def test_is_registered_unknown_with_fallback(self, static_registry): + # Unknown type but fallback exists → registered + assert static_registry.is_registered("nonexistent") is True + + def test_is_registered_unknown_no_fallback(self, yaml_config_path): + # Modify config to remove fallback + with open(yaml_config_path) as f: + config = yaml.safe_load(f) + config.pop("fallback", None) + with tempfile.NamedTemporaryFile(mode="w", suffix=".yml", delete=False) as f: + yaml.dump(config, f) + no_fallback_path = f.name + try: + registry = StaticAgentRegistry(no_fallback_path) + assert registry.is_registered("nonexistent") is False + finally: + os.unlink(no_fallback_path) + + def test_get_agent_info(self, static_registry): + info = static_registry.get_agent_info("router") + assert info["agent_type"] == "router" + assert info["class_name"] == "StaticAgent" + assert info["metadata"]["static_backend"] is True + + def test_list_agent_info(self, static_registry): + infos = static_registry.list_agent_info() + assert len(infos) >= 2 + types = [i["agent_type"] for i in infos] + assert "router" in types + assert "error_analysis" in types + + @pytest.mark.asyncio + async def test_end_to_end_query(self, static_registry): + """Registry → get_agent → process → response.""" + agent = static_registry.get_agent("router", None) + response = await agent.process("Hello!") + assert "Hello" in response.content or "help" in response.content.lower() + assert response.confidence == ConfidenceLevel.HIGH + + @pytest.mark.asyncio + async def test_end_to_end_fallback(self, static_registry): + agent = static_registry.get_agent("unknown_type", None) + response = await agent.process("xyzzy") + assert "no matching rule" in response.content.lower() + + def test_static_registry_is_agent_registry(self, static_registry): + from galaxy.agents.registry import AgentRegistry + + assert isinstance(static_registry, AgentRegistry) From 85d7efcddd93451f8e578188979dae7fb3640798 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 10 Mar 2026 23:49:58 -0400 Subject: [PATCH 2/5] Add ChatGXY E2E Selenium tests Two tests covering ChatGXY using the static agent backend for deterministic assertions (no LLM calls): - test_chat_greeting_flow: activity visible, send/receive, feedback, metadata - test_multi_turn_and_new_chat: multi-turn conversation, new chat reset Adds chatgxy component selectors to navigation.yml and navigate_to_chatgxy() helper to navigates_galaxy.py. Co-Authored-By: Claude Opus 4.6 --- client/src/utils/navigation/navigation.yml | 22 +++++++ lib/galaxy/selenium/navigates_galaxy.py | 26 ++++++++ lib/galaxy_test/selenium/test_chatgxy.py | 75 ++++++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 lib/galaxy_test/selenium/test_chatgxy.py diff --git a/client/src/utils/navigation/navigation.yml b/client/src/utils/navigation/navigation.yml index 553187ddb12..2db7e8dcc48 100644 --- a/client/src/utils/navigation/navigation.yml +++ b/client/src/utils/navigation/navigation.yml @@ -1454,6 +1454,28 @@ job_details: selector: '//td[@id="galaxy-tool-id"][normalize-space(text()) = "${tool_id}"]' tool_exit_code: '#exit-code' +chatgxy: + selectors: + activity: '#activity-chatgxy' + _: '.chatgxy-container' + header: '.chatgxy-header' + new_chat_button: '.chatgxy-header .btn-outline-primary' + delete_chat_button: '.chatgxy-header .btn-outline-danger' + messages: '.chat-messages' + input: '#chat-input' + send_button: '.send-button' + welcome_message: '.system-notice' + query_cell: '.entry-query .query-text' + response_cell: '.entry-response' + response_content: '.entry-response .response-content' + agent_indicator: '.entry-response .agent-indicator' + loading: '.loading-entry' + feedback_up: '.entry-response .feedback-btn[title="Helpful"]' + feedback_down: '.entry-response .feedback-btn[title="Not helpful"]' + feedback_ack: '.entry-response .feedback-ack' + meta_tag: '.entry-response .meta-tag' + + zip_import_wizard: selectors: _: '.zip-import-wizard' diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index 77c03b1b844..cc538777f51 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -1608,6 +1608,32 @@ class NavigatesGalaxy(HasDriverProxy[WaitType]): self.components.invocations.activity.wait_for_and_click() self.components.invocations.activity_expand.wait_for_and_click() + def navigate_to_chatgxy(self): + self.home() + self.components.chatgxy.activity.wait_for_and_click() + + def chatgxy_ensure_new_chat(self): + """Ensure ChatGXY center panel shows an empty conversation.""" + chatgxy = self.components.chatgxy + chatgxy._.wait_for_visible() + if len(chatgxy.query_cell.all()) > 0 or len(chatgxy.response_content.all()) > 0: + chatgxy.new_chat_button.wait_for_and_click() + self._chatgxy_assert_chat_empty() + + def chatgxy_send_message(self, text): + """Type a message, click send, and wait for the response to appear.""" + chatgxy = self.components.chatgxy + chatgxy.input.wait_for_and_send_keys(text) + chatgxy.send_button.wait_for_and_click() + chatgxy.loading.wait_for_absent_or_hidden() + chatgxy.response_content.wait_for_visible() + + @retry_during_transitions + def _chatgxy_assert_chat_empty(self): + chatgxy = self.components.chatgxy + assert len(chatgxy.query_cell.all()) == 0 + assert len(chatgxy.response_content.all()) == 0 + def navigate_to_pages(self): self.home() self.components.pages.activity.wait_for_and_click() diff --git a/lib/galaxy_test/selenium/test_chatgxy.py b/lib/galaxy_test/selenium/test_chatgxy.py new file mode 100644 index 00000000000..b9d0caf0fa2 --- /dev/null +++ b/lib/galaxy_test/selenium/test_chatgxy.py @@ -0,0 +1,75 @@ +"""E2E tests for the ChatGXY conversational AI interface. + +Uses the static agent backend for deterministic assertions — no LLM calls. +Skipped when agents are not configured (skip_without_agents decorator). +""" + +from galaxy_test.base.populators import skip_without_agents +from .framework import ( + retry_assertion_during_transitions, + selenium_test, + SeleniumTestCase, +) + + +class TestChatGXY(SeleniumTestCase): + ensure_registered = True + + @skip_without_agents + @selenium_test + def test_chat_greeting_flow(self): + """Activity visible, send greeting, get response, give feedback, check metadata.""" + self.home() + chatgxy = self.components.chatgxy + + # Activity bar icon visible + chatgxy.activity.wait_for_visible() + + # Navigate, start fresh + self.navigate_to_chatgxy() + self.chatgxy_ensure_new_chat() + + # Send greeting + self.chatgxy_send_message("Hello!") + + # Verify query cell and response + assert chatgxy.query_cell.wait_for_text() == "Hello!" + + @retry_assertion_during_transitions + def assert_response(): + assert "Hello" in chatgxy.response_content.wait_for_text() + + assert_response() + + # Agent indicator and metadata tags present + chatgxy.agent_indicator.wait_for_visible() + assert len(chatgxy.meta_tag.all()) >= 1 + + # Feedback + chatgxy.feedback_up.wait_for_and_click() + assert "Thanks" in chatgxy.feedback_ack.wait_for_text() + + @skip_without_agents + @selenium_test + def test_multi_turn_and_new_chat(self): + """Two queries build up conversation, 'New' resets it.""" + self.navigate_to_chatgxy() + self.chatgxy_ensure_new_chat() + chatgxy = self.components.chatgxy + + self.chatgxy_send_message("Hello!") + self.chatgxy_send_message("How do I analyze RNA-seq data?") + + @retry_assertion_during_transitions + def assert_two_exchanges(): + assert len(chatgxy.query_cell.all()) == 2 + responses = chatgxy.response_content.all() + assert len(responses) >= 2 + # Second response should be domain-specific + assert "HISAT2" in responses[-1].text or "RNA-seq" in responses[-1].text + + assert_two_exchanges() + + # New chat resets + chatgxy.new_chat_button.wait_for_and_click() + self._chatgxy_assert_chat_empty() From 2a0f9155be665f56fa03b7e5eb6fc109f4a6e24e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 11 Mar 2026 01:55:08 -0400 Subject: [PATCH 3/5] Add GalaxyWizard E2E Selenium tests, fix detect_errors stderr redirect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add data-description attributes to GalaxyWizard.vue and DatasetError.vue for test selectors. Add navigation helpers and Selenium test exercising the inline error analysis widget with the static agent backend. Fix broken stderr redirect in detect_errors.xml (>2& → >&2). Co-Authored-By: Claude Opus 4.6 --- .../DatasetInformation/DatasetError.vue | 2 +- client/src/components/GalaxyWizard.vue | 26 +++++-- client/src/utils/navigation/navigation.yml | 29 ++++++++ lib/galaxy/selenium/navigates_galaxy.py | 15 ++++ .../selenium/test_galaxy_wizard.py | 69 +++++++++++++++++++ test/functional/tools/detect_errors.xml | 4 +- test/unit/tool_util/test_parsing.py | 4 +- 7 files changed, 138 insertions(+), 11 deletions(-) create mode 100644 lib/galaxy_test/selenium/test_galaxy_wizard.py diff --git a/client/src/components/DatasetInformation/DatasetError.vue b/client/src/components/DatasetInformation/DatasetError.vue index 2daea735b57..946e255c815 100644 --- a/client/src/components/DatasetInformation/DatasetError.vue +++ b/client/src/components/DatasetInformation/DatasetError.vue @@ -155,7 +155,7 @@ onMounted(async () => { may not always be accurate.

- +