Add GalaxyWizard E2E Selenium tests, fix detect_errors stderr redirect

Add data-description attributes to GalaxyWizard.vue and DatasetError.vue
for test selectors. Add navigation helpers and Selenium test exercising
the inline error analysis widget with the static agent backend.

Fix broken stderr redirect in detect_errors.xml (>2& → >&2).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
John Chilton
2026-03-18 14:54:16 -04:00
co-authored by Claude Opus 4.6
parent 85d7efcddd
commit 2a0f9155be
7 changed files with 138 additions and 11 deletions
@@ -155,7 +155,7 @@ onMounted(async () => {
may not always be accurate.
</span>
</p>
<BCard v-if="'tool_stderr' in jobDetails" class="mb-2">
<BCard v-if="'tool_stderr' in jobDetails" class="mb-2" data-description="galaxy wizard card">
<GalaxyWizard
view="error"
:query="jobDetails.tool_stderr ?? ''"
+20 -6
View File
@@ -91,20 +91,29 @@ async function sendFeedback(value: "up" | "down") {
</script>
<template>
<div>
<GButton v-if="!queryResponse" class="w-100" variant="info" :disabled="busy" @click="submitQuery">
<div data-description="galaxy wizard">
<GButton
v-if="!queryResponse"
class="w-100"
variant="info"
:disabled="busy"
data-description="galaxy wizard analyze button"
@click="submitQuery">
<span v-if="!busy"> Let our Help Wizard Figure it out! </span>
<LoadingSpan v-else message="Thinking" />
</GButton>
<div :class="props.view == 'wizard' && 'mt-4'">
<div v-if="busy">
<div v-if="busy" data-description="galaxy wizard loading">
<BSkeleton animation="wave" width="85%" />
<BSkeleton animation="wave" width="55%" />
<BSkeleton animation="wave" width="70%" />
</div>
<div v-else>
<!-- eslint-disable-next-line vue/no-v-html -->
<div class="chatResponse" v-html="renderMarkdown(queryResponse)" />
<div
class="chatResponse"
data-description="galaxy wizard response"
v-html="renderMarkdown(queryResponse)" />
<template v-if="errorMessage">
<hr class="error-divider" />
@@ -112,13 +121,17 @@ async function sendFeedback(value: "up" | "down") {
</template>
</div>
<div v-if="queryResponse && !hasError" class="feedback-buttons mt-2">
<div
v-if="queryResponse && !hasError"
class="feedback-buttons mt-2"
data-description="galaxy wizard feedback">
<hr class="w-100" />
<h4>Was this answer helpful?</h4>
<GButton
color="green"
:disabled="feedback !== null"
:class="{ submitted: feedback === 'up' }"
data-description="galaxy wizard feedback up"
@click="sendFeedback('up')">
<FontAwesomeIcon :icon="faThumbsUp" fixed-width />
</GButton>
@@ -126,11 +139,12 @@ async function sendFeedback(value: "up" | "down") {
color="red"
:disabled="feedback !== null"
:class="{ submitted: feedback === 'down' }"
data-description="galaxy wizard feedback down"
@click="sendFeedback('down')">
<FontAwesomeIcon :icon="faThumbsDown" fixed-width />
</GButton>
<i v-if="!feedback">This feedback helps us improve our responses.</i>
<i v-else>Thank you for your feedback!</i>
<i v-else data-description="galaxy wizard feedback ack">Thank you for your feedback!</i>
</div>
</div>
</div>
@@ -1475,6 +1475,35 @@ chatgxy:
feedback_ack: '.entry-response .feedback-ack'
meta_tag: '.entry-response .meta-tag'
galaxy_wizard:
selectors:
_:
selector: 'galaxy wizard'
type: data-description
analyze_button:
selector: 'galaxy wizard analyze button'
type: data-description
loading:
selector: 'galaxy wizard loading'
type: data-description
response:
selector: 'galaxy wizard response'
type: data-description
feedback_section:
selector: 'galaxy wizard feedback'
type: data-description
feedback_up:
selector: 'galaxy wizard feedback up'
type: data-description
feedback_down:
selector: 'galaxy wizard feedback down'
type: data-description
feedback_ack:
selector: 'galaxy wizard feedback ack'
type: data-description
wizard_card:
selector: 'galaxy wizard card'
type: data-description
zip_import_wizard:
selectors:
+15
View File
@@ -1634,6 +1634,21 @@ class NavigatesGalaxy(HasDriverProxy[WaitType]):
assert len(chatgxy.query_cell.all()) == 0
assert len(chatgxy.response_content.all()) == 0
def navigate_to_dataset_error(self, hid):
"""Display a dataset and click the error tab."""
self.display_dataset(hid)
error_tab = self.wait_for_selector_clickable(
".nav-item[title='View error information for this dataset'] > a.nav-link"
)
error_tab.click()
def galaxy_wizard_analyze(self):
"""Click the wizard analyze button and wait for the response."""
wizard = self.components.galaxy_wizard
wizard.analyze_button.wait_for_and_click()
# Button disappears once queryResponse is set (v-if="!queryResponse")
wizard.analyze_button.wait_for_absent_or_hidden()
def navigate_to_pages(self):
self.home()
self.components.pages.activity.wait_for_and_click()
@@ -0,0 +1,69 @@
"""E2E tests for the GalaxyWizard inline error analysis widget.
Uses the static agent backend for deterministic assertions — no LLM calls.
Skipped when agents are not configured (skip_without_agents decorator).
"""
from galaxy_test.base.populators import skip_without_agents
from .framework import (
managed_history,
retry_assertion_during_transitions,
selenium_test,
SeleniumTestCase,
)
class TestGalaxyWizard(SeleniumTestCase):
ensure_registered = True
def create_failed_dataset(self):
"""Run detect_errors tool with stderr output to produce a failed dataset."""
history_id = self.current_history_id()
inputs = {
"stdoutmsg": "",
"stderrmsg": "error: tool configuration failure detected",
"exit_code": "6",
}
response = self.dataset_populator.run_tool("detect_errors", inputs, history_id)
self.dataset_populator.wait_for_history(history_id, assert_ok=False)
failed_hid = response["outputs"][0]["hid"]
return history_id, failed_hid
@skip_without_agents
@selenium_test
@managed_history
def test_wizard_error_analysis_flow(self):
"""Create failed dataset, analyze error, verify response and feedback."""
# Setup: create a failed dataset via API
history_id, failed_hid = self.create_failed_dataset()
self.history_panel_wait_for_hid_state(failed_hid, "error")
self.screenshot("galaxy_wizard_error_dataset_in_history")
# Navigate to error view
self.navigate_to_dataset_error(failed_hid)
wizard = self.components.galaxy_wizard
# Wizard section visible (agents configured)
wizard.analyze_button.wait_for_visible()
self.screenshot("galaxy_wizard_before_analyze")
# Click analyze, wait for response
self.galaxy_wizard_analyze()
self.screenshot("galaxy_wizard_response_received")
# Verify response content from static backend
@retry_assertion_during_transitions
def assert_response():
assert "tool configuration issue" in wizard.response.wait_for_text()
assert_response()
# Feedback
wizard.feedback_up.wait_for_and_click()
@retry_assertion_during_transitions
def assert_feedback():
assert "Thank you" in wizard.feedback_ack.wait_for_text()
assert_feedback()
self.screenshot("galaxy_wizard_feedback_submitted")
+2 -2
View File
@@ -15,7 +15,7 @@
echo '$stdoutmsg' &&
#end if
#if str($stderrmsg) != ""
>2& echo '$stderrmsg' &&
>&2 echo '$stderrmsg' &&
#end if
sh -c 'exit $exit_code'
]]>
@@ -37,10 +37,10 @@ sh -c 'exit $exit_code'
<param name="exit_code" value="3" />
<assert_stdout>
<has_line line="Log: some program message of interest"/>
<has_line line="Warning: Low space on device"/>
</assert_stdout>
<assert_stderr>
<has_line line="Warning: Exit code 3 (Low disk space)"/>
<has_line line="Warning: Low space on device"/>
</assert_stderr>
</test>
<!-- unccessful run (fatal exit code + warning in output messages are detected) -->
+2 -2
View File
@@ -948,8 +948,8 @@ class TestExpectations(FunctionalTestToolTestCase):
tests = tests_dict["tests"]
assert len(tests) == 10
test_0 = tests[0]
assert len(test_0["stderr"]) == 1
assert len(test_0["stdout"]) == 2
assert len(test_0["stderr"]) == 2
assert len(test_0["stdout"]) == 1
class TestExpectationsCommandVersion(FunctionalTestToolTestCase):