opensource/n8n
3
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-08-29 01:39:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(editor): Split test case detail into config and latest-run panes (no-changelog) (#34371)

Browse Source
This commit is contained in:
Benjamin Schroth
2026-08-10 16:34:51 +02:00
committed by GitHub
parent 5532736d47
commit ec92ffcd56
20 changed files with 715 additions and 3551 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/frontend/@n8n/i18n/src/locales/en.json
+6 -67
View File
@@ -6328,11 +6328,6 @@
"evaluations.setupWizard.step4.button": "Run evaluation",
"evaluations.setupWizard.step4.altButton": "Run in editor",
"evaluations.setupWizard.limitReached": "Limit reached. Your plan includes custom metrics for one workflow only. Upgrade for unlimited use or delete the workflow with existing evaluation runs.",
"evaluations.wizardSidepanel.cancel": "Cancel",
"evaluations.wizardSidepanel.step1.newCustomCheck": "New custom check",
"evaluations.wizardSidepanel.step1.exploreMoreChecks": "Explore more checks",
"evaluations.wizardSidepanel.step1.judgeLabel": "Model",
"evaluations.wizardSidepanel.step1.judgeMissingCredential": "Pick a credential to use this provider",
"evaluations.wizardSidepanel.metric.judgeTag": "AI-judged",
"evaluations.wizardSidepanel.metric.correctness.label": "Correctness",
"evaluations.wizardSidepanel.metric.correctness.description": "Does the AI’s answer mean the same thing as your expected output? An AI judge gives a 1–5 score.",
@@ -6344,8 +6339,6 @@
"evaluations.wizardSidepanel.metric.categorization.description": "The AI’s answer must be identical to your expected output.",
"evaluations.wizardSidepanel.metric.toolsUsed.label": "Right tools used",
"evaluations.wizardSidepanel.metric.toolsUsed.description": "Did the AI use the tools you expected it to?",
"evaluations.wizardSidepanel.metric.customMetrics.label": "Custom Metrics",
"evaluations.wizardSidepanel.metric.customMetrics.description": "Define your own metric(s).",
"evaluations.addToDataset.button.label": "Add to evaluation dataset",
"evaluations.addToDataset.button.tooltip.noConfig": "Create an evaluation with a data table dataset to add executions to it",
"evaluations.addToDataset.title": "Add execution to evaluation dataset",
@@ -6361,57 +6354,6 @@
"evaluations.addToDataset.success.title": "Execution added to dataset",
"evaluations.addToDataset.error.candidate": "Could not load execution data",
"evaluations.addToDataset.error.submit": "Could not add execution to dataset",
"evaluations.wizardSidepanel.step2.aiNode": "AI node",
"evaluations.wizardSidepanel.step2.aiNode.placeholder": "Pick an AI node",
"evaluations.wizardSidepanel.step2.extendToSlice": "Need to evaluate a wider section of the workflow?",
"evaluations.wizardSidepanel.step2.resetToAiNode": "Select single node",
"evaluations.wizardSidepanel.step2.start": "Start",
"evaluations.wizardSidepanel.step2.start.placeholder": "Pick a node",
"evaluations.wizardSidepanel.step2.end": "End",
"evaluations.wizardSidepanel.step2.end.placeholder": "Pick a node",
"evaluations.wizardSidepanel.step2.input": "Input",
"evaluations.wizardSidepanel.step2.input.placeholder": "Type the input you want to test with",
"evaluations.wizardSidepanel.step2.expected": "Expected output",
"evaluations.wizardSidepanel.step2.expected.placeholder": "Type the output you’d expect from your workflow",
"evaluations.wizardSidepanel.step2.expected.tooltip": "Checks compare the real output against this.",
"evaluations.wizardSidepanel.step2.expectedAnswer": "Expected answer",
"evaluations.wizardSidepanel.step2.expectedTools": "Expected tools",
"evaluations.wizardSidepanel.step2.expectedFieldPlaceholder": "Type the value checks should compare against",
"evaluations.wizardSidepanel.step2.inputs.heading": "Inputs",
"evaluations.wizardSidepanel.step2.expected.heading": "Expected outputs",
"evaluations.wizardSidepanel.step2.persistError": "Couldn’t save your test case. Please try again.",
"evaluations.wizardSidepanel.step2.dispatchError": "Saved your test case, but couldn’t start the evaluation run. Click Next again to retry.",
"evaluations.wizardSidepanel.step3.running": "Running tests…",
"evaluations.wizardSidepanel.step3.scoring": "Scoring…",
"evaluations.wizardSidepanel.step3.caseLabel": "Case #{index}",
"evaluations.wizardSidepanel.step3.aiAnswer": "AI’s answer",
"evaluations.wizardSidepanel.step3.averageLabel": "Average",
"evaluations.wizardSidepanel.step3.passedLabel": "Passed",
"evaluations.wizardSidepanel.step3.outputMeta": "Tokens: {tokens} · Time: {time}",
"evaluations.wizardSidepanel.step3.noRun": "Run the evaluation to see results.",
"evaluations.wizardSidepanel.gate.message": "Run your workflow at least once first — it tells us what inputs your AI takes.",
"evaluations.wizardSidepanel.step.chooseSystem.title": "What are we evaluating?",
"evaluations.wizardSidepanel.step.chooseSystem.description": "Pick the AI node in your workflow you want to check.",
"evaluations.wizardSidepanel.step.setupChecks.title": "How should we score each case?",
"evaluations.wizardSidepanel.step.setupChecks.description": "Correctness is the most common way to score an AI’s answers — we’ve selected it for you. Add more checks if you want to score on other dimensions too.",
"evaluations.wizardSidepanel.step.addTestCases.title": "Add the cases you want to test",
"evaluations.wizardSidepanel.step.addTestCases.description": "Give the AI an example input, then write what you’d want it to do. n8n uses each case to score your AI when you run your Eval.",
"evaluations.wizardSidepanel.step.addTestCases.runFirst": "Run your workflow first so we can help you add a test case",
"evaluations.wizardSidepanel.step.addTestCases.runButton": "Run workflow",
"evaluations.wizardSidepanel.step.results.title": "Your Eval results",
"evaluations.wizardSidepanel.step.results.description": "Here’s how your AI did on each case. Run again to see if your changes helped.",
"evaluations.wizardSidepanel.nav.chooseSystem": "Choose system",
"evaluations.wizardSidepanel.nav.setupChecks": "Setup checks",
"evaluations.wizardSidepanel.nav.addTestCases": "Add test cases",
"evaluations.wizardSidepanel.nav.runTests": "Run tests",
"evaluations.wizardSidepanel.nav.runAgain": "Run again",
"evaluations.wizardSidepanel.nav.viewResults": "View Results",
"evaluations.wizardSidepanel.nav.next.checks": "Next: Choose how to score",
"evaluations.wizardSidepanel.nav.next.cases": "Next: Add cases",
"evaluations.wizardSidepanel.nav.next.run": "Next: Run Eval",
"evaluations.wizardSidepanel.nav.back": "Back",
"evaluations.wizardSidepanel.nav.editEvals": "Edit evals",
"evaluations.wizardSidepanel.nav.viewDetailedResults": "View detailed results",
"evaluations.wizardSidepanel.hydrate.error": "Couldn’t load your previous setup. Starting with a blank wizard.",
"evaluations.canvasInfoCard.title": "Setup evaluations for workflows",
"evaluations.canvasInfoCard.description": "Catch issues early and build with confidence with automated tests using model-graded checks.",
@@ -6426,18 +6368,11 @@
"evaluations.emptyState.measurePerformance.title": "Measure performance",
"evaluations.emptyState.measurePerformance.description": "Track performance and areas for improvement in workflows.",
"evaluations.emptyState.getStarted": "Get started",
"evaluations.wizardSidepanel.customCheck.title": "Custom check",
"evaluations.wizardSidepanel.customCheck.name": "Name",
"evaluations.wizardSidepanel.customCheck.name.placeholder": "My custom check",
"evaluations.wizardSidepanel.customCheck.expression": "Expression",
"evaluations.wizardSidepanel.customCheck.expression.hint": "Must return a numeric or boolean value. Reference workflow data with $json (e.g. $json.output.length).",
"evaluations.wizardSidepanel.customCheck.expression.placeholder": "e.g. $json.output.length > 0",
"evaluations.wizardSidepanel.customCheck.expressionTag": "Expression",
"evaluations.wizardSidepanel.customCheck.submit": "Add check",
"evaluations.wizardSidepanel.customCheck.remove": "Remove custom check",
"evaluations.tests.addTestCase": "Add test case",
"evaluations.tests.runAll": "Run all",
"evaluations.tests.runAll.error": "Couldn't run all test cases. Please try again.",
"evaluations.tests.persistError": "Couldn’t save your test case. Please try again.",
"evaluations.tests.dispatchError": "Saved your test case, but couldn’t start the evaluation run. Please try again.",
"evaluations.tests.list.heading": "Test cases",
"evaluations.tests.list.empty": "No test cases yet. Add one below or start from a recent run.",
"evaluations.tests.list.caseLabel": "Test case {index}",
@@ -6482,9 +6417,13 @@
"evaluations.tests.detail.expected.heading": "Expected output",
"evaluations.tests.detail.run": "Run",
"evaluations.tests.detail.runAgain": "Run again",
"evaluations.tests.detail.config": "Configuration",
"evaluations.tests.detail.latestRun": "Latest run",
"evaluations.tests.detail.when": "When",
"evaluations.tests.detail.receivesInput": "receives input",
"evaluations.tests.detail.output": "Output",
"evaluations.tests.detail.expectedAnswer": "Expected answer",
"evaluations.tests.detail.expectedTools": "Expected tools",
"evaluations.tests.detail.addMetric": "Add check",
"evaluations.tests.detail.addMetric.custom": "Custom",
"evaluations.tests.detail.removeMetric": "Remove metric",
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/Tests/TestCaseDetail.test.ts
+73 -11
View File
@@ -1,11 +1,13 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { createTestingPinia } from '@pinia/testing';
import userEvent from '@testing-library/user-event';
import { waitFor } from '@testing-library/vue';
import { ref } from 'vue';
import { createComponentRenderer } from '@/__tests__/render';
import TestCaseDetail from './TestCaseDetail.vue';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { useEvaluationStore } from '../../evaluation.store';
// ─── Module-level mocks ───────────────────────────────────────────────────────
@@ -84,16 +86,13 @@ vi.mock('@/features/execution/executions/executions.store', () => ({
}));
// Stub heavy children not under test.
vi.mock('./TestCaseResultCard.vue', () => ({
vi.mock('./TestCaseRunResult.vue', () => ({
default: {
name: 'TestCaseResultCard',
props: ['index'],
template: '<div :data-test-id="`stub-result-card-${index}`" />',
name: 'TestCaseRunResult',
props: ['index', 'expanded', 'separated'],
template: '<div :data-test-id="`stub-run-result-${index}`" />',
},
}));
vi.mock('../WizardSidepanel/CustomCheckModal.vue', () => ({
default: { name: 'CustomCheckModal', template: '<div />' },
}));
// ─── Renderer ────────────────────────────────────────────────────────────────
@@ -128,6 +127,39 @@ describe('TestCaseDetail', () => {
expect(mockPersistAndRunCase).toHaveBeenCalledWith('initial');
});
// The Config pane's collapse state lives on the reka-ui CollapsibleContent
// wrapping our config content.
function configPaneState(container: Element): string | null | undefined {
return container
.querySelector('[data-test-id="tests-detail-config"]')
?.closest('[data-state]')
?.getAttribute('data-state');
}
it('collapses the Config pane once a run dispatches successfully', async () => {
setup();
const { getByTestId, container } = renderComponent();
expect(configPaneState(container)).toBe('open');
await userEvent.click(getByTestId('tests-detail-run'));
// handleRun collapses Config only after persistAndRunCase resolves true.
await waitFor(() => expect(configPaneState(container)).toBe('closed'));
});
it('leaves the Config pane open when the run fails to dispatch', async () => {
setup();
mockPersistAndRunCase.mockResolvedValueOnce(false);
const { getByTestId, container } = renderComponent();
await userEvent.click(getByTestId('tests-detail-run'));
// Let the (failed) handler settle, then confirm Config was never collapsed —
// otherwise the user is stranded on a closed pane with no result to show.
await new Promise((resolve) => setTimeout(resolve, 0));
expect(configPaneState(container)).toBe('open');
});
it('returns to the list when the breadcrumb root is clicked', async () => {
const store = setup();
const { getByTestId } = renderComponent();
@@ -180,16 +212,46 @@ describe('TestCaseDetail', () => {
expect(getByText('$json.x > 1')).toBeInTheDocument();
});
it('reuses the overview result card once a run is active for the case', async () => {
it('shows the latest-run pane when the pinned run covers the case', async () => {
const store = setup();
const evaluationStore = useEvaluationStore();
store.setActiveRow(2, 20);
store.setActiveRunId('run-1');
// The pinned run has this row's execution → pane visible, keyed by row index.
evaluationStore.testCaseExecutionsById = {
c1: { id: 'c1', testRunId: 'run-1', runIndex: 2, status: 'success' },
} as never;
const { findByTestId } = renderComponent();
// Keyed by the active row index (= the case's runIndex).
expect(await findByTestId('stub-result-card-2')).toBeInTheDocument();
expect(await findByTestId('stub-run-result-2')).toBeInTheDocument();
});
it('does not show the result card before a run is triggered', () => {
it('hides the latest-run pane when the pinned run has no execution for this case', () => {
const store = setup();
const evaluationStore = useEvaluationStore();
// The pinned run ran row 0 only...
evaluationStore.testRunsById = { 'run-1': { id: 'run-1', status: 'success' } } as never;
evaluationStore.testCaseExecutionsById = {
c0: { id: 'c0', testRunId: 'run-1', runIndex: 0, status: 'success' },
} as never;
// ...but we navigated to row 1 (whose sticky activeRunId is still run-1).
store.setActiveRow(1, 11);
store.setActiveRunId('run-1');
const { queryByTestId } = renderComponent();
// No empty pane, no false "running" — the run didn't cover this case.
expect(queryByTestId('tests-detail-results')).toBeNull();
});
it('shows the latest-run pane while a run is dispatching for this case', async () => {
const store = setup();
store.setActiveRow(0, 10);
const { getByTestId, findByTestId } = renderComponent();
// No execution seeded yet; the dispatch flag keeps the pane visible so the
// "running" state shows before the first poll lands the execution.
await userEvent.click(getByTestId('tests-detail-run'));
expect(await findByTestId('tests-detail-results')).toBeInTheDocument();
});
it('does not show the latest-run pane before a run is triggered', () => {
setup();
const { queryByTestId } = renderComponent();
expect(queryByTestId('tests-detail-results')).toBeNull();
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/Tests/TestCaseDetail.vue
+182 -110
View File
@@ -1,11 +1,12 @@
<script setup lang="ts">
import { computed, ref } from 'vue';
import { computed, ref, watch } from 'vue';
import { storeToRefs } from 'pinia';
import { useDebounceFn } from '@vueuse/core';
import { useI18n, type BaseTextKey } from '@n8n/i18n';
import {
N8nActionDropdown,
N8nButton,
N8nCollapsiblePanel,
N8nIcon,
N8nInlineTextEdit,
N8nInput,
@@ -19,6 +20,7 @@ import { useMessage } from '@/app/composables/useMessage';
import { injectWorkflowDocumentStore } from '@/app/stores/workflowDocument.store';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { useEvaluationStore } from '../../evaluation.store';
import { useTestCasePersistence } from '../../composables/useTestCasePersistence';
import { useSliceInputs } from '../../composables/useSliceInputs';
import {
@@ -26,7 +28,7 @@ import {
CANNED_METRIC_EXPECTED_FIELDS,
type CannedMetricKey,
} from '../../evaluation.constants';
import TestCaseResultCard from './TestCaseResultCard.vue';
import TestCaseRunResult from './TestCaseRunResult.vue';
// Each metric renders as a sentence under "Output".
const METRIC_SENTENCE_KEY: Record<CannedMetricKey, BaseTextKey> = {
@@ -38,6 +40,7 @@ const METRIC_SENTENCE_KEY: Record<CannedMetricKey, BaseTextKey> = {
};
const wizardStore = useEvaluationsWizardSidepanelStore();
const evaluationStore = useEvaluationStore();
const workflowDocumentStore = injectWorkflowDocumentStore();
const locale = useI18n();
const message = useMessage();
@@ -194,15 +197,66 @@ function toggleTool(name: string) {
// ─── Run / result ────────────────────────────────────────────────────────────
// Once a run has been triggered for this case, show its outcome with the same
// result card used on the overview list, keyed by the row's index (which equals
// the case's runIndex). `persistAndRunCase` sets both before the run dispatches.
const resultCardIndex = computed<number | null>(() =>
activeRunId.value !== null ? activeRowIndex.value : null,
// The two panes: config and the latest run. Both open on entry; starting a run
// collapses config and opens the run pane, so focus shifts to the result.
const configOpen = ref(true);
const runOpen = ref(true);
// The run pinned to this pane. `activeRunId` is a sticky global (survives
// navigation, re-pinned to the newest run on hydration), so it identifies "the
// run in view" but NOT that the run covers this case — that's checked below.
const pinnedRunId = computed(() => activeRunId.value);
// Does the pinned run have a case execution for the active row? A run started
// for a different case (or a "Run all" still spinning up) won't, so this keeps
// its result from being shown against — or reported as running for — this case.
const pinnedRunCoversRow = computed(() => {
const runId = pinnedRunId.value;
const row = activeRowIndex.value;
if (runId === null || row === null) return false;
return Object.values(evaluationStore.testCaseExecutionsById ?? {}).some(
(c) => c.testRunId === runId && (c.runIndex ?? 0) === row,
);
});
const pinnedRunTerminal = computed(() => {
const runId = pinnedRunId.value;
if (runId === null) return false;
const status = evaluationStore.testRunsById[runId]?.status;
return Boolean(status) && !['new', 'running'].includes(status);
});
// Set while a run we just started for THIS row hasn't surfaced its execution
// yet, so the pane can show "running" before the first poll seeds the row.
// Cleared once the execution lands or the run finishes without one (so a run
// that never covers this row can't leave the pane stuck).
const isDispatching = ref(false);
watch([pinnedRunCoversRow, pinnedRunTerminal], ([covers, terminal]) => {
if (covers || terminal) isDispatching.value = false;
});
// Show the run pane only when the pinned run pertains to this row — it already
// has this row's execution, or we're mid-dispatch for it.
const showRunPane = computed(
() => activeRowIndex.value !== null && (pinnedRunCoversRow.value || isDispatching.value),
);
// Row index the result is keyed by (equals the case's runIndex).
const runResultIndex = computed(() => activeRowIndex.value);
async function handleRun() {
await persistAndRunCase(activeRunId.value ? 'run_again' : 'initial');
// Mark dispatch so the pane shows "running" through the persist chain and the
// gap before polling seeds this row's execution.
isDispatching.value = true;
const ok = await persistAndRunCase(activeRunId.value ? 'run_again' : 'initial');
if (ok) {
configOpen.value = false;
runOpen.value = true;
} else {
// Nothing dispatched: drop the flag so the pane doesn't linger on a spinner,
// and Config stays open (never collapsed) so the user isn't stranded.
isDispatching.value = false;
}
}
</script>
@@ -259,117 +313,128 @@ async function handleRun() {
</header>
<div :class="$style.body">
<!-- When <node> receives input -->
<div :class="$style.block">
<!-- Node under test is configured at the suite level; read-only here. -->
<div :class="$style.sentence">
<N8nText size="small" color="text-dark">
{{ locale.baseText('evaluations.tests.detail.when') }}
</N8nText>
<N8nText size="small" color="text-dark" bold data-test-id="tests-detail-ai-node">
{{ aiNodeName }}
</N8nText>
<N8nText size="small" color="text-dark">
{{ locale.baseText('evaluations.tests.detail.receivesInput') }}
</N8nText>
</div>
<div :class="$style.indented">
<div
v-for="name in sliceInputs.fieldNames"
:key="`input-${name}`"
:class="$style.field"
:data-test-id="`tests-detail-input-${name}`"
>
<N8nText size="small" color="text-dark">{{ name }}</N8nText>
<N8nInput
:model-value="inputs[name] ?? ''"
type="textarea"
:rows="2"
size="small"
@update:model-value="onInputEdit(name, $event)"
/>
</div>
</div>
</div>
<!-- Output: one sentence per metric -->
<div :class="$style.block">
<N8nText size="small" color="text-dark" bold>
{{ locale.baseText('evaluations.tests.detail.output') }}
</N8nText>
<div :class="$style.indented">
<div
v-for="metric in selectedCannedMetrics"
:key="metric.key"
:class="$style.metric"
:data-test-id="`tests-detail-metric-${metric.key}`"
>
<N8nText size="small" color="text-dark">{{ metricSentence(metric.key) }}</N8nText>
<!-- Tool-usage: pick from the node's connected tools (per-case expected) -->
<div v-if="metric.key === 'toolsUsed'" :class="$style.tools">
<button
v-for="tool in connectedTools"
:key="tool"
type="button"
:class="$style.toolItem"
:data-test-id="`tests-detail-tool-${tool}`"
@click="toggleTool(tool)"
>
<span :class="[$style.box, selectedTools.has(tool) ? $style.boxChecked : null]">
<N8nIcon v-if="selectedTools.has(tool)" icon="check" size="xsmall" />
</span>
<N8nText size="small" color="text-dark">{{ tool }}</N8nText>
</button>
<N8nText v-if="connectedTools.length === 0" size="small" color="text-light">
{{ locale.baseText('evaluations.tests.detail.tools.empty') }}
<!-- Config: the trigger, its inputs and the expected output -->
<N8nCollapsiblePanel
v-model="configOpen"
:title="locale.baseText('evaluations.tests.detail.config')"
>
<div :class="$style.paneContent" data-test-id="tests-detail-config">
<!-- When <node> receives input -->
<div :class="$style.block">
<!-- Node under test is configured at the suite level; read-only here. -->
<div :class="$style.sentence">
<N8nText size="small" color="text-dark">
{{ locale.baseText('evaluations.tests.detail.when') }}
</N8nText>
<N8nText size="small" color="text-dark" bold data-test-id="tests-detail-ai-node">
{{ aiNodeName }}
</N8nText>
<N8nText size="small" color="text-dark">
{{ locale.baseText('evaluations.tests.detail.receivesInput') }}
</N8nText>
</div>
<!-- Metrics with an expected value (similarity / categorization) -->
<N8nInput
v-else-if="expectedFieldFor(metric.key)"
:model-value="expectedValues[expectedFieldFor(metric.key)!.name] ?? ''"
type="textarea"
:rows="3"
size="small"
:data-test-id="`tests-detail-expected-${expectedFieldFor(metric.key)!.name}`"
@update:model-value="onExpectedEdit(expectedFieldFor(metric.key)!.name, $event)"
/>
<div :class="$style.indented">
<div
v-for="name in sliceInputs.fieldNames"
:key="`input-${name}`"
:class="$style.field"
:data-test-id="`tests-detail-input-${name}`"
>
<N8nText size="small" color="text-dark">{{ name }}</N8nText>
<N8nInput
:model-value="inputs[name] ?? ''"
type="textarea"
:rows="2"
size="small"
@update:model-value="onInputEdit(name, $event)"
/>
</div>
</div>
</div>
<!-- Custom checks (expression based) — defined at the suite level, read-only here -->
<div
v-for="check in customChecks"
:key="check.id"
:class="$style.metric"
:data-test-id="`tests-detail-custom-${check.id}`"
>
<N8nText size="small" color="text-dark">
{{ locale.baseText('evaluations.tests.metric.custom.sentence') }}
</N8nText>
<N8nText
size="small"
color="text-dark"
:class="$style.expression"
:data-test-id="`tests-detail-custom-expression-${check.id}`"
>
{{ check.expression }}
<!-- Output: one sentence per metric -->
<div :class="$style.block">
<N8nText size="small" color="text-dark" bold>
{{ locale.baseText('evaluations.tests.detail.output') }}
</N8nText>
<div :class="$style.indented">
<div
v-for="metric in selectedCannedMetrics"
:key="metric.key"
:class="$style.metric"
:data-test-id="`tests-detail-metric-${metric.key}`"
>
<N8nText size="small" color="text-dark">{{ metricSentence(metric.key) }}</N8nText>
<!-- Tool-usage: pick from the node's connected tools (per-case expected) -->
<div v-if="metric.key === 'toolsUsed'" :class="$style.tools">
<button
v-for="tool in connectedTools"
:key="tool"
type="button"
:class="$style.toolItem"
:data-test-id="`tests-detail-tool-${tool}`"
@click="toggleTool(tool)"
>
<span :class="[$style.box, selectedTools.has(tool) ? $style.boxChecked : null]">
<N8nIcon v-if="selectedTools.has(tool)" icon="check" size="xsmall" />
</span>
<N8nText size="small" color="text-dark">{{ tool }}</N8nText>
</button>
<N8nText v-if="connectedTools.length === 0" size="small" color="text-light">
{{ locale.baseText('evaluations.tests.detail.tools.empty') }}
</N8nText>
</div>
<!-- Metrics with an expected value (similarity / categorization) -->
<N8nInput
v-else-if="expectedFieldFor(metric.key)"
:model-value="expectedValues[expectedFieldFor(metric.key)!.name] ?? ''"
type="textarea"
:rows="3"
size="small"
:data-test-id="`tests-detail-expected-${expectedFieldFor(metric.key)!.name}`"
@update:model-value="onExpectedEdit(expectedFieldFor(metric.key)!.name, $event)"
/>
</div>
<!-- Custom checks (expression based) — defined at the suite level, read-only here -->
<div
v-for="check in customChecks"
:key="check.id"
:class="$style.metric"
:data-test-id="`tests-detail-custom-${check.id}`"
>
<N8nText size="small" color="text-dark">
{{ locale.baseText('evaluations.tests.metric.custom.sentence') }}
</N8nText>
<N8nText
size="small"
color="text-dark"
:class="$style.expression"
:data-test-id="`tests-detail-custom-expression-${check.id}`"
>
{{ check.expression }}
</N8nText>
</div>
</div>
</div>
</div>
</div>
</N8nCollapsiblePanel>
<!-- Result: same card as the overview list, for a consistent look. -->
<div
v-if="resultCardIndex !== null"
:class="$style.block"
data-test-id="tests-detail-results"
<!-- Latest run: the outcome of the pinned run for this case -->
<N8nCollapsiblePanel
v-if="showRunPane && runResultIndex !== null"
v-model="runOpen"
:title="locale.baseText('evaluations.tests.detail.latestRun')"
>
<TestCaseResultCard :index="resultCardIndex" />
</div>
<div :class="$style.paneContent" data-test-id="tests-detail-results">
<!-- Pin the exact run and only fetch the full output while open. -->
<TestCaseRunResult :index="runResultIndex" :run-id="pinnedRunId" :expanded="runOpen" />
</div>
</N8nCollapsiblePanel>
</div>
</div>
</template>
@@ -425,7 +490,14 @@ async function handleRun() {
padding: var(--spacing--sm) var(--spacing--md) var(--spacing--md);
display: flex;
flex-direction: column;
gap: var(--spacing--lg);
gap: var(--spacing--sm);
}
// Inner layout of a collapsible pane: its blocks stacked vertically.
.paneContent {
display: flex;
flex-direction: column;
gap: var(--spacing--md);
}
.block {
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/Tests/TestCaseResultCard.vue
+5 -304
View File
@@ -1,37 +1,10 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue';
import { useI18n, type BaseTextKey } from '@n8n/i18n';
import { computed, ref } from 'vue';
import { useI18n } from '@n8n/i18n';
import { N8nIcon, N8nText } from '@n8n/design-system';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { useEvaluationStore } from '../../evaluation.store';
import { useExecutionsStore } from '@/features/execution/executions/executions.store';
import type { IExecutionResponse } from '@/features/execution/executions/executions.types';
import { injectWorkflowDocumentStore } from '@/app/stores/workflowDocument.store';
import { CANNED_METRICS, cannedMetricLabelKey, getErrorBaseKey } from '../../evaluation.constants';
import {
casePassed,
extractCaseAnswer,
formatDuration,
formatMetricPercent,
formatMetricRawScore,
formatShortDateTime,
formatTokens,
getMetricCategory,
getOperationalMetricEntries,
getUserDefinedMetricNames,
} from '../../evaluation.utils';
import type { TestCaseExecutionRecord, TestRunRecord } from '../../evaluation.api';
const TERMINAL_CASE_STATUSES = ['success', 'error', 'warning', 'cancelled'];
// Labels for the operational metrics shown as gray text under the results.
const OPERATIONAL_LABEL: Record<string, BaseTextKey> = {
promptTokens: 'evaluations.tests.results.metric.promptTokens',
completionTokens: 'evaluations.tests.results.metric.completionTokens',
totalTokens: 'evaluations.tests.results.metric.totalTokens',
executionTime: 'evaluations.tests.results.metric.executionTime',
};
import TestCaseRunResult from './TestCaseRunResult.vue';
const props = defineProps<{
index: number;
@@ -39,9 +12,6 @@ const props = defineProps<{
const locale = useI18n();
const wizardStore = useEvaluationsWizardSidepanelStore();
const evaluationStore = useEvaluationStore();
const executionsStore = useExecutionsStore();
const workflowDocumentStore = injectWorkflowDocumentStore();
const expanded = ref(false);
@@ -58,141 +28,6 @@ const inputEntries = computed<Array<{ name: string; value: string }>>(() => {
return Object.entries(row).map(([name, value]) => ({ name, value: String(value) }));
});
// Runs for this workflow, oldest first — used to derive a stable run number.
const workflowRuns = computed<TestRunRecord[]>(() => {
const workflowId = workflowDocumentStore.value?.workflowId;
if (!workflowId) return [];
return [...(evaluationStore.testRunsByWorkflowId[workflowId] ?? [])].sort(
(a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime(),
);
});
const latestRun = computed<TestRunRecord | undefined>(
() => workflowRuns.value[workflowRuns.value.length - 1],
);
const runNumber = computed(() => workflowRuns.value.length);
const runDateLabel = computed(() => {
const run = latestRun.value;
if (!run) return '';
return formatShortDateTime(run.runAt ?? run.createdAt);
});
// This row's case execution within the latest run.
const caseExecution = computed<TestCaseExecutionRecord | undefined>(() => {
const runId = latestRun.value?.id;
if (!runId) return undefined;
return Object.values(evaluationStore.testCaseExecutionsById ?? {}).find(
(c) => c.testRunId === runId && (c.runIndex ?? 0) === props.index,
);
});
// A run is in progress for this case until its execution reaches a terminal
// status (or, before it's seeded, while the run itself is new/running).
const isRunning = computed(() => {
const run = latestRun.value;
if (!run) return false;
if (!['new', 'running'].includes(run.status)) return false;
const status = caseExecution.value?.status;
return !status || !TERMINAL_CASE_STATUSES.includes(status);
});
const hasResult = computed(
() =>
Boolean(caseExecution.value && TERMINAL_CASE_STATUSES.includes(caseExecution.value.status)) &&
!isRunning.value,
);
const runPassed = computed(() => caseExecution.value?.status === 'success');
// A terminal case that didn't pass — we surface why beneath the run label.
const runFailed = computed(() => hasResult.value && !runPassed.value);
type MetricBadge = { key: string; label: string; text: string; passed: boolean };
// Per-check score badges for this case (excludes token/time metrics).
const metricBadges = computed<MetricBadge[]>(() => {
const metrics = caseExecution.value?.metrics;
if (!metrics) return [];
return getUserDefinedMetricNames(metrics).map((key) => {
const value = metrics[key];
const canned = CANNED_METRICS.find((m) => m.key === key);
const label = canned ? locale.baseText(cannedMetricLabelKey(canned)) : key;
const category = getMetricCategory(key);
if (category === 'aiBased') {
// AI judges grade 1–5; ≥3/5 reads as "acceptable" for the at-a-glance
// badge colour. No other surface renders AI pass/fail, so this
// card-local threshold can't disagree with one.
return { key, label, text: formatMetricRawScore(value, { category }), passed: value >= 3 };
}
// Deterministic checks pass only on a perfect score, via the shared
// `casePassed` — so a case can't read "pass" here yet "fail" on the run
// detail / wizard, which use the same helper.
return {
key,
label,
text: formatMetricPercent(value, { category }),
passed: casePassed(value),
};
});
});
// Operational metrics (tokens, time) — plain gray text, no percentages.
const operationalMetrics = computed<Array<{ key: string; label: string; text: string }>>(() => {
return getOperationalMetricEntries(caseExecution.value?.metrics).map(({ key, value }) => {
const label = OPERATIONAL_LABEL[key] ? locale.baseText(OPERATIONAL_LABEL[key]) : key;
const text =
key === 'executionTime' ? formatDuration(value) : formatTokens(value, { withUnit: false });
return { key, label, text };
});
});
// The node under test whose output is the case's answer.
const endNodeName = computed(() => wizardStore.answerNodeName);
// The compiled config run has no setOutputs node, so `caseExecution.outputs` is
// empty; the real answer is the end node's output in the run. Fetch the full
// execution lazily (the output only renders when expanded) and extract from it.
const caseExecutionData = ref<IExecutionResponse | null>(null);
async function loadCaseExecution(executionId: string) {
try {
caseExecutionData.value = (await executionsStore.fetchExecution(executionId)) ?? null;
} catch (error) {
console.warn('[TestCaseResultCard] failed to load case execution', error);
}
}
// Load the full execution when the card is expanded (for the output) or when
// the case failed (so its error message is shown without needing to expand).
watch(
[expanded, runFailed, () => caseExecution.value?.executionId],
([isExpanded, failed, executionId]) => {
if (!executionId || (!isExpanded && !failed)) return;
if (caseExecutionData.value?.id === executionId) return;
void loadCaseExecution(executionId);
},
{ immediate: true },
);
const outputText = computed(() =>
extractCaseAnswer(caseExecutionData.value, endNodeName.value, caseExecution.value?.outputs),
);
// Why the run failed: the workflow's own error message when available, else the
// mapped error-code message, else a generic fallback.
const failureReason = computed(() => {
if (!runFailed.value) return '';
const executionError = caseExecutionData.value?.data?.resultData?.error?.message;
if (executionError) return executionError;
const errorKey = getErrorBaseKey(caseExecution.value?.errorCode);
if (errorKey && locale.exists(errorKey)) {
return locale.baseText(errorKey, { interpolate: { link: '' } }).trim();
}
return locale.baseText('evaluations.tests.results.runFailed');
});
function toggle() {
expanded.value = !expanded.value;
}
@@ -243,84 +78,8 @@ function openEdit() {
</div>
</div>
<!-- Running -->
<div
v-if="isRunning"
:class="$style.runSection"
:data-test-id="`tests-result-running-${index}`"
>
<div :class="$style.runLine">
<N8nIcon icon="spinner" size="small" :spin="true" color="text-light" />
<N8nText size="small" color="text-light">
{{ locale.baseText('evaluations.tests.results.running') }}
</N8nText>
</div>
</div>
<!-- Latest run result -->
<div
v-else-if="hasResult"
:class="$style.runSection"
:data-test-id="`tests-result-result-${index}`"
>
<div :class="$style.runLine">
<N8nIcon
:icon="runPassed ? 'check' : 'x'"
size="small"
:color="runPassed ? 'success' : 'danger'"
/>
<N8nText size="small" :color="runPassed ? 'success' : 'danger'">
{{
locale.baseText('evaluations.tests.results.runLabel', {
interpolate: { number: String(runNumber), date: runDateLabel },
})
}}
</N8nText>
</div>
<!-- Failure reason (shown for a failed run, collapsed or expanded) -->
<N8nText
v-if="runFailed && failureReason"
size="small"
color="danger"
:class="$style.failureReason"
:data-test-id="`tests-result-error-${index}`"
>
{{ failureReason }}
</N8nText>
<template v-if="expanded">
<N8nText size="small" color="text-light">
{{ locale.baseText('evaluations.tests.detail.output') }}
</N8nText>
</template>
<div :class="$style.badges">
<span
v-for="badge in metricBadges"
:key="badge.key"
:class="[$style.badge, badge.passed ? $style.badgePass : $style.badgeFail]"
:data-test-id="`tests-result-badge-${index}-${badge.key}`"
>
{{ badge.label }} <strong>{{ badge.text }}</strong>
</span>
</div>
<N8nText v-if="expanded && outputText" size="small" color="text-dark" :class="$style.output">
{{ outputText }}
</N8nText>
<!-- Operational metrics: gray text, only when expanded, no percentages -->
<div
v-if="expanded && operationalMetrics.length > 0"
:class="$style.operational"
:data-test-id="`tests-result-operational-${index}`"
>
<N8nText v-for="m in operationalMetrics" :key="m.key" size="xsmall" color="text-light">
{{ m.label }} {{ m.text }}
</N8nText>
</div>
</div>
<!-- Run outcome — separated from the title/definition by a full-bleed border. -->
<TestCaseRunResult :index="index" :expanded="expanded" separated />
</div>
</template>
@@ -400,62 +159,4 @@ function openEdit() {
gap: var(--spacing--4xs);
margin: 0;
}
// Full-bleed top border separates the title/input from the run result.
.runSection {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
border-top: var(--border);
margin: 0 calc(-1 * var(--spacing--md));
padding: var(--spacing--sm) var(--spacing--md) 0;
}
.runLine {
display: flex;
align-items: center;
gap: var(--spacing--2xs);
}
.badges {
display: flex;
flex-wrap: wrap;
gap: var(--spacing--2xs);
}
.badge {
display: inline-flex;
align-items: center;
gap: var(--spacing--4xs);
padding: var(--spacing--4xs) var(--spacing--2xs);
border-radius: var(--radius--sm);
font-size: var(--font-size--2xs);
}
.badgePass {
background-color: var(--color--success--tint-2, #e9f7ef);
color: var(--color--success--shade-1, #1a8d4a);
}
.badgeFail {
background-color: var(--color--danger--tint-2, #fdecea);
color: var(--color--danger--shade-1, #b3261e);
}
.output {
white-space: pre-wrap;
word-break: break-word;
}
.failureReason {
white-space: pre-wrap;
word-break: break-word;
}
.operational {
display: flex;
flex-wrap: wrap;
gap: var(--spacing--2xs) var(--spacing--sm);
margin-top: var(--spacing--3xs);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/Tests/TestCaseRunResult.test.ts
+96
View File
@@ -0,0 +1,96 @@
import { describe, it, expect, vi } from 'vitest';
import { createTestingPinia } from '@pinia/testing';
import { ref } from 'vue';
import { createComponentRenderer } from '@/__tests__/render';
import TestCaseRunResult from './TestCaseRunResult.vue';
import { useEvaluationStore } from '../../evaluation.store';
vi.mock('@n8n/i18n', async (importOriginal) => ({
...(await importOriginal()),
useI18n: () => ({
baseText: (key: string, opts?: { interpolate?: Record<string, string> }) => {
if (opts?.interpolate) {
return Object.entries(opts.interpolate).reduce(
(str, [k, v]) => str.replace(`{${k}}`, v),
key,
);
}
return key;
},
}),
}));
vi.mock('@/app/stores/workflowDocument.store', () => ({
injectWorkflowDocumentStore: () => ref({ workflowId: 'wf-1' }),
}));
vi.mock('@/features/execution/executions/executions.store', () => ({
useExecutionsStore: () => ({ fetchExecution: vi.fn().mockResolvedValue(null) }),
}));
const renderComponent = createComponentRenderer(TestCaseRunResult);
// Two terminal runs for the same workflow; the older passed this row, the newer
// failed it — so which run is rendered is unambiguous from the badge score.
function seedTwoRuns() {
createTestingPinia({ stubActions: false });
const evaluationStore = useEvaluationStore();
evaluationStore.testRunsById = {
'run-old': {
id: 'run-old',
workflowId: 'wf-1',
status: 'success',
createdAt: '2026-06-19T20:00:00.000Z',
runAt: '2026-06-19T20:00:00.000Z',
},
'run-new': {
id: 'run-new',
workflowId: 'wf-1',
status: 'success',
createdAt: '2026-06-19T21:00:00.000Z',
runAt: '2026-06-19T21:00:00.000Z',
},
} as never;
evaluationStore.testCaseExecutionsById = {
eo: {
id: 'eo',
testRunId: 'run-old',
runIndex: 0,
status: 'success',
metrics: { stringSimilarity: 1 },
},
en: {
id: 'en',
testRunId: 'run-new',
runIndex: 0,
status: 'success',
metrics: { stringSimilarity: 0 },
},
} as never;
return evaluationStore;
}
describe('TestCaseRunResult', () => {
it('renders the pinned run when a runId is given, not the newest run', () => {
seedTwoRuns();
const { getByTestId } = renderComponent({ props: { index: 0, runId: 'run-old' } });
// run-old scored this row 100% — proves it pinned the older run, not run-new.
expect(getByTestId('tests-result-badge-0-stringSimilarity').textContent).toContain('100%');
});
it('falls back to the newest run when no runId is given (overview cards)', () => {
seedTwoRuns();
const { getByTestId } = renderComponent({ props: { index: 0 } });
// run-new is chronologically newest and scored this row 0%.
expect(getByTestId('tests-result-badge-0-stringSimilarity').textContent).toContain('0%');
});
it('renders nothing when the target run has no execution for this row', () => {
seedTwoRuns();
// run-old is terminal and never ran row 1 → no running state, no result.
const { queryByTestId } = renderComponent({ props: { index: 1, runId: 'run-old' } });
expect(queryByTestId('tests-result-running-1')).toBeNull();
expect(queryByTestId('tests-result-result-1')).toBeNull();
});
});
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/Tests/TestCaseRunResult.vue
+344
View File
@@ -0,0 +1,344 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue';
import { useI18n, type BaseTextKey } from '@n8n/i18n';
import { N8nIcon, N8nText } from '@n8n/design-system';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { useEvaluationStore } from '../../evaluation.store';
import { useExecutionsStore } from '@/features/execution/executions/executions.store';
import type { IExecutionResponse } from '@/features/execution/executions/executions.types';
import { injectWorkflowDocumentStore } from '@/app/stores/workflowDocument.store';
import { CANNED_METRICS, cannedMetricLabelKey, getErrorBaseKey } from '../../evaluation.constants';
import {
casePassed,
extractCaseAnswer,
formatDuration,
formatMetricPercent,
formatMetricRawScore,
formatShortDateTime,
formatTokens,
getMetricCategory,
getOperationalMetricEntries,
getUserDefinedMetricNames,
} from '../../evaluation.utils';
import type { TestCaseExecutionRecord, TestRunRecord } from '../../evaluation.api';
const TERMINAL_CASE_STATUSES = ['success', 'error', 'warning', 'cancelled'];
// Labels for the operational metrics shown as gray text under the results.
const OPERATIONAL_LABEL: Record<string, BaseTextKey> = {
promptTokens: 'evaluations.tests.results.metric.promptTokens',
completionTokens: 'evaluations.tests.results.metric.completionTokens',
totalTokens: 'evaluations.tests.results.metric.totalTokens',
executionTime: 'evaluations.tests.results.metric.executionTime',
};
const props = withDefaults(
defineProps<{
index: number;
// Which run to render. Omitted on the overview cards → the workflow's newest
// run. The detail view pins a specific run so a later, unrelated run landing
// (e.g. a "Run all") can't silently swap in.
runId?: string | null;
// Show the full output (Output label, answer text, operational metrics).
expanded?: boolean;
// Draw a full-bleed top separator — used when embedded below a card's title.
separated?: boolean;
}>(),
{ runId: null, expanded: false, separated: false },
);
const locale = useI18n();
const wizardStore = useEvaluationsWizardSidepanelStore();
const evaluationStore = useEvaluationStore();
const executionsStore = useExecutionsStore();
const workflowDocumentStore = injectWorkflowDocumentStore();
// Runs for this workflow, oldest first — used to derive a stable run number.
const workflowRuns = computed<TestRunRecord[]>(() => {
const workflowId = workflowDocumentStore.value?.workflowId;
if (!workflowId) return [];
return [...(evaluationStore.testRunsByWorkflowId[workflowId] ?? [])].sort(
(a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime(),
);
});
// The run this card renders: the pinned one in the detail view, else the newest.
const targetRun = computed<TestRunRecord | undefined>(() =>
props.runId !== null
? workflowRuns.value.find((r) => r.id === props.runId)
: workflowRuns.value[workflowRuns.value.length - 1],
);
// 1-based position of the shown run among all runs — its "Run #N".
const runNumber = computed(() => {
const id = targetRun.value?.id;
if (!id) return 0;
return workflowRuns.value.findIndex((r) => r.id === id) + 1;
});
const runDateLabel = computed(() => {
const run = targetRun.value;
if (!run) return '';
return formatShortDateTime(run.runAt ?? run.createdAt);
});
// This row's case execution within the shown run.
const caseExecution = computed<TestCaseExecutionRecord | undefined>(() => {
const runId = targetRun.value?.id;
if (!runId) return undefined;
return Object.values(evaluationStore.testCaseExecutionsById ?? {}).find(
(c) => c.testRunId === runId && (c.runIndex ?? 0) === props.index,
);
});
// A run is in progress for this case until its execution reaches a terminal
// status (or, before it's seeded, while the run itself is new/running).
const isRunning = computed(() => {
const run = targetRun.value;
if (!run) return false;
if (!['new', 'running'].includes(run.status)) return false;
const status = caseExecution.value?.status;
return !status || !TERMINAL_CASE_STATUSES.includes(status);
});
const hasResult = computed(
() =>
Boolean(caseExecution.value && TERMINAL_CASE_STATUSES.includes(caseExecution.value.status)) &&
!isRunning.value,
);
const runPassed = computed(() => caseExecution.value?.status === 'success');
// A terminal case that didn't pass — we surface why beneath the run label.
const runFailed = computed(() => hasResult.value && !runPassed.value);
type MetricBadge = { key: string; label: string; text: string; passed: boolean };
// Per-check score badges for this case (excludes token/time metrics).
const metricBadges = computed<MetricBadge[]>(() => {
const metrics = caseExecution.value?.metrics;
if (!metrics) return [];
return getUserDefinedMetricNames(metrics).map((key) => {
const value = metrics[key];
const canned = CANNED_METRICS.find((m) => m.key === key);
const label = canned ? locale.baseText(cannedMetricLabelKey(canned)) : key;
const category = getMetricCategory(key);
if (category === 'aiBased') {
// AI judges grade 1–5; ≥3/5 reads as "acceptable" for the at-a-glance
// badge colour. No other surface renders AI pass/fail, so this
// card-local threshold can't disagree with one.
return { key, label, text: formatMetricRawScore(value, { category }), passed: value >= 3 };
}
// Deterministic checks pass only on a perfect score, via the shared
// `casePassed` — so a case can't read "pass" here yet "fail" on the run
// detail / wizard, which use the same helper.
return {
key,
label,
text: formatMetricPercent(value, { category }),
passed: casePassed(value),
};
});
});
// Operational metrics (tokens, time) — plain gray text, no percentages.
const operationalMetrics = computed<Array<{ key: string; label: string; text: string }>>(() => {
return getOperationalMetricEntries(caseExecution.value?.metrics).map(({ key, value }) => {
const label = OPERATIONAL_LABEL[key] ? locale.baseText(OPERATIONAL_LABEL[key]) : key;
const text =
key === 'executionTime' ? formatDuration(value) : formatTokens(value, { withUnit: false });
return { key, label, text };
});
});
// The node under test whose output is the case's answer.
const endNodeName = computed(() => wizardStore.answerNodeName);
// The compiled config run has no setOutputs node, so `caseExecution.outputs` is
// empty; the real answer is the end node's output in the run. Fetch the full
// execution lazily (the output only renders when expanded) and extract from it.
const caseExecutionData = ref<IExecutionResponse | null>(null);
async function loadCaseExecution(executionId: string) {
try {
caseExecutionData.value = (await executionsStore.fetchExecution(executionId)) ?? null;
} catch (error) {
console.warn('[TestCaseRunResult] failed to load case execution', error);
}
}
// Load the full execution when the output is shown (expanded) or when the case
// failed (so its error message is shown without needing to expand).
watch(
[() => props.expanded, runFailed, () => caseExecution.value?.executionId],
([isExpanded, failed, executionId]) => {
if (!executionId || (!isExpanded && !failed)) return;
if (caseExecutionData.value?.id === executionId) return;
void loadCaseExecution(executionId);
},
{ immediate: true },
);
const outputText = computed(() =>
extractCaseAnswer(caseExecutionData.value, endNodeName.value, caseExecution.value?.outputs),
);
// Why the run failed: the workflow's own error message when available, else the
// mapped error-code message, else a generic fallback.
const failureReason = computed(() => {
if (!runFailed.value) return '';
const executionError = caseExecutionData.value?.data?.resultData?.error?.message;
if (executionError) return executionError;
const errorKey = getErrorBaseKey(caseExecution.value?.errorCode);
if (errorKey && locale.exists(errorKey)) {
return locale.baseText(errorKey, { interpolate: { link: '' } }).trim();
}
return locale.baseText('evaluations.tests.results.runFailed');
});
</script>
<template>
<!-- Running -->
<div
v-if="isRunning"
:class="[$style.runSection, { [$style.separated]: separated }]"
:data-test-id="`tests-result-running-${index}`"
>
<div :class="$style.runLine">
<N8nIcon icon="spinner" size="small" :spin="true" color="text-light" />
<N8nText size="small" color="text-light">
{{ locale.baseText('evaluations.tests.results.running') }}
</N8nText>
</div>
</div>
<!-- Latest run result -->
<div
v-else-if="hasResult"
:class="[$style.runSection, { [$style.separated]: separated }]"
:data-test-id="`tests-result-result-${index}`"
>
<div :class="$style.runLine">
<N8nIcon
:icon="runPassed ? 'check' : 'x'"
size="small"
:color="runPassed ? 'success' : 'danger'"
/>
<N8nText size="small" :color="runPassed ? 'success' : 'danger'">
{{
locale.baseText('evaluations.tests.results.runLabel', {
interpolate: { number: String(runNumber), date: runDateLabel },
})
}}
</N8nText>
</div>
<!-- Failure reason (shown for a failed run, collapsed or expanded) -->
<N8nText
v-if="runFailed && failureReason"
size="small"
color="danger"
:class="$style.failureReason"
:data-test-id="`tests-result-error-${index}`"
>
{{ failureReason }}
</N8nText>
<template v-if="expanded">
<N8nText size="small" color="text-light">
{{ locale.baseText('evaluations.tests.detail.output') }}
</N8nText>
</template>
<div :class="$style.badges">
<span
v-for="badge in metricBadges"
:key="badge.key"
:class="[$style.badge, badge.passed ? $style.badgePass : $style.badgeFail]"
:data-test-id="`tests-result-badge-${index}-${badge.key}`"
>
{{ badge.label }} <strong>{{ badge.text }}</strong>
</span>
</div>
<N8nText v-if="expanded && outputText" size="small" color="text-dark" :class="$style.output">
{{ outputText }}
</N8nText>
<!-- Operational metrics: gray text, only when expanded, no percentages -->
<div
v-if="expanded && operationalMetrics.length > 0"
:class="$style.operational"
:data-test-id="`tests-result-operational-${index}`"
>
<N8nText v-for="m in operationalMetrics" :key="m.key" size="xsmall" color="text-light">
{{ m.label }} {{ m.text }}
</N8nText>
</div>
</div>
</template>
<style module lang="scss">
.runSection {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
}
// Full-bleed top border separates the run result from the title/definition
// above it (used when embedded in a card whose padding is `--spacing--md`).
.separated {
border-top: var(--border);
margin: 0 calc(-1 * var(--spacing--md));
padding: var(--spacing--sm) var(--spacing--md) 0;
}
.runLine {
display: flex;
align-items: center;
gap: var(--spacing--2xs);
}
.badges {
display: flex;
flex-wrap: wrap;
gap: var(--spacing--2xs);
}
.badge {
display: inline-flex;
align-items: center;
gap: var(--spacing--4xs);
padding: var(--spacing--4xs) var(--spacing--2xs);
border-radius: var(--radius--sm);
font-size: var(--font-size--2xs);
}
.badgePass {
background-color: var(--color--success--tint-2, #e9f7ef);
color: var(--color--success--shade-1, #1a8d4a);
}
.badgeFail {
background-color: var(--color--danger--tint-2, #fdecea);
color: var(--color--danger--shade-1, #b3261e);
}
.output {
white-space: pre-wrap;
word-break: break-word;
}
.failureReason {
white-space: pre-wrap;
word-break: break-word;
}
.operational {
display: flex;
flex-wrap: wrap;
gap: var(--spacing--2xs) var(--spacing--sm);
margin-top: var(--spacing--3xs);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/CheckResultCard.vue
-169
View File
@@ -1,169 +0,0 @@
<script setup lang="ts">
import { N8nText } from '@n8n/design-system';
import type { IconName } from '@n8n/design-system';
import CheckHeader from './CheckHeader.vue';
defineProps<{
icon: IconName;
iconBg?: string;
iconFg?: string;
title: string;
description?: string;
badge?: string;
badgeIcon?: IconName;
category?: string;
loading?: boolean;
scoreLabel?: string;
scoreText?: string;
scorePercent?: number;
loadingLabel?: string;
outputLabel?: string;
outputText?: string;
outputMeta?: string;
}>();
</script>
<template>
<div :class="[$style.card, loading ? $style.cardLoading : null]">
<CheckHeader
:icon="icon"
:icon-bg="iconBg"
:icon-fg="iconFg"
:title="title"
:badge="badge"
:badge-icon="badgeIcon"
>
<template #description>
<N8nText
v-if="loading && loadingLabel"
size="xsmall"
color="text-light"
:class="$style.subtitle"
>
{{ loadingLabel }}
</N8nText>
<N8nText v-else-if="description" size="small" color="text-base" :class="$style.subtitle">
{{ description }}
</N8nText>
</template>
</CheckHeader>
<template v-if="loading">
<div :class="$style.progressTrack">
<div :class="[$style.progressFill, $style.progressFillIndeterminate]"></div>
</div>
</template>
<template v-else>
<div v-if="scoreText" :class="$style.scoreRow">
<N8nText size="xsmall" color="text-base">{{ scoreLabel }}</N8nText>
<N8nText size="xsmall" bold color="text-dark">{{ scoreText }}</N8nText>
</div>
<div v-if="scorePercent !== undefined" :class="$style.progressTrack">
<div
:class="[$style.progressFill, category ? $style[`progressFill_${category}`] : null]"
:style="{ width: `${scorePercent}%` }"
></div>
</div>
<div v-if="outputLabel || outputText || outputMeta" :class="$style.outputBlock">
<N8nText v-if="outputLabel" size="xsmall" color="text-base">
{{ outputLabel }}
</N8nText>
<N8nText v-if="outputText" size="small" color="text-dark" :class="$style.outputText">
{{ outputText }}
</N8nText>
<N8nText v-if="outputMeta" size="xsmall" color="text-light">
{{ outputMeta }}
</N8nText>
</div>
</template>
</div>
</template>
<style module lang="scss">
.card {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
padding: var(--spacing--sm);
border: var(--border);
border-radius: var(--radius--xs);
background-color: var(--background--surface);
}
.cardLoading .subtitle {
opacity: 0.85;
}
.subtitle {
display: block;
line-height: 1.4;
}
.scoreRow {
display: flex;
justify-content: space-between;
align-items: center;
}
.progressTrack {
width: 100%;
height: 6px;
border-radius: 3px;
background-color: var(--background--subtle);
overflow: hidden;
}
.progressFill {
height: 100%;
background-color: var(--background--brand);
transition: width var(--duration--snappy) ease;
}
.progressFillIndeterminate {
width: 40%;
background-color: var(--background--brand);
animation: progressFillSlide 1.4s ease-in-out infinite;
}
@keyframes progressFillSlide {
0% {
transform: translateX(-40%);
}
100% {
transform: translateX(250%);
}
}
.progressFill_aiBased {
background-color: var(--color--green--shade-1, #1a8d4a);
}
.progressFill_stringSimilarity {
background-color: var(--color--purple--shade-1, #6b3fc4);
}
.progressFill_categorization {
background-color: var(--color--yellow--shade-1, #c98a04);
}
.progressFill_toolsUsed {
background-color: var(--color--teal--shade-1, #128172);
}
.progressFill_custom {
background-color: var(--background--brand);
}
.outputBlock {
display: flex;
flex-direction: column;
gap: var(--spacing--3xs);
padding-top: var(--spacing--2xs);
border-top: var(--border);
}
.outputText {
display: -webkit-box;
-webkit-line-clamp: 3;
-webkit-box-orient: vertical;
overflow: hidden;
line-height: 1.4;
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/CheckScore.vue
-38
View File
@@ -1,38 +0,0 @@
<script setup lang="ts">
import { computed } from 'vue';
import { N8nIcon, N8nText } from '@n8n/design-system';
import type { TestCaseExecutionRecord } from '../../evaluation.api';
import { casePassed, formatMetricAverage, type ResultCheck } from '../../evaluation.utils';
// Renders a single check's result for a case: AI-judged checks show the
// out-of-5 score, pass/fail checks show a ✓/✗ marker (pass = a perfect score).
const props = defineProps<{
check: ResultCheck;
testCase: TestCaseExecutionRecord;
}>();
const value = computed(() => props.testCase.metrics?.[props.check.key]);
const scoreText = computed(() => formatMetricAverage(value.value, { category: 'aiBased' }));
const passed = computed(() => casePassed(value.value));
</script>
<template>
<N8nText v-if="check.isAiJudged" size="xsmall" bold color="text-dark">{{ scoreText }}</N8nText>
<N8nIcon
v-else
:icon="passed ? 'check' : 'x'"
size="xsmall"
:class="passed ? $style.passIcon : $style.failIcon"
/>
</template>
<style module lang="scss">
.passIcon {
color: var(--color--success);
}
.failIcon {
color: var(--color--danger);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/CustomCheckModal.vue
-138
View File
@@ -1,138 +0,0 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue';
import { storeToRefs } from 'pinia';
import { useI18n } from '@n8n/i18n';
import {
N8nButton,
N8nDialog,
N8nDialogHeader,
N8nDialogTitle,
N8nInput,
N8nText,
} from '@n8n/design-system';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
const wizardStore = useEvaluationsWizardSidepanelStore();
const locale = useI18n();
const { isCustomCheckModalOpen } = storeToRefs(wizardStore);
const name = ref('');
const expression = ref('');
watch(isCustomCheckModalOpen, (open) => {
if (!open) {
name.value = '';
expression.value = '';
}
});
const canSubmit = computed(() => Boolean(name.value.trim()) && Boolean(expression.value.trim()));
function handleSubmit() {
if (!canSubmit.value) return;
wizardStore.addCustomCheck({
name: name.value.trim(),
expression: expression.value,
});
wizardStore.closeCustomCheckModal();
}
function handleOpenChange(open: boolean) {
if (!open) wizardStore.closeCustomCheckModal();
}
</script>
<template>
<N8nDialog :open="isCustomCheckModalOpen" size="xlarge" @update:open="handleOpenChange">
<N8nDialogHeader>
<N8nDialogTitle>
{{ locale.baseText('evaluations.wizardSidepanel.customCheck.title') }}
</N8nDialogTitle>
</N8nDialogHeader>
<div :class="$style.body" data-test-id="evaluations-wizard-sidepanel-custom-check-modal">
<div :class="$style.form">
<div :class="$style.field">
<N8nText size="small" bold color="text-dark" tag="label">
{{ locale.baseText('evaluations.wizardSidepanel.customCheck.name') }}
</N8nText>
<N8nInput
v-model="name"
size="medium"
:placeholder="
locale.baseText('evaluations.wizardSidepanel.customCheck.name.placeholder')
"
data-test-id="custom-check-name-input"
/>
</div>
<div :class="$style.field">
<N8nText size="small" bold color="text-dark" tag="label">
{{ locale.baseText('evaluations.wizardSidepanel.customCheck.expression') }}
</N8nText>
<N8nText size="xsmall" color="text-light">
{{ locale.baseText('evaluations.wizardSidepanel.customCheck.expression.hint') }}
</N8nText>
<N8nInput
v-model="expression"
type="textarea"
:rows="6"
size="medium"
:placeholder="
locale.baseText('evaluations.wizardSidepanel.customCheck.expression.placeholder')
"
:class="$style.expressionInput"
data-test-id="custom-check-expression-input"
/>
</div>
</div>
<div :class="$style.footer">
<N8nButton
variant="solid"
size="medium"
type="button"
:disabled="!canSubmit"
data-test-id="custom-check-submit"
@click="handleSubmit"
>
{{ locale.baseText('evaluations.wizardSidepanel.customCheck.submit') }}
</N8nButton>
</div>
</div>
</N8nDialog>
</template>
<style module lang="scss">
.body {
display: flex;
flex-direction: column;
gap: var(--spacing--md);
padding-top: var(--spacing--sm);
}
.form {
display: flex;
flex-direction: column;
gap: var(--spacing--sm);
}
.field {
display: flex;
flex-direction: column;
gap: var(--spacing--3xs);
}
.expressionInput {
font-family: var(--font-family--monospace, monospace);
}
.footer {
display: flex;
justify-content: flex-end;
padding-top: var(--spacing--xs);
border-top: var(--border);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/EvaluationsWizardSidepanel.test.ts
-1058
View File
File diff suppressed because it is too large Load Diff
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/EvaluationsWizardSidepanel.vue
-963
View File
@@ -1,963 +0,0 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue';
import { storeToRefs } from 'pinia';
import { useRouter } from 'vue-router';
import { useI18n } from '@n8n/i18n';
import { N8nButton, N8nIcon, N8nText } from '@n8n/design-system';
import { useTelemetry } from '@n8n/composables/useTelemetry';
import { injectWorkflowDocumentStore } from '@/app/stores/workflowDocument.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
import { useExecutionsStore } from '@/features/execution/executions/executions.store';
import type { IExecutionResponse } from '@/features/execution/executions/executions.types';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { useEvaluationStore } from '../../evaluation.store';
import {
extractCaseAnswer,
formatMetricAverage,
formatMetricLabel,
formatMetricPercent,
getMetricCategory,
getUserDefinedMetricNames,
type ResultCheck,
} from '../../evaluation.utils';
import type { TestCaseExecutionRecord } from '../../evaluation.api';
import {
CANNED_METRICS,
LLM_JUDGE_METRIC_KEYS,
BUILTIN_PRIMARY_CHECK_KEY,
BUILTIN_MORE_CHECK_KEYS,
getExpectedFieldsForMetrics,
type CannedMetricKey,
} from '../../evaluation.constants';
import { useSliceInputs } from '../../composables/useSliceInputs';
import { useUserExecutions } from '../../composables/useUserExecutions';
import { useAiRootNodes } from '../../composables/useAiRootNodes';
import { useRunEvalWorkflow } from '../../composables/useRunEvalWorkflow';
import { useDefaultJudgeSelection } from '../../composables/useDefaultJudgeSelection';
import { useWizardPersistence } from './useWizardPersistence';
import { useWizardHydration } from './useWizardHydration';
import CheckCard from './CheckCard.vue';
import CheckResultCard from './CheckResultCard.vue';
import ResultsCaseRow from './ResultsCaseRow.vue';
import TestCaseForm from './TestCaseForm.vue';
import SystemSelector from './SystemSelector.vue';
import CustomCheckModal from './CustomCheckModal.vue';
import { VIEWS } from '@/app/constants/navigation';
const wizardStore = useEvaluationsWizardSidepanelStore();
const locale = useI18n();
const router = useRouter();
const telemetry = useTelemetry();
const workflowDocumentStore = injectWorkflowDocumentStore();
const workflowsStore = useWorkflowsStore();
const executionsStore = useExecutionsStore();
const { fetchLatestUserExecution } = useUserExecutions();
const evaluationStore = useEvaluationStore();
const {
activeStep,
selectedMetricKeys,
judgeSelectionByMetric,
aiNodeName,
isSliceMode,
startNodeName,
endNodeName,
inputs,
expectedValues,
datasetExpectedByRow,
customChecks,
} = storeToRefs(wizardStore);
const cannedMetrics = computed(() => CANNED_METRICS);
const aiRootNodes = useAiRootNodes();
const defaultJudgeSelection = useDefaultJudgeSelection();
const { runWorkflow } = useRunEvalWorkflow();
// Checks step: Correctness is always shown; the remaining built-ins are revealed
// behind "Explore more checks". Auto-open if a hidden built-in is already selected
// (e.g. a hydrated config), so a selected check is never invisible.
const showMoreChecks = ref(false);
const primaryCheckMetric = computed(() =>
CANNED_METRICS.find((m) => m.key === BUILTIN_PRIMARY_CHECK_KEY),
);
const moreCheckMetrics = computed(() =>
CANNED_METRICS.filter((m) => BUILTIN_MORE_CHECK_KEYS.includes(m.key)),
);
const visibleCheckMetrics = computed(() => {
// Order-preserving, de-duped by key.
const shown = new Map<CannedMetricKey, (typeof CANNED_METRICS)[number]>();
if (primaryCheckMetric.value) shown.set(primaryCheckMetric.value.key, primaryCheckMetric.value);
if (showMoreChecks.value) {
for (const m of moreCheckMetrics.value) shown.set(m.key, m);
}
// Any selected canned metric must be visible even when it is outside the
// curated primary/more lists — e.g. a `helpfulness` or `stringSimilarity`
// config created by the agent/API or hydrated from a saved eval. Without
// this the selection lives in the store but has no card, so the step looks
// unconfigured.
for (const key of selectedMetricKeys.value) {
if (shown.has(key)) continue;
const metric = CANNED_METRICS.find((m) => m.key === key);
if (metric) shown.set(key, metric);
}
return [...shown.values()];
});
watch(
[selectedMetricKeys, customChecks],
([keys, checks]) => {
if (showMoreChecks.value) return;
const hasHiddenBuiltin = keys.some((k) => BUILTIN_MORE_CHECK_KEYS.includes(k));
if (hasHiddenBuiltin || checks.length > 0) {
showMoreChecks.value = true;
}
},
{ immediate: true },
);
// Watching judgeSelectionByMetric so a hydrate that drops some keys still
// triggers a re-seed of the rest.
watch(
[() => wizardStore.isOpen, defaultJudgeSelection, judgeSelectionByMetric],
([isOpen, defaultSelection, currentSelections]) => {
if (!isOpen || !defaultSelection) return;
for (const key of LLM_JUDGE_METRIC_KEYS) {
if (currentSelections[key]) continue;
wizardStore.setJudgeSelection(key, defaultSelection);
}
},
{ immediate: true },
);
// Gate (ticket "Step 0"): the wizard is blocked until the workflow has had at
// least one successful, non-evaluation execution. `probeComplete` guards against
// flashing the gate before the execution lookup resolves.
// Declared before the probe watcher (which runs immediately): the probe writes
// to `fallbackUserExecution` during setup, so it must exist by then.
const fallbackUserExecution = ref<IExecutionResponse | null>(null);
const sliceInputs = useSliceInputs({ fallbackExecution: fallbackUserExecution });
const probeComplete = ref(false);
async function runExecutionProbe() {
try {
await Promise.all([
Promise.resolve(workflowsStore.fetchLastSuccessfulExecution()),
loadFallbackUserExecution(),
]);
} finally {
probeComplete.value = true;
}
}
watch(
[activeStep, aiRootNodes],
([step, nodes]) => {
if (step !== 0 && step !== 2) return;
void runExecutionProbe();
if (wizardStore.aiNodeName) return;
const first = nodes[0];
if (first) wizardStore.setAiNodeName(first.name);
},
{ immediate: true },
);
// True once the workflow has a usable successful execution to seed inputs from.
const hasRun = computed(() => sliceInputs.value.hasExecution);
// An eval that already has a run (pinned this session, restored on reload, or
// reached via "Edit evals") is past the first-run gate — editing it must never
// be re-blocked, regardless of what executions are currently detectable.
const isExistingEval = computed(() => wizardStore.activeRunId !== null);
// The gate only guards first-time authoring. The results step (3) and any
// existing eval bypass it (each has its own loading/empty handling).
const showGate = computed(
() => probeComplete.value && !hasRun.value && activeStep.value !== 3 && !isExistingEval.value,
);
const showProbeLoading = computed(
() => !hasRun.value && !probeComplete.value && activeStep.value !== 3 && !isExistingEval.value,
);
// Skip evaluation runs — after a few wizard sessions, lastSuccessfulExecution
// would always be the compiled eval workflow, not the user's graph.
async function loadFallbackUserExecution() {
try {
fallbackUserExecution.value = await fetchLatestUserExecution();
} catch {
fallbackUserExecution.value = null;
}
}
const expectedFields = computed(() => getExpectedFieldsForMetrics(selectedMetricKeys.value));
watch(
sliceInputs,
({ values, fieldNames }) => {
if (fieldNames.length === 0) return;
wizardStore.seedInputs(values);
},
{ immediate: true },
);
const { persistAndDispatch, isPersisting } = useWizardPersistence();
const { hydrate } = useWizardHydration();
// Hydrate from the latest persisted EvaluationConfig whenever the wizard is
// opened — the store's `open()` resets state first, so this safely overrides
// the empty defaults without clobbering anything the user has already typed.
// Also reset + re-hydrate on a genuine workflow switch so prior-workflow
// selections do not leak into the new workflow's pane.
// Errors inside `hydrate` are swallowed by the composable (it toasts) so we
// don't need to handle them here.
// Local mirror of n8n-core PLACEHOLDER_EMPTY_WORKFLOW_ID (frontend can't import n8n-core).
const NEW_WORKFLOW_ID = '__EMPTY__';
watch(
() => [wizardStore.isOpen, workflowDocumentStore.value?.workflowId] as const,
([isOpen]) => {
const id = workflowDocumentStore.value?.workflowId;
// Compare against the store-persisted last workflow id rather than the
// watcher's previous value: the pane unmounts when the focus panel closes
// between workflows, so on remount the watcher has no previous value and
// would otherwise miss the switch (leaving a prior run's results visible).
const prevId = wizardStore.lastWorkflowId;
// Reset only on a genuine switch between saved workflows. Skip the
// new→saved id transition (prevId placeholder/empty) so a user's
// in-progress selections on a brand-new workflow aren't wiped on save.
if (prevId && prevId !== NEW_WORKFLOW_ID && id && id !== prevId) {
wizardStore.reset();
}
if (id) wizardStore.setLastWorkflowId(id);
if (isOpen) void hydrate();
},
{ immediate: true },
);
const runs = computed(() => {
const id = workflowDocumentStore.value.workflowId;
return Object.values(evaluationStore.testRunsById ?? {}).filter(
({ workflowId }) => workflowId === id,
);
});
// Pin to the run dispatched by THIS session via activeRunId; "newest in map"
// would flash an older run between step transition and the new run arriving.
const latestRun = computed(() => {
const pinnedId = wizardStore.activeRunId;
if (pinnedId) return evaluationStore.testRunsById?.[pinnedId];
return runs.value[runs.value.length - 1];
});
// All cases for the active run, ordered by their position in the dataset.
const latestRunCases = computed<TestCaseExecutionRecord[]>(() => {
const runId = latestRun.value?.id;
if (!runId) return [];
return Object.values(evaluationStore.testCaseExecutionsById ?? {})
.filter((c) => c.testRunId === runId)
.sort((a, b) => (a.runIndex ?? 0) - (b.runIndex ?? 0));
});
// The checks rendered on the results page, derived from the run's metric keys so
// canned and custom checks are handled uniformly. AI-judged checks show an
// average %; the rest are pass/fail.
const resultChecks = computed<ResultCheck[]>(() => {
const metrics = latestRun.value?.metrics;
if (!metrics) return [];
return getUserDefinedMetricNames(metrics).map((key) => {
const canned = CANNED_METRICS.find((m) => m.key === key);
return {
key,
label: canned ? locale.baseText(canned.labelKey) : formatMetricLabel(key),
description: canned ? locale.baseText(canned.descriptionKey) : undefined,
isAiJudged: getMetricCategory(key) === 'aiBased',
// Mirror the Step-2 check tile; custom checks fall back to the code icon.
icon: canned?.icon ?? 'code',
iconBg: canned?.tileBg,
iconFg: canned?.tileFg,
};
});
});
const sliceEndNodeName = computed(() => wizardStore.answerNodeName);
// The expected-output values for a case, taken from the dataset row at the
// case's `runIndex` (rows are seeded one-per-row in order). Falls back to the
// Step-2 first-row values when the per-row data isn't hydrated.
function caseExpectedValues(testCase: TestCaseExecutionRecord): Record<string, string> {
const index = testCase.runIndex ?? 0;
return datasetExpectedByRow.value[index] ?? expectedValues.value;
}
const executionsByCaseId = ref<Record<string, IExecutionResponse | null>>({});
// The AI's answer for a case: the end node's output during the test run,
// stripped of its JSON envelope (output > text > response > …). Falls back to
// the case's persisted `outputs` only when the execution isn't loaded.
function caseAnswer(testCase: TestCaseExecutionRecord): string {
return extractCaseAnswer(
executionsByCaseId.value[testCase.id],
sliceEndNodeName.value,
testCase.outputs,
);
}
async function loadExecutionForCase(caseId: string, executionId: string) {
if (caseId in executionsByCaseId.value) return;
executionsByCaseId.value = { ...executionsByCaseId.value, [caseId]: null };
try {
const execution = await executionsStore.fetchExecution(executionId);
executionsByCaseId.value = {
...executionsByCaseId.value,
[caseId]: execution ?? null,
};
} catch (error) {
console.warn('[evaluations wizard] failed to load case execution', error);
}
}
// Prefetch each case's full execution so caseAnswer can fall back to the
// end-node output when a case has no persisted `outputs`.
watch(
latestRunCases,
(cases) => {
for (const c of cases) {
if (c.executionId) void loadExecutionForCase(c.id, c.executionId);
}
},
{ immediate: true },
);
// On reload/hot-nav into step 4 (results), the store's run poller never ran — fetch
// case executions here so the output block isn't stuck empty.
watch(
[activeStep, latestRun],
([step, run]) => {
if (step !== 3 || !run) return;
if (['new', 'running'].includes(run.status)) return;
const workflowId = workflowDocumentStore.value?.workflowId;
if (!workflowId) return;
void evaluationStore.fetchTestCaseExecutions({ workflowId, runId: run.id });
},
{ immediate: true },
);
const isRunning = computed(() =>
latestRun.value ? ['new', 'running'].includes(latestRun.value.status) : false,
);
const showLoadingState = computed(
() => isRunning.value || (activeStep.value === 3 && !latestRun.value),
);
// Fire once per run when the results step settles on a finished run, so the
// "view detailed results" navigation isn't the only signal that a user saw
// their scores inline. Deduped by run id so the watcher re-firing (poll
// updates, case prefetch) doesn't emit repeats.
const trackedResultsRunId = ref<string | null>(null);
watch(
[activeStep, latestRun, showLoadingState],
([step, run, loading]) => {
if (step !== 3 || loading || !run || trackedResultsRunId.value === run.id) return;
trackedResultsRunId.value = run.id;
telemetry.track('User viewed evaluation results', {
workflow_id: workflowDocumentStore.value?.workflowId,
run_id: run.id,
test_case_count: latestRunCases.value.length,
metric_count: getUserDefinedMetricNames(run.metrics).length,
});
},
{ immediate: true },
);
// Per-step metadata, indexed by activeStep (0-3); WizardStep is constrained to those values.
const STEP_META = [
{
telemetryName: 'choose_system',
title: 'evaluations.wizardSidepanel.step.chooseSystem.title',
description: 'evaluations.wizardSidepanel.step.chooseSystem.description',
},
{
telemetryName: 'setup_scorers',
title: 'evaluations.wizardSidepanel.step.setupChecks.title',
description: 'evaluations.wizardSidepanel.step.setupChecks.description',
},
{
telemetryName: 'add_test_cases',
title: 'evaluations.wizardSidepanel.step.addTestCases.title',
description: 'evaluations.wizardSidepanel.step.addTestCases.description',
},
{
telemetryName: 'results',
title: 'evaluations.wizardSidepanel.step.results.title',
description: 'evaluations.wizardSidepanel.step.results.description',
},
] as const;
// Emit a view event when a step's content is shown (not while gated/loading).
// Deduped per step, reset on close, so revisiting a step doesn't re-fire.
const trackedSteps = new Set<number>();
watch(
[() => wizardStore.isOpen, activeStep, showGate, showProbeLoading],
([isOpen, step, gated, loading]) => {
if (!isOpen) {
trackedSteps.clear();
return;
}
if (gated || loading || trackedSteps.has(step)) return;
trackedSteps.add(step);
telemetry.track('User viewed evaluation config wizard step', {
workflow_id: workflowDocumentStore.value?.workflowId,
step_name: STEP_META[step].telemetryName,
step_index: step + 1,
});
},
{ immediate: true },
);
// step0Complete: system chosen (node or slice)
const step0Complete = computed(() => {
if (isSliceMode.value) {
return Boolean(startNodeName.value && endNodeName.value);
}
return Boolean(aiNodeName.value);
});
// step1Complete: at least one check selected, with judge selection for any LLM-judge metric
const step1Complete = computed(() => {
if (selectedMetricKeys.value.length === 0 && customChecks.value.length === 0) return false;
for (const key of selectedMetricKeys.value) {
if (LLM_JUDGE_METRIC_KEYS.has(key) && !judgeSelectionByMetric.value[key]) return false;
}
return true;
});
// step2Complete: has execution + all inputs + all expected values filled
const step2Complete = computed(() => {
if (!sliceInputs.value.hasExecution) return false;
const inputsFilled = sliceInputs.value.fieldNames.every(
(name) => (inputs.value[name] ?? '').length > 0,
);
const expectedFilled = expectedFields.value.every(
(f) => (expectedValues.value[f.name] ?? '').length > 0,
);
return inputsFilled && expectedFilled;
});
// LLM-judge metrics carry an "AI-judged" badge. The judge model itself is
// auto-selected from the workflow's own chat-model sub-node (see
// useDefaultJudgeSelection) — there is no manual picker in the wizard.
function isLlmJudgeMetric(key: CannedMetricKey): boolean {
return LLM_JUDGE_METRIC_KEYS.has(key);
}
const titleKey = computed(() => STEP_META[activeStep.value].title);
const descriptionKey = computed(() => STEP_META[activeStep.value].description);
async function handleNext() {
const current = activeStep.value;
if (current === 0) {
wizardStore.goNext();
return;
}
if (current === 1) {
wizardStore.goNext();
return;
}
if (current === 2) {
const ok = await persistAndDispatch('initial');
if (!ok) return;
wizardStore.goNext();
}
}
function handleCancel() {
wizardStore.close();
}
function handleBack() {
wizardStore.goBack();
}
async function handleRunAgain() {
await persistAndDispatch('run_again');
}
function handleViewResults() {
const runId = wizardStore.activeRunId ?? latestRun.value?.id;
const workflowId = workflowDocumentStore.value?.workflowId;
if (runId && workflowId) {
void router.push({
name: VIEWS.EVALUATION_RUNS_DETAIL,
params: { workflowId, runId },
});
}
wizardStore.close();
}
</script>
<template>
<div :class="$style.sidepanel" data-test-id="evaluations-wizard-sidepanel">
<div
v-if="showProbeLoading"
:class="$style.gate"
data-test-id="evaluations-wizard-sidepanel-probe-loading"
>
<N8nIcon icon="spinner" size="medium" :spin="true" />
</div>
<div v-else-if="showGate" :class="$style.gate" data-test-id="evaluations-wizard-sidepanel-gate">
<N8nIcon icon="info" size="large" :class="$style.gateIcon" />
<N8nText size="small" color="text-base" :class="$style.gateMessage">
{{ locale.baseText('evaluations.wizardSidepanel.gate.message') }}
</N8nText>
<N8nButton
size="small"
type="button"
data-test-id="evaluations-wizard-sidepanel-gate-run"
@click="runWorkflow"
>
{{ locale.baseText('evaluations.wizardSidepanel.step.addTestCases.runButton') }}
</N8nButton>
</div>
<template v-else>
<div :class="$style.progressBar" data-test-id="evaluations-wizard-sidepanel-progress">
<div
v-for="step in 4"
:key="step"
:class="[
$style.progressSegment,
activeStep >= step - 1 ? $style.progressSegmentActive : null,
]"
></div>
</div>
<header :class="$style.header">
<N8nText tag="h2" size="large" color="text-dark" bold :class="$style.title">
{{ locale.baseText(titleKey) }}
</N8nText>
<N8nText size="small" color="text-base" :class="$style.description">
{{ locale.baseText(descriptionKey) }}
</N8nText>
</header>
<div :class="$style.body">
<section v-if="activeStep === 0" :class="$style.section">
<SystemSelector />
</section>
<section v-if="activeStep === 1" :class="$style.section">
<ul :class="$style.checkList">
<li v-for="metric in visibleCheckMetrics" :key="metric.key">
<CheckCard
:icon="metric.icon"
:icon-bg="metric.tileBg"
:icon-fg="metric.tileFg"
:title="locale.baseText(metric.labelKey)"
:description="locale.baseText(metric.descriptionKey)"
:badge="
isLlmJudgeMetric(metric.key)
? locale.baseText('evaluations.wizardSidepanel.metric.judgeTag')
: undefined
"
:badge-icon="isLlmJudgeMetric(metric.key) ? 'wand-sparkles' : undefined"
:selected="selectedMetricKeys.includes(metric.key)"
:data-test-id="`evaluations-wizard-sidepanel-metric-${metric.key}`"
@toggle="wizardStore.toggleMetric(metric.key)"
/>
</li>
<li
v-if="!showMoreChecks"
:class="[$style.addCard]"
role="button"
tabindex="0"
data-test-id="evaluations-wizard-sidepanel-explore-more-checks"
@click="showMoreChecks = true"
@keydown.enter.prevent="showMoreChecks = true"
@keydown.space.prevent="showMoreChecks = true"
>
<span :class="$style.addCardIcon">
<N8nIcon icon="plus" size="small" />
</span>
<N8nText size="small" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step1.exploreMoreChecks') }}
</N8nText>
</li>
<!-- Custom checks (and the affordance to add one) appear only after
the user expands "Explore more checks". -->
<template v-if="showMoreChecks">
<li v-for="check in customChecks" :key="check.id">
<CheckCard
icon="code"
:title="check.name"
:badge="locale.baseText('evaluations.wizardSidepanel.customCheck.expressionTag')"
:selected="true"
:removable="true"
:remove-aria-label="
locale.baseText('evaluations.wizardSidepanel.customCheck.remove')
"
:data-test-id="`evaluations-wizard-sidepanel-custom-check-${check.id}`"
:remove-test-id="`evaluations-wizard-sidepanel-custom-check-remove-${check.id}`"
@remove="wizardStore.removeCustomCheck(check.id)"
/>
</li>
<li
:class="[$style.addCard]"
role="button"
tabindex="0"
data-test-id="evaluations-wizard-sidepanel-new-custom-check"
@click="wizardStore.openCustomCheckModal()"
@keydown.enter.prevent="wizardStore.openCustomCheckModal()"
@keydown.space.prevent="wizardStore.openCustomCheckModal()"
>
<span :class="$style.addCardIcon">
<N8nIcon icon="plus" size="small" />
</span>
<N8nText size="small" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step1.newCustomCheck') }}
</N8nText>
</li>
</template>
</ul>
</section>
<section v-if="activeStep === 2" :class="$style.section">
<TestCaseForm :slice-inputs="sliceInputs" />
</section>
<section v-if="activeStep === 3" :class="$style.section">
<div
v-if="showLoadingState"
:class="$style.runningState"
data-test-id="evaluations-wizard-sidepanel-running"
>
<N8nIcon icon="spinner" size="medium" :spin="true" />
<N8nText size="small" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step3.running') }}
</N8nText>
</div>
<ul
v-if="showLoadingState"
:class="$style.checkList"
data-test-id="evaluations-wizard-sidepanel-loading-skeletons"
>
<li
v-for="metric in cannedMetrics.filter((m) => selectedMetricKeys.includes(m.key))"
:key="`skeleton-${metric.key}`"
>
<CheckResultCard
:icon="metric.icon"
:icon-bg="metric.tileBg"
:icon-fg="metric.tileFg"
:title="locale.baseText(metric.labelKey)"
:badge="
isLlmJudgeMetric(metric.key)
? locale.baseText('evaluations.wizardSidepanel.metric.judgeTag')
: undefined
"
:badge-icon="isLlmJudgeMetric(metric.key) ? 'wand-sparkles' : undefined"
:category="metric.category"
:loading="true"
:loading-label="locale.baseText('evaluations.wizardSidepanel.step3.scoring')"
/>
</li>
</ul>
<div
v-if="latestRun && !showLoadingState"
:class="$style.results"
data-test-id="evaluations-wizard-sidepanel-results"
>
<ul :class="$style.checkList">
<li v-for="check in resultChecks" :key="check.key">
<CheckResultCard
:icon="check.icon"
:icon-bg="check.iconBg"
:icon-fg="check.iconFg"
:title="check.label"
:description="check.description"
:badge="
check.isAiJudged
? locale.baseText('evaluations.wizardSidepanel.metric.judgeTag')
: undefined
"
:badge-icon="check.isAiJudged ? 'wand-sparkles' : undefined"
:score-label="
check.isAiJudged
? locale.baseText('evaluations.wizardSidepanel.step3.averageLabel')
: locale.baseText('evaluations.wizardSidepanel.step3.passedLabel')
"
:score-text="
check.isAiJudged
? formatMetricAverage(latestRun.metrics?.[check.key], { category: 'aiBased' })
: formatMetricPercent(latestRun.metrics?.[check.key])
"
:data-test-id="`evaluations-wizard-sidepanel-result-card-${check.key}`"
/>
</li>
</ul>
<ResultsCaseRow
v-for="(testCase, index) in latestRunCases"
:key="testCase.id"
:case-index="index + 1"
:test-case="testCase"
:checks="resultChecks"
:expected-fields="expectedFields"
:expected-values="caseExpectedValues(testCase)"
:ai-answer="caseAnswer(testCase)"
:run-metrics="latestRun.metrics"
/>
</div>
<div
v-if="!latestRun && !showLoadingState"
:class="$style.emptyResult"
data-test-id="evaluations-wizard-sidepanel-no-run"
>
<N8nText size="small" color="text-light">
{{ locale.baseText('evaluations.wizardSidepanel.step3.noRun') }}
</N8nText>
</div>
</section>
</div>
<footer :class="$style.footer">
<N8nButton
v-if="activeStep === 0"
variant="ghost"
size="small"
type="button"
data-test-id="evaluations-wizard-sidepanel-cancel"
@click.stop="handleCancel"
>
{{ locale.baseText('evaluations.wizardSidepanel.cancel') }}
</N8nButton>
<N8nButton
v-else-if="activeStep === 1 || activeStep === 2"
variant="ghost"
size="small"
type="button"
data-test-id="evaluations-wizard-sidepanel-back"
@click.stop="handleBack"
>
{{ locale.baseText('evaluations.wizardSidepanel.nav.back') }}
</N8nButton>
<N8nButton
v-else-if="activeStep === 3"
variant="ghost"
size="small"
type="button"
data-test-id="evaluations-wizard-sidepanel-edit-evals"
@click.stop="wizardStore.setStep(0)"
>
{{ locale.baseText('evaluations.wizardSidepanel.nav.editEvals') }}
</N8nButton>
<span :class="$style.footerSpacer" />
<N8nButton
v-if="activeStep < 3"
variant="outline"
size="small"
type="button"
:loading="isPersisting"
:disabled="
(activeStep === 0 && !step0Complete) ||
(activeStep === 1 && !step1Complete) ||
(activeStep === 2 && !step2Complete)
"
data-test-id="evaluations-wizard-sidepanel-next"
@click.stop="handleNext"
>
<span v-if="activeStep === 0">
{{ locale.baseText('evaluations.wizardSidepanel.nav.next.checks') }}
</span>
<span v-else-if="activeStep === 1">
{{ locale.baseText('evaluations.wizardSidepanel.nav.next.cases') }}
</span>
<span v-else>
{{ locale.baseText('evaluations.wizardSidepanel.nav.next.run') }}
</span>
</N8nButton>
<template v-else>
<N8nButton
variant="ghost"
size="small"
type="button"
:loading="isPersisting"
data-test-id="evaluations-wizard-sidepanel-run-again"
@click.stop="handleRunAgain"
>
{{ locale.baseText('evaluations.wizardSidepanel.nav.runAgain') }}
</N8nButton>
<N8nButton
variant="outline"
size="small"
type="button"
data-test-id="evaluations-wizard-sidepanel-view-results"
@click.stop="handleViewResults"
>
{{ locale.baseText('evaluations.wizardSidepanel.nav.viewDetailedResults') }}
</N8nButton>
</template>
</footer>
</template>
<CustomCheckModal />
</div>
</template>
<style module lang="scss">
.sidepanel {
display: flex;
flex-direction: column;
width: 100%;
height: 100%;
background-color: var(--background--surface);
overflow: hidden;
}
.gate {
flex: 1 1 auto;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: var(--spacing--sm);
padding: var(--spacing--xl) var(--spacing--md);
text-align: center;
}
.gateIcon {
color: var(--color--secondary);
}
.gateMessage {
max-width: 300px;
line-height: 1.4;
}
.progressBar {
display: grid;
grid-template-columns: repeat(4, 1fr);
gap: 4px;
padding: var(--spacing--xs) var(--spacing--md) 0;
}
.progressSegment {
height: 3px;
border-radius: 2px;
background-color: var(--background--subtle);
transition: background-color var(--duration--snappy) ease;
}
.progressSegmentActive {
background-color: var(--background--brand);
}
.header {
display: flex;
flex-direction: column;
gap: var(--spacing--3xs);
padding: var(--spacing--md);
padding-bottom: var(--spacing--xs);
}
.title {
margin: 0;
font-size: var(--font-size--md, 18px);
}
.description {
display: block;
max-width: 340px;
}
.body {
flex: 1 1 auto;
overflow-y: auto;
padding: var(--spacing--xs) var(--spacing--md) var(--spacing--md);
}
.section {
display: flex;
flex-direction: column;
gap: var(--spacing--sm);
}
.checkList {
list-style: none;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
}
.results {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
}
.addCard {
display: grid;
grid-template-columns: auto 1fr;
align-items: center;
gap: var(--spacing--xs);
padding: var(--spacing--sm);
border: var(--border);
border-radius: var(--radius--xs);
background-color: var(--background--surface);
color: var(--color--text--tint-1);
cursor: pointer;
outline: none;
transition: border-color var(--duration--snappy) ease;
&:hover,
&:focus-visible {
border-color: var(--border-color--strong);
}
}
.addCardIcon {
display: inline-flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
border-radius: 6px;
background-color: var(--background--subtle);
color: var(--color--text--tint-1);
}
.runningState {
display: flex;
align-items: center;
gap: var(--spacing--xs);
padding: var(--spacing--md);
border: var(--border);
border-radius: var(--border-radius--base);
background-color: var(--background--surface);
}
.emptyResult {
padding: var(--spacing--md);
border: 1px dashed var(--border-color--strong);
border-radius: var(--border-radius--base);
text-align: center;
}
.footer {
display: flex;
align-items: center;
gap: var(--spacing--xs);
padding: var(--spacing--sm) var(--spacing--md);
border-top: var(--border);
background-color: var(--background--surface);
}
.footerSpacer {
flex: 1 1 auto;
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/ResultsCaseRow.test.ts
-94
View File
@@ -1,94 +0,0 @@
import { describe, it, expect } from 'vitest';
import userEvent from '@testing-library/user-event';
import { createComponentRenderer } from '@/__tests__/render';
import ResultsCaseRow from './ResultsCaseRow.vue';
import type { TestCaseExecutionRecord } from '../../evaluation.api';
import type { ResultCheck } from '../../evaluation.utils';
import type { ExpectedField } from '../../evaluation.constants';
const renderComponent = createComponentRenderer(ResultsCaseRow);
const checks: ResultCheck[] = [
{ key: 'correctness', label: 'Correctness', isAiJudged: true, icon: 'badge-check' },
{ key: 'categorization', label: 'Exact match', isAiJudged: false, icon: 'tags' },
];
const expectedFields: ExpectedField[] = [
{ name: 'expectedAnswer', labelKey: 'evaluations.wizardSidepanel.step2.expectedAnswer' },
];
function makeCase(metrics: Record<string, number>): TestCaseExecutionRecord {
return {
id: 'case-1',
executionId: null,
status: 'success',
createdAt: '',
updatedAt: '',
runAt: null,
metrics,
inputs: { expectedAnswer: 'Hello there' },
outputs: { output: 'Hi!' },
};
}
const baseProps = () => ({
caseIndex: 1,
testCase: makeCase({ correctness: 1, categorization: 1, totalTokens: 2509, executionTime: 6300 }),
checks,
expectedFields,
expectedValues: {},
aiAnswer: 'Hi!',
runMetrics: {},
});
describe('ResultsCaseRow', () => {
it('renders the collapsed row with case label and per-check scores', () => {
const { getByText, getByTestId, queryByTestId } = renderComponent({ props: baseProps() });
expect(getByText('Case #1')).toBeInTheDocument();
// AI-judged checks show the out-of-5 score.
expect(getByText('1 / 5')).toBeInTheDocument();
expect(getByTestId('evaluations-wizard-sidepanel-case-1')).toBeInTheDocument();
// Detail is collapsed by default.
expect(queryByTestId('evaluations-wizard-sidepanel-case-detail-1')).toBeNull();
});
it('shows a pass tick for a perfect pass/fail score', () => {
const { container } = renderComponent({ props: baseProps() });
expect(container.querySelector('[data-icon="check"]')).not.toBeNull();
expect(container.querySelector('[data-icon="x"]')).toBeNull();
});
it('shows a fail cross when a pass/fail check is not a perfect score', () => {
const props = { ...baseProps(), testCase: makeCase({ correctness: 1, categorization: 0 }) };
const { container } = renderComponent({ props });
expect(container.querySelector('[data-icon="x"]')).not.toBeNull();
});
it('expands on caret click to reveal expected output, the AI answer and metrics', async () => {
const { getByTestId, getByText, queryByText } = renderComponent({ props: baseProps() });
// Expected/answer values are not in the DOM until expanded.
expect(queryByText('Hello there')).toBeNull();
await userEvent.click(getByTestId('evaluations-wizard-sidepanel-case-toggle-1'));
expect(getByTestId('evaluations-wizard-sidepanel-case-detail-1')).toBeInTheDocument();
expect(getByText('Hello there')).toBeInTheDocument(); // expected output
expect(getByText('Hi!')).toBeInTheDocument(); // AI's answer
expect(getByText(/Tokens: 2,509\b/)).toBeInTheDocument(); // operational metrics, no "t" unit
});
it('falls back to the Step-2 expected value when the case has no dataset value', async () => {
const props = {
...baseProps(),
testCase: { ...makeCase({ correctness: 5, categorization: 1 }), inputs: {} },
expectedValues: { expectedAnswer: 'From step 2' },
};
const { getByTestId, getByText } = renderComponent({ props });
await userEvent.click(getByTestId('evaluations-wizard-sidepanel-case-toggle-1'));
expect(getByText('From step 2')).toBeInTheDocument();
});
});
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/ResultsCaseRow.vue
-235
View File
@@ -1,235 +0,0 @@
<script setup lang="ts">
import { computed, ref } from 'vue';
import { useI18n, type BaseTextKey } from '@n8n/i18n';
import { N8nIcon, N8nText } from '@n8n/design-system';
import type { TestCaseExecutionRecord } from '../../evaluation.api';
import type { ExpectedField } from '../../evaluation.constants';
import {
formatDuration,
formatTokens,
stringifyValue,
type ResultCheck,
} from '../../evaluation.utils';
import CheckScore from './CheckScore.vue';
const props = defineProps<{
caseIndex: number;
testCase: TestCaseExecutionRecord;
checks: ResultCheck[];
expectedFields: ExpectedField[];
// The expected-output values from the dataset's `expectedAnswer`/`expectedTools`
// columns, keyed by field name. Sourced from the wizard store, which holds the
// Step-2 entry and is rehydrated from the data table on open. (The test-case
// record's `inputs` comes from the Set Inputs node and never carries these.)
expectedValues: Record<string, string>;
aiAnswer: string;
runMetrics?: Record<string, number> | null;
}>();
const locale = useI18n();
const expanded = ref(false);
const caseLabel = computed(() =>
locale.baseText('evaluations.wizardSidepanel.step3.caseLabel', {
interpolate: { index: props.caseIndex },
}),
);
const expectedEntries = computed(() =>
props.expectedFields.map((field) => ({
name: field.name,
label: locale.baseText(field.labelKey as BaseTextKey),
// The dataset's expected value (via the store); the case's own `inputs` is a
// last-resort fallback should a Set Inputs node ever echo the column.
value:
props.expectedValues[field.name] || stringifyValue(props.testCase.inputs?.[field.name]) || '',
})),
);
const operationalMeta = computed(() => {
const tokens = props.testCase.metrics?.totalTokens ?? props.runMetrics?.totalTokens;
const time = props.testCase.metrics?.executionTime ?? props.runMetrics?.executionTime;
return locale.baseText('evaluations.wizardSidepanel.step3.outputMeta', {
interpolate: { tokens: formatTokens(tokens, { withUnit: false }), time: formatDuration(time) },
});
});
</script>
<template>
<div :class="$style.row" :data-test-id="`evaluations-wizard-sidepanel-case-${caseIndex}`">
<button
type="button"
:class="$style.head"
:aria-expanded="expanded"
:data-test-id="`evaluations-wizard-sidepanel-case-toggle-${caseIndex}`"
@click="expanded = !expanded"
>
<N8nText size="small" bold color="text-dark" :class="$style.caseLabel">
{{ caseLabel }}
</N8nText>
<span :class="$style.scores">
<span v-for="check in checks" :key="check.key" :class="$style.score">
<N8nText size="xsmall" color="text-base">{{ check.label }}</N8nText>
<CheckScore :check="check" :test-case="testCase" />
</span>
</span>
<N8nIcon
icon="chevron-down"
size="small"
:class="[$style.caret, expanded ? $style.caretOpen : null]"
/>
</button>
<div
v-if="expanded"
:class="$style.detail"
:data-test-id="`evaluations-wizard-sidepanel-case-detail-${caseIndex}`"
>
<div v-for="field in expectedEntries" :key="`expected-${field.name}`" :class="$style.field">
<N8nText size="xsmall" color="text-base">{{ field.label }}</N8nText>
<N8nText size="small" color="text-dark" :class="$style.fieldValue">
{{ field.value || '–' }}
</N8nText>
</div>
<div :class="$style.field">
<N8nText size="xsmall" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step3.aiAnswer') }}
</N8nText>
<N8nText size="small" color="text-dark" :class="$style.fieldValue">
{{ aiAnswer || '–' }}
</N8nText>
</div>
<div :class="$style.checkScores">
<div v-for="check in checks" :key="`score-${check.key}`" :class="$style.checkScoreRow">
<span :class="$style.checkLabel">
<span
:class="$style.checkIcon"
:style="
check.iconBg || check.iconFg
? { backgroundColor: check.iconBg, color: check.iconFg }
: undefined
"
>
<N8nIcon :icon="check.icon" size="xsmall" />
</span>
<N8nText size="small" color="text-dark">{{ check.label }}</N8nText>
</span>
<CheckScore :check="check" :test-case="testCase" />
</div>
</div>
<N8nText size="xsmall" color="text-light">{{ operationalMeta }}</N8nText>
</div>
</div>
</template>
<style module lang="scss">
.row {
border: var(--border);
border-radius: var(--radius--xs);
background-color: var(--background--surface);
overflow: hidden;
}
.head {
display: flex;
align-items: center;
gap: var(--spacing--xs);
width: 100%;
padding: var(--spacing--sm);
background: transparent;
border: none;
cursor: pointer;
text-align: left;
&:hover,
&:focus-visible {
background-color: var(--background--subtle);
}
}
.caseLabel {
flex-shrink: 0;
}
.scores {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: var(--spacing--3xs) var(--spacing--xs);
flex: 1 1 auto;
min-width: 0;
}
.score {
display: inline-flex;
align-items: center;
gap: var(--spacing--3xs);
}
.caret {
flex-shrink: 0;
color: var(--color--text--tint-1);
transition: transform var(--duration--snappy) ease;
}
.caretOpen {
transform: rotate(180deg);
}
.detail {
display: flex;
flex-direction: column;
gap: var(--spacing--sm);
padding: var(--spacing--sm);
border-top: var(--border);
}
.field {
display: flex;
flex-direction: column;
gap: var(--spacing--3xs);
}
.fieldValue {
white-space: pre-wrap;
word-break: break-word;
}
.checkScores {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
padding-top: var(--spacing--2xs);
border-top: var(--border);
}
.checkScoreRow {
display: flex;
align-items: center;
justify-content: space-between;
gap: var(--spacing--xs);
}
.checkLabel {
display: inline-flex;
align-items: center;
gap: var(--spacing--2xs);
min-width: 0;
}
.checkIcon {
display: inline-flex;
align-items: center;
justify-content: center;
flex-shrink: 0;
width: 20px;
height: 20px;
border-radius: var(--radius--2xs);
background-color: var(--background--subtle);
color: var(--color--text--tint-1);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/SystemSelector.vue
-168
View File
@@ -1,168 +0,0 @@
<script setup lang="ts">
import { computed } from 'vue';
import { storeToRefs } from 'pinia';
import { useI18n } from '@n8n/i18n';
import { N8nOption, N8nSelect, N8nText } from '@n8n/design-system';
import { isSubNodeType } from 'n8n-workflow';
import { injectWorkflowDocumentStore } from '@/app/stores/workflowDocument.store';
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { useAiRootNodes } from '../../composables/useAiRootNodes';
const wizardStore = useEvaluationsWizardSidepanelStore();
const locale = useI18n();
const workflowDocumentStore = injectWorkflowDocumentStore();
const nodeTypesStore = useNodeTypesStore();
const { aiNodeName, isSliceMode, startNodeName, endNodeName } = storeToRefs(wizardStore);
const aiRootNodes = useAiRootNodes();
// AI sub-nodes live off ai_* connections, never on the main chain we evaluate.
const nodeNameOptions = computed(() =>
workflowDocumentStore.value.allNodes
.filter((node) => !isSubNodeType(nodeTypesStore.getNodeType(node.type) ?? null))
.map((node) => ({ name: node.name })),
);
</script>
<template>
<div :class="$style.formBlock">
<div
v-if="!isSliceMode"
:class="$style.sliceFields"
data-test-id="evaluations-wizard-sidepanel-ai-node-picker"
>
<div :class="$style.field">
<N8nText size="xsmall" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step2.aiNode') }}
</N8nText>
<N8nSelect
v-model="aiNodeName"
size="small"
filterable
:placeholder="locale.baseText('evaluations.wizardSidepanel.step2.aiNode.placeholder')"
data-test-id="evaluations-wizard-sidepanel-ai-node-select"
>
<N8nOption
v-for="node in aiRootNodes"
:key="node.name"
:label="node.name"
:value="node.name"
/>
</N8nSelect>
<button
type="button"
:class="$style.sliceModeLink"
data-test-id="evaluations-wizard-sidepanel-extend-to-slice"
@click="wizardStore.enterSliceMode()"
>
{{ locale.baseText('evaluations.wizardSidepanel.step2.extendToSlice') }}
</button>
</div>
</div>
<div
v-else
:class="$style.sliceFields"
data-test-id="evaluations-wizard-sidepanel-slice-picker"
>
<div :class="$style.field">
<N8nText size="xsmall" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step2.start') }}
</N8nText>
<N8nSelect
v-model="startNodeName"
size="small"
filterable
:placeholder="locale.baseText('evaluations.wizardSidepanel.step2.start.placeholder')"
data-test-id="evaluations-wizard-sidepanel-start-select"
>
<N8nOption
v-for="node in nodeNameOptions"
:key="node.name"
:label="node.name"
:value="node.name"
/>
</N8nSelect>
</div>
<div :class="$style.field">
<N8nText size="xsmall" color="text-base">
{{ locale.baseText('evaluations.wizardSidepanel.step2.end') }}
</N8nText>
<N8nSelect
v-model="endNodeName"
size="small"
filterable
:placeholder="locale.baseText('evaluations.wizardSidepanel.step2.end.placeholder')"
data-test-id="evaluations-wizard-sidepanel-end-select"
>
<N8nOption
v-for="node in nodeNameOptions"
:key="node.name"
:label="node.name"
:value="node.name"
/>
</N8nSelect>
<button
type="button"
:class="$style.sliceModeLink"
data-test-id="evaluations-wizard-sidepanel-reset-to-ai-node"
@click="wizardStore.exitSliceMode()"
>
{{ locale.baseText('evaluations.wizardSidepanel.step2.resetToAiNode') }}
</button>
</div>
</div>
</div>
</template>
<style module lang="scss">
.formBlock {
display: flex;
flex-direction: column;
border: var(--border);
border-radius: var(--radius--xs);
background-color: var(--background--surface);
overflow: hidden;
}
.sliceModeLink {
align-self: flex-start;
background: transparent;
border: none;
padding: 0;
margin-top: var(--spacing--3xs);
color: var(--color--primary, var(--background--brand));
font-size: var(--font-size--2xs);
cursor: pointer;
text-decoration: none;
&:hover,
&:focus-visible {
text-decoration: underline;
}
&:focus-visible {
outline: 1px solid var(--focus--border-color);
outline-offset: 2px;
}
}
.sliceFields {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
padding: var(--spacing--sm);
background-color: var(--background--subtle);
}
.field {
display: flex;
flex-direction: column;
gap: var(--spacing--3xs);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/TestCaseForm.test.ts
-48
View File
@@ -1,48 +0,0 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { createTestingPinia } from '@pinia/testing';
import { createComponentRenderer } from '@/__tests__/render';
import TestCaseForm from './TestCaseForm.vue';
vi.mock('@n8n/i18n', async (importOriginal) => ({
...(await importOriginal()),
useI18n: () => ({ baseText: (key: string) => key }),
}));
vi.mock('@/app/composables/useWorkflowId', async () => {
const { computed } = await import('vue');
const { useWorkflowsStore } = await import('@/app/stores/workflows.store');
return {
useWorkflowId: () => computed(() => useWorkflowsStore().workflowId),
useRouteWorkflowId: () => computed(() => useWorkflowsStore().workflowId),
};
});
const renderComponent = createComponentRenderer(TestCaseForm);
describe('TestCaseForm', () => {
beforeEach(() => {
createTestingPinia({ stubActions: false });
});
it('shows the Input heading and an input field for each slice field', () => {
const { getByTestId } = renderComponent({
props: {
sliceInputs: { fieldNames: ['query'], values: { query: '' }, hasExecution: true },
},
});
expect(getByTestId('evaluations-wizard-sidepanel-input-heading')).toBeInTheDocument();
expect(getByTestId('evaluations-wizard-sidepanel-input-query')).toBeInTheDocument();
});
it('hides the Input heading when there are no input fields', () => {
const { queryByTestId } = renderComponent({
props: {
sliceInputs: { fieldNames: [], values: {}, hasExecution: true },
},
});
expect(queryByTestId('evaluations-wizard-sidepanel-input-heading')).toBeNull();
});
});
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/TestCaseForm.vue
-139
View File
@@ -1,139 +0,0 @@
<script setup lang="ts">
import { computed } from 'vue';
import { storeToRefs } from 'pinia';
import { useI18n, type BaseTextKey } from '@n8n/i18n';
import { N8nIcon, N8nInput, N8nText, N8nTooltip } from '@n8n/design-system';
import { useEvaluationsWizardSidepanelStore } from '../../wizardSidepanel.store';
import { getExpectedFieldsForMetrics } from '../../evaluation.constants';
const props = withDefaults(
defineProps<{
sliceInputs: {
fieldNames: readonly string[];
values: Record<string, string>;
hasExecution: boolean;
};
/** Which parts to render. Defaults to both (wizard behavior). */
section?: 'all' | 'inputs' | 'expected';
}>(),
{ section: 'all' },
);
const wizardStore = useEvaluationsWizardSidepanelStore();
const locale = useI18n();
const { inputs, expectedValues, selectedMetricKeys } = storeToRefs(wizardStore);
const expectedFields = computed(() => getExpectedFieldsForMetrics(selectedMetricKeys.value));
const showInputs = computed(() => props.section !== 'expected');
const showExpected = computed(() => props.section !== 'inputs');
</script>
<template>
<div :class="$style.formBlock">
<template v-if="showInputs">
<div
v-if="props.sliceInputs.fieldNames.length > 0"
:class="$style.heading"
data-test-id="evaluations-wizard-sidepanel-input-heading"
>
<N8nText size="xsmall" bold color="text-dark">
{{ locale.baseText('evaluations.wizardSidepanel.step2.input') }}
</N8nText>
</div>
<div
v-for="name in props.sliceInputs.fieldNames"
:key="`input-${name}`"
:class="$style.field"
:data-test-id="`evaluations-wizard-sidepanel-input-${name}`"
>
<N8nText size="xsmall" color="text-base">
{{ name }}
</N8nText>
<N8nInput
:model-value="inputs[name] ?? ''"
type="textarea"
:rows="3"
size="small"
:placeholder="locale.baseText('evaluations.wizardSidepanel.step2.input.placeholder')"
@update:model-value="wizardStore.setInputValue(name, $event)"
/>
</div>
</template>
<div
v-for="field in showExpected ? expectedFields : []"
:key="`expected-${field.name}`"
:class="$style.field"
:data-test-id="`evaluations-wizard-sidepanel-expected-${field.name}`"
>
<div :class="$style.fieldLabel">
<N8nText size="xsmall" color="text-base">
{{ locale.baseText(field.labelKey as BaseTextKey) }}
</N8nText>
<N8nTooltip
placement="top"
:content="locale.baseText('evaluations.wizardSidepanel.step2.expected.tooltip')"
>
<N8nIcon icon="info" size="xsmall" :class="$style.fieldLabelInfo" />
</N8nTooltip>
</div>
<N8nInput
:model-value="expectedValues[field.name] ?? ''"
type="textarea"
:rows="3"
size="small"
:placeholder="locale.baseText('evaluations.wizardSidepanel.step2.expectedFieldPlaceholder')"
@update:model-value="wizardStore.setExpectedValue(field.name, $event)"
/>
</div>
</div>
</template>
<style module lang="scss">
.formBlock {
display: flex;
flex-direction: column;
border: var(--border);
border-radius: var(--radius--xs);
background-color: var(--background--surface);
overflow: hidden;
}
.heading {
padding: var(--spacing--sm) var(--spacing--sm) 0;
}
.field {
display: flex;
flex-direction: column;
gap: var(--spacing--3xs);
padding: var(--spacing--sm);
& + & {
border-top: var(--border);
}
:global(.el-textarea__inner) {
border: none;
padding: 0;
background-color: transparent;
box-shadow: none;
resize: none;
}
}
.fieldLabel {
display: flex;
align-items: center;
gap: var(--spacing--3xs);
}
.fieldLabelInfo {
color: var(--color--text--tint-1);
}
</style>
packages/frontend/editor-ui/src/features/ai/evaluation.ee/components/WizardSidepanel/useWizardPersistence.ts
+1 -1
View File
@@ -193,7 +193,7 @@ export function useWizardPersistence() {
await evaluationStore.fetchTestRuns(workflowId);
return true;
} catch (error) {
toast.showError(error, locale.baseText('evaluations.wizardSidepanel.step2.dispatchError'));
toast.showError(error, locale.baseText('evaluations.tests.dispatchError'));
return false;
} finally {
isPersisting.value = false;
packages/frontend/editor-ui/src/features/ai/evaluation.ee/composables/useEvaluationPersistenceHelpers.ts
+1 -1
View File
@@ -313,7 +313,7 @@ export function useEvaluationPersistenceHelpers() {
}
function showPersistError(error: unknown) {
toast.showError(error, locale.baseText('evaluations.wizardSidepanel.step2.persistError'));
toast.showError(error, locale.baseText('evaluations.tests.persistError'));
}
return { ensureConfig, ensureDataTable, rollback, resolveSlice, showPersistError };
packages/frontend/editor-ui/src/features/ai/evaluation.ee/composables/useTestCasePersistence.ts
+3 -3
View File
@@ -281,7 +281,7 @@ export function useTestCasePersistence() {
await evaluationStore.fetchTestRuns(workflowId);
return true;
} catch (error) {
toast.showError(error, locale.baseText('evaluations.wizardSidepanel.step2.dispatchError'));
toast.showError(error, locale.baseText('evaluations.tests.dispatchError'));
return false;
}
}
@@ -371,7 +371,7 @@ export function useTestCasePersistence() {
if (!config) {
toast.showError(
new Error('No evaluation config found. Run a single test case first.'),
locale.baseText('evaluations.wizardSidepanel.step2.persistError'),
locale.baseText('evaluations.tests.persistError'),
);
return false;
}
@@ -396,7 +396,7 @@ export function useTestCasePersistence() {
await evaluationStore.fetchTestRuns(workflowId);
return true;
} catch (error) {
toast.showError(error, locale.baseText('evaluations.wizardSidepanel.step2.dispatchError'));
toast.showError(error, locale.baseText('evaluations.tests.dispatchError'));
return false;
}
}
packages/frontend/editor-ui/src/features/ai/evaluation.ee/evaluation.constants.ts
+4 -4
View File
@@ -105,17 +105,17 @@ export type ExpectedField = {
export const CANNED_METRIC_EXPECTED_FIELDS: Partial<Record<CannedMetricKey, ExpectedField>> = {
correctness: {
name: 'expectedAnswer',
labelKey: 'evaluations.wizardSidepanel.step2.expectedAnswer',
labelKey: 'evaluations.tests.detail.expectedAnswer',
},
stringSimilarity: {
name: 'expectedAnswer',
labelKey: 'evaluations.wizardSidepanel.step2.expectedAnswer',
labelKey: 'evaluations.tests.detail.expectedAnswer',
},
categorization: {
name: 'expectedAnswer',
labelKey: 'evaluations.wizardSidepanel.step2.expectedAnswer',
labelKey: 'evaluations.tests.detail.expectedAnswer',
},
toolsUsed: { name: 'expectedTools', labelKey: 'evaluations.wizardSidepanel.step2.expectedTools' },
toolsUsed: { name: 'expectedTools', labelKey: 'evaluations.tests.detail.expectedTools' },
};
export function getExpectedFieldsForMetrics(