feat(ai-builder): Show tool hard and soft failures in agent session trace - timeline (#36405)

Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
This commit is contained in:
Anne Aguirre
2026-08-18 10:51:57 +00:00
committed by GitHub
co-authored by cubic-dev-ai[bot]
parent 207c29f8e3
commit 12cae8cbd9
7 changed files with 521 additions and 29 deletions
@@ -1522,7 +1522,8 @@
"agentSessions.timeline.memoryUpdated": "Memory updated",
"agentSessions.timeline.openForm": "Open form",
"agentSessions.timeline.workflowError": "Workflow call did not produce an execution",
"agentSessions.timeline.nodeError": "Tool experienced an error",
"agentSessions.timeline.toolError": "Tool call failed",
"agentSessions.timeline.failed": "Failed",
"agentSessions.timeline.filter": "Filter",
"agentSessions.timeline.clearFilter": "Clear",
"agentSessions.timeline.suspended": "Suspended",
@@ -1,4 +1,3 @@
/* eslint-disable import-x/no-extraneous-dependencies -- test-only patterns */
import { describe, it, expect, vi } from 'vitest';
import { mount } from '@vue/test-utils';
import { createRouter, createMemoryHistory, type Router } from 'vue-router';
@@ -137,6 +136,24 @@ describe('SessionDetailPanel — workflow branches', () => {
expect(w.find('[data-test-id="wf-log-viewer"]').exists()).toBe(true);
});
it('shows a soft-failure callout alongside the workflow execution viewer', () => {
const w = mountIt({
kind: 'workflow',
executionId: 'e1',
timestamp: 0,
workflowId: 'wf-1',
workflowName: 'WF',
workflowExecutionId: 'exec-1',
toolSuccess: true,
toolOutput: { status: 'error', error: 'Node X failed' },
});
const callout = w.find('[data-test-id="workflow-error-callout"]');
expect(callout.exists()).toBe(true);
expect(callout.text()).toContain('Node X failed');
expect(w.find('[data-test-id="wf-log-viewer"]').exists()).toBe(true);
expect(w.find('[data-test-id="detail-tool-error-badge"]').exists()).toBe(true);
});
it('opens the full execution in a new tab when the header button is clicked', async () => {
const openSpy = vi.spyOn(window, 'open').mockImplementation(() => null);
const w = mountIt({
@@ -282,6 +299,65 @@ describe('SessionDetailPanel — other kinds', () => {
expect(w.find('[data-test-id="tool-io-view"]').exists()).toBe(false);
});
it('shows a generic tool soft-failure message and header icon', () => {
const w = mountIt({
kind: 'tool',
executionId: 'e1',
timestamp: 0,
toolName: 'load_skill',
toolSuccess: true,
toolOutput: { success: false, error: 'Skill not found' },
});
const callout = w.find('[data-test-id="tool-error-callout"]');
expect(callout.exists()).toBe(true);
expect(callout.text()).toContain('Skill not found');
expect(w.find('[data-test-id="detail-tool-error-badge"]').exists()).toBe(true);
});
it('shows a nested integration error message', () => {
const w = mountIt({
kind: 'tool',
executionId: 'e1',
timestamp: 0,
toolName: 'slack_action',
toolSuccess: true,
toolOutput: {
ok: false,
error: { code: 'NO_MESSAGE_CONTEXT', message: 'No message context' },
},
});
expect(w.find('[data-test-id="tool-error-callout"]').text()).toContain('No message context');
});
it('shows an MCP structuredContent error message', () => {
const w = mountIt({
kind: 'tool',
executionId: 'e1',
timestamp: 0,
toolName: 'github_create_issue',
toolSuccess: true,
toolOutput: {
isError: true,
content: [{ type: 'text', text: '{"error":"Repository not found"}' }],
structuredContent: { error: 'Repository not found' },
},
});
expect(w.find('[data-test-id="tool-error-callout"]').text()).toContain('Repository not found');
});
it('does not show a failure callout or header icon for a successful tool', () => {
const w = mountIt({
kind: 'tool',
executionId: 'e1',
timestamp: 0,
toolName: 'http',
toolSuccess: true,
toolOutput: { ok: true },
});
expect(w.find('[data-test-id="tool-error-callout"]').exists()).toBe(false);
expect(w.find('[data-test-id="detail-tool-error-badge"]').exists()).toBe(false);
});
it('renders the ToolIoView for node tool calls', () => {
const w = mountIt({
kind: 'node',
@@ -312,9 +388,7 @@ describe('SessionDetailPanel — other kinds', () => {
});
const callout = w.find('[data-test-id="node-error-callout"]');
expect(callout.exists()).toBe(true);
expect(callout.text()).toContain(
'Tool experienced an error: Node does not have any credentials set',
);
expect(callout.text()).toContain('Tool call failed: Node does not have any credentials set');
expect(w.get('[data-test-id="detail-tool-error-badge"]').text()).toBe('Error');
});
@@ -333,7 +407,7 @@ describe('SessionDetailPanel — other kinds', () => {
});
const callout = w.find('[data-test-id="node-error-callout"]');
expect(callout.exists()).toBe(true);
expect(callout.text()).toContain('Tool experienced an error');
expect(callout.text()).toContain('Tool call failed');
expect(callout.text()).not.toContain(':');
});
@@ -1,4 +1,3 @@
/* eslint-disable import-x/no-extraneous-dependencies -- test-only patterns */
import { describe, it, expect, vi } from 'vitest';
import { mount } from '@vue/test-utils';
import SessionTimelineChart from '../components/SessionTimelineChart.vue';
@@ -116,6 +115,51 @@ describe('SessionTimelineChart', () => {
expect(blocks[2].element.getAttribute('data-selected')).toBe('true');
});
it('marks a generic tool soft-failure block as failed', () => {
const w = mountChart({
items: [
item({
kind: 'tool',
toolSuccess: true,
toolOutput: { success: false, error: 'boom' },
}),
],
});
const block = w.get('[data-test-id="timeline-block"]');
expect(block.attributes('data-error')).toBe('true');
expect(block.classes()).toContain('error');
});
it('marks a workflow soft-failure block as failed', () => {
const w = mountChart({
items: [
item({
kind: 'workflow',
toolSuccess: true,
toolOutput: { status: 'error', error: 'boom' },
}),
],
});
const block = w.get('[data-test-id="timeline-block"]');
expect(block.attributes('data-error')).toBe('true');
expect(block.classes()).toContain('error');
});
it('does not mark a successful tool block as failed', () => {
const w = mountChart({
items: [
item({
kind: 'tool',
toolSuccess: true,
toolOutput: { ok: true },
}),
],
});
const block = w.get('[data-test-id="timeline-block"]');
expect(block.attributes('data-error')).toBeUndefined();
expect(block.classes()).not.toContain('error');
});
it('renders the localized "Idle" pill text inside each idle segment', () => {
const w = mountChart({ idleRanges: [{ start: 1500, end: 2000 }] });
const idle = w.find('[data-test-id="timeline-idle"]');
@@ -0,0 +1,106 @@
/* eslint-disable import-x/no-extraneous-dependencies, @typescript-eslint/no-unsafe-assignment -- test-only patterns: @vue/test-utils is a transitive devDep, mock reads */
import { describe, it, expect, vi } from 'vitest';
import { mount } from '@vue/test-utils';
import type { TimelineItem } from '../session-timeline.types';
vi.mock('@n8n/i18n', () => ({
useI18n: () => ({ baseText: (key: string) => key }),
}));
vi.mock('vue-router', () => ({
useRouter: () => ({ resolve: () => ({ href: '/wf/1' }) }),
}));
vi.mock('@/app/utils/formatters/dateFormatter', () => ({
convertToDisplayDate: () => ({ date: '', time: '00:00' }),
}));
vi.mock('@n8n/utils/string/truncate', () => ({
truncate: (value: string) => value,
}));
vi.mock('../utils/delegate-tool', () => ({
delegateLabel: () => 'Sub-agent',
isDelegateSubAgentTool: () => false,
}));
vi.mock('../utils/toolDisplayName', () => ({
formatToolNameForDisplay: (name: string) => name,
resolveToolNameForDisplay: (name: string) => name,
}));
const STUBS = {
N8nTooltip: { template: '<span><slot /></span>' },
N8nIcon: {
props: ['icon', 'size'],
template:
'<span :data-icon="icon" :data-testid="icon ? `icon-${icon}` : undefined"><slot /></span>',
},
N8nBadge: {
props: ['theme', 'size'],
template: '<span data-test-id="timeline-tool-error-badge"><slot /></span>',
},
SessionTimelinePill: {
props: ['kind'],
template: '<span :data-testid="pill" :data-kind="kind" />',
},
};
function item(partial: Partial<TimelineItem>): TimelineItem {
return {
kind: 'tool',
executionId: 'e1',
timestamp: 1000,
toolName: 'http',
...partial,
} as TimelineItem;
}
async function renderComponent(it: TimelineItem) {
const { default: SessionTimelineRow } = await import('../components/SessionTimelineRow.vue');
return mount(SessionTimelineRow, {
props: { item: it, selected: false },
global: { stubs: STUBS },
});
}
describe('SessionTimelineRow', () => {
it('renders the failure icon for a generic tool soft-failure', async () => {
const wrapper = await renderComponent(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: { success: false, error: 'boom' },
}),
);
expect(wrapper.find('[data-test-id="timeline-tool-error-badge"]').exists()).toBe(true);
}, 30_000);
it('renders the failure icon for a workflow soft-failure (success true, status error)', async () => {
const wrapper = await renderComponent(
item({
kind: 'workflow',
toolSuccess: true,
toolOutput: { status: 'error', error: 'node X failed' },
}),
);
expect(wrapper.find('[data-test-id="timeline-tool-error-badge"]').exists()).toBe(true);
});
it('does not render the failure icon for a successful tool call', async () => {
const wrapper = await renderComponent(
item({ kind: 'tool', toolSuccess: true, toolOutput: { ok: true } }),
);
expect(wrapper.find('[data-test-id="timeline-tool-error-badge"]').exists()).toBe(false);
});
it('does not render the failure icon for an in-flight tool call', async () => {
const wrapper = await renderComponent(item({ kind: 'tool', toolSuccess: undefined }));
expect(wrapper.find('[data-test-id="timeline-tool-error-badge"]').exists()).toBe(false);
});
it('does not render the failure icon for non-tool kinds', async () => {
const wrapper = await renderComponent(item({ kind: 'user', toolSuccess: false }));
expect(wrapper.find('[data-test-id="timeline-tool-error-badge"]').exists()).toBe(false);
});
});
@@ -10,7 +10,9 @@ import {
flattenExecutionsToTimelineItems,
itemStatusFilterKey,
matchesSearch,
isErroredToolCallTimelineItem,
matchesTimelineFilters,
timelineItemErrorMessage,
} from '../session-timeline.utils';
import type { TimelineItem } from '../session-timeline.types';
@@ -663,3 +665,188 @@ describe('flattenExecutionsToTimelineItems', () => {
expect(items.map((i) => i.content)).toEqual(['a', 'b']);
});
});
describe('isErroredToolCallTimelineItem', () => {
it.each([
['runtime hard failure', { kind: 'tool', toolSuccess: false }],
['toolOutcome error', { kind: 'tool', toolOutcome: 'error' }],
['generic error string', { kind: 'tool', toolSuccess: true, toolOutput: { error: 'boom' } }],
[
'integration error object',
{
kind: 'tool',
toolSuccess: true,
toolOutput: { ok: false, error: { code: 'ACTION_FAILED', message: 'Action failed' } },
},
],
[
'workflow error status',
{ kind: 'workflow', toolSuccess: true, toolOutput: { status: 'error' } },
],
[
'delegate failed status',
{ kind: 'tool', toolSuccess: true, toolOutput: { status: 'failed' } },
],
[
'workspace success false',
{ kind: 'tool', toolSuccess: true, toolOutput: { success: false } },
],
['integration ok false', { kind: 'tool', toolSuccess: true, toolOutput: { ok: false } }],
['MCP isError true', { kind: 'tool', toolSuccess: true, toolOutput: { isError: true } }],
] satisfies Array<[string, Partial<TimelineItem>]>)('flags %s', (_label, partial) => {
expect(isErroredToolCallTimelineItem(item(partial))).toBe(true);
});
it.each([
['empty error string', { toolOutput: { error: '' } }],
['empty nested error message', { toolOutput: { error: { message: '' } } }],
['success status', { toolOutput: { status: 'success' } }],
['success true', { toolOutput: { success: true } }],
['ok true', { toolOutput: { ok: true } }],
['isError false', { toolOutput: { isError: false } }],
['non-record output', { toolOutput: 'failed' }],
['in-flight call', { toolSuccess: undefined, toolOutput: undefined }],
] satisfies Array<[string, Partial<TimelineItem>]>)('does not flag %s', (_label, partial) => {
expect(
isErroredToolCallTimelineItem(item({ kind: 'tool', toolSuccess: true, ...partial })),
).toBe(false);
});
it('does not flag user/agent/suspension kinds', () => {
expect(isErroredToolCallTimelineItem(item({ kind: 'user', toolSuccess: false }))).toBe(false);
expect(isErroredToolCallTimelineItem(item({ kind: 'agent', toolSuccess: false }))).toBe(false);
expect(isErroredToolCallTimelineItem(item({ kind: 'suspension', toolSuccess: false }))).toBe(
false,
);
});
});
describe('timelineItemErrorMessage', () => {
it('extracts the error message from a thrown tool call', () => {
expect(
timelineItemErrorMessage(
item({ kind: 'tool', toolSuccess: false, toolOutput: { error: 'timed out' } }),
),
).toBe('timed out');
});
it('extracts the error message from a workflow soft-failure', () => {
expect(
timelineItemErrorMessage(
item({
kind: 'workflow',
toolSuccess: true,
toolOutput: { status: 'error', error: 'node X failed' },
}),
),
).toBe('node X failed');
});
it('extracts a nested integration error message', () => {
expect(
timelineItemErrorMessage(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: {
ok: false,
error: { code: 'NO_MESSAGE_CONTEXT', message: 'No message context' },
},
}),
),
).toBe('No message context');
});
it('extracts MCP structuredContent.error', () => {
expect(
timelineItemErrorMessage(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: {
isError: true,
content: [{ type: 'text', text: '{"error":"Workflow not found"}' }],
structuredContent: { error: 'Workflow not found' },
},
}),
),
).toBe('Workflow not found');
});
it('extracts MCP structuredContent.error.message', () => {
expect(
timelineItemErrorMessage(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: {
isError: true,
content: [{ type: 'text', text: 'ignored' }],
structuredContent: { error: { message: 'Tool execution failed' } },
},
}),
),
).toBe('Tool execution failed');
});
it('extracts MCP text content when structuredContent has no error', () => {
expect(
timelineItemErrorMessage(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: {
isError: true,
content: [{ type: 'text', text: 'Access denied by user' }],
},
}),
),
).toBe('Access denied by user');
});
it('extracts MCP JSON text envelopes without dumping extra payload fields', () => {
expect(
timelineItemErrorMessage(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: {
isError: true,
content: [
{
type: 'text',
text: JSON.stringify({ error: 'Selector not found', snapshot: '<huge>' }),
},
],
},
}),
),
).toBe('Selector not found');
});
it('returns empty string when no error message is present', () => {
expect(
timelineItemErrorMessage(item({ kind: 'tool', toolSuccess: false, toolOutput: {} })),
).toBe('');
expect(
timelineItemErrorMessage(
item({ kind: 'workflow', toolSuccess: true, toolOutput: { status: 'error' } }),
),
).toBe('');
expect(
timelineItemErrorMessage(
item({
kind: 'tool',
toolSuccess: true,
toolOutput: { isError: true, content: [{ type: 'image', data: 'abc' }] },
}),
),
).toBe('');
});
it('returns empty string for non-failed items', () => {
expect(
timelineItemErrorMessage(item({ kind: 'tool', toolSuccess: true, toolOutput: { ok: true } })),
).toBe('');
});
});
@@ -25,8 +25,10 @@ import ToolIoView from './ToolIoView.vue';
import type { TimelineItem } from '../session-timeline.types';
import {
hitlTimelineName,
isErroredToolCallTimelineItem,
isSubAgentTimelineItem,
linkedToolDisplayName,
timelineItemErrorMessage,
timelineItemStatus,
} from '../session-timeline.utils';
import { delegateLabel } from '../utils/delegate-tool';
@@ -197,23 +199,22 @@ const headerIcon = computed((): IconName => {
return 'clock';
});
const nodeErrorMessage = computed((): string => {
const isFailed = computed((): boolean =>
props.item ? isErroredToolCallTimelineItem(props.item) : false,
);
/**
* Error message for a failed tool/workflow/node call. It surfaces a string,
* nested `toolOutput.error.message`, or MCP `structuredContent.error` / text
* content when available. Soft-failure payloads are detected in
* `isErroredToolCallTimelineItem`.
*/
const errorMessage = computed((): string => {
const item = props.item;
if (
!item ||
item.kind !== 'node' ||
(item.toolOutcome !== 'error' &&
!(item.toolOutcome === undefined && item.toolSuccess === false))
) {
return '';
}
const prefix = i18n.baseText('agentSessions.timeline.nodeError');
const output = item.toolOutput;
if (output && typeof output === 'object' && 'error' in output) {
const err = (output as { error: unknown }).error;
if (typeof err === 'string' && err.length > 0) return `${prefix}: ${err}`;
}
return prefix;
if (!item || !isFailed.value) return '';
const prefix = i18n.baseText('agentSessions.timeline.toolError');
const message = timelineItemErrorMessage(item);
return message ? `${prefix}: ${message}` : prefix;
});
const workflowFormOutput = computed((): { formUrl: string; message: string } | null => {
@@ -299,6 +300,14 @@ const workflowFormOutput = computed((): { formUrl: string; message: string } | n
</template>
<template v-else-if="item.kind === 'workflow'">
<N8nCallout
v-if="isFailed"
theme="danger"
data-test-id="workflow-error-callout"
:class="$style.errorCallout"
>
{{ errorMessage }}
</N8nCallout>
<WorkflowExecutionLogViewer
v-if="item.workflowExecutionId && item.workflowId"
:key="`${item.workflowId}:${item.workflowExecutionId}`"
@@ -338,6 +347,14 @@ const workflowFormOutput = computed((): { formUrl: string; message: string } | n
</template>
<template v-else-if="item.kind === 'tool'">
<N8nCallout
v-if="isFailed"
theme="danger"
data-test-id="tool-error-callout"
:class="$style.errorCallout"
>
{{ errorMessage }}
</N8nCallout>
<template v-if="actionCard">
<RichInteractionCard :input="actionCard" :output="ensureParsed(item.toolOutput)" />
</template>
@@ -356,8 +373,8 @@ const workflowFormOutput = computed((): { formUrl: string; message: string } | n
</template>
<template v-else-if="item.kind === 'node'">
<N8nCallout v-if="nodeErrorMessage" theme="danger" data-test-id="node-error-callout">
{{ nodeErrorMessage }}
<N8nCallout v-if="errorMessage" theme="danger" data-test-id="node-error-callout">
{{ errorMessage }}
</N8nCallout>
<ToolIoView
:name="(item.nodeDisplayName ?? formatToolNameForDisplay(item.toolName)) || 'node'"
@@ -426,6 +443,10 @@ const workflowFormOutput = computed((): { formUrl: string; message: string } | n
white-space: nowrap;
}
.errorCallout {
margin-bottom: var(--spacing--2xs);
}
.container {
display: flex;
flex-direction: column;
@@ -28,15 +28,74 @@ export function isSubAgentTimelineItem(item: TimelineItem): boolean {
return item.kind === 'tool' && isDelegateSubAgentTool(item.toolName);
}
function errorTextFromValue(value: unknown): string {
if (typeof value === 'string' && value.length > 0) return value;
if (isRecord(value) && typeof value.message === 'string' && value.message.length > 0) {
return value.message;
}
return '';
}
/** MCP CallToolResult stores the message in structuredContent.error or text content. */
function mcpErrorMessage(output: Record<string, unknown>): string {
if (isRecord(output.structuredContent)) {
const fromStructured = errorTextFromValue(output.structuredContent.error);
if (fromStructured) return fromStructured;
}
if (!Array.isArray(output.content)) return '';
for (const block of output.content) {
if (!isRecord(block) || block.type !== 'text' || typeof block.text !== 'string') continue;
const text = block.text.trim();
if (!text) continue;
try {
const parsed: unknown = JSON.parse(text);
if (typeof parsed === 'string' && parsed.length > 0) return parsed;
if (isRecord(parsed)) {
const fromJson = errorTextFromValue(parsed.error);
if (fromJson) return fromJson;
}
} catch {
return text;
}
}
return '';
}
/**
* A tool/workflow/node call is failed when the runtime recorded an error
* outcome, or when a built-in tool returned a soft-failure payload instead
* of throwing. In-flight calls without output are not failed.
*/
export function isErroredToolCallTimelineItem(item: TimelineItem): boolean {
if (item.kind !== 'tool' && item.kind !== 'node' && item.kind !== 'workflow') return false;
if (item.kind !== 'tool' && item.kind !== 'workflow' && item.kind !== 'node') return false;
if (item.toolOutcome === 'error') return true;
if (item.toolOutcome === undefined && item.toolSuccess === false) return true;
if (!isRecord(item.toolOutput)) return false;
const { error, status, success, ok, isError } = item.toolOutput;
const hasErrorMessage =
(typeof error === 'string' && error.length > 0) ||
(isRecord(error) && typeof error.message === 'string' && error.message.length > 0);
return (
item.toolOutcome === 'error' ||
(item.toolOutcome === undefined && item.toolSuccess === false) ||
(item.kind === 'workflow' && isRecord(item.toolOutput) && item.toolOutput.status === 'error')
hasErrorMessage ||
status === 'error' ||
status === 'failed' ||
success === false ||
ok === false ||
isError === true
);
}
/** Extracts a human-readable error message from a failed item's tool output. */
export function timelineItemErrorMessage(item: TimelineItem): string {
if (!isErroredToolCallTimelineItem(item)) return '';
const output = item.toolOutput;
if (!isRecord(output)) return '';
return errorTextFromValue(output.error) || mcpErrorMessage(output);
}
export function hitlTimelineNameKey(item: TimelineItem): BaseTextKey | undefined {
if (item.hitlRequestType !== 'approval') return undefined;
if (item.kind === 'suspension') return 'agentSessions.timeline.approvalRequestForTool';