feat(ai-builder): Consolidate builder telemetry (#22644)

This commit is contained in:
Mutasem Aldmour
2025-12-08 10:56:38 +01:00
committed by GitHub
parent 4df815e6dd
commit dcbc24d3f7
36 changed files with 1376 additions and 528 deletions
@@ -99,7 +99,10 @@ export async function runSingleTest(
// Generate workflow
const startTime = Date.now();
await consumeGenerator(
agent.chat(getChatPayload(testCase.prompt, testCase.id, opts?.featureFlags), userId),
agent.chat(
getChatPayload('single-eval', testCase.prompt, testCase.id, opts?.featureFlags),
userId,
),
);
const generationTime = Date.now() - startTime;
@@ -356,7 +356,10 @@ async function runSingleGeneration(
// Generate workflow
await consumeGenerator(
agent.chat(getChatPayload(inputs.prompt, runId, featureFlags), `pairwise-gen-${generationIndex}`),
agent.chat(
getChatPayload('pairewise-gen', inputs.prompt, runId, featureFlags),
`pairwise-gen-${generationIndex}`,
),
);
const state = await agent.getState(runId, `pairwise-gen-${generationIndex}`);
@@ -441,7 +444,16 @@ function createPairwiseWorkflowGenerator(
// Run all generations in parallel
const generationResults = await Promise.all(
Array.from({ length: numGenerations }, async (_, i) => {
return await runSingleGeneration(parsedNodeTypes, llm, numJudges, inputs, i, log, featureFlags, tracer);
return await runSingleGeneration(
parsedNodeTypes,
llm,
numJudges,
inputs,
i,
log,
featureFlags,
tracer,
);
}),
);
@@ -838,7 +850,12 @@ export async function runLocalPairwiseEvaluation(options: LocalPairwiseOptions):
const genStartTime = Date.now();
const runId = generateRunId();
const agent = createAgent(parsedNodeTypes, llm, undefined, featureFlags);
await consumeGenerator(agent.chat(getChatPayload(prompt, runId, featureFlags), `local-gen-${genIndex}`));
await consumeGenerator(
agent.chat(
getChatPayload('pairwise-local', prompt, runId, featureFlags),
`local-gen-${genIndex}`,
),
);
const state = await agent.getState(runId, `local-gen-${genIndex}`);
if (!state.values || !isWorkflowStateValues(state.values)) {
@@ -46,7 +46,10 @@ function createWorkflowGenerator(
// Create agent for this run
const agent = createAgent(parsedNodeTypes, llm, tracer);
await consumeGenerator(
agent.chat(getChatPayload(messageContent, runId, featureFlags), 'langsmith-eval-user'),
agent.chat(
getChatPayload('langsmith-evals', messageContent, runId, featureFlags),
'langsmith-eval-user',
),
);
// Get generated workflow with validation
@@ -6,6 +6,7 @@ import { Client } from 'langsmith';
import type { INodeTypeDescription } from 'n8n-workflow';
import { join } from 'path';
import pc from 'picocolors';
import { v4 as uuid } from 'uuid';
import { anthropicClaudeSonnet45 } from '../../src/llm-config';
import type { BuilderFeatureFlags, ChatPayload } from '../../src/workflow-builder-agent';
@@ -278,18 +279,20 @@ export async function consumeGenerator<T>(gen: AsyncGenerator<T>) {
}
export function getChatPayload(
evalType: string,
message: string,
id: string,
workflowId: string,
featureFlags?: BuilderFeatureFlags,
): ChatPayload {
return {
id: `${evalType}-${uuid()}`,
featureFlags: featureFlags ?? {
multiAgent: true,
templateExamples: false,
},
message,
workflowContext: {
currentWorkflow: { id, nodes: [], connections: {} },
currentWorkflow: { id: workflowId, nodes: [], connections: {} },
},
};
}
@@ -61,10 +61,10 @@ export class AiWorkflowBuilderService {
});
}
private async getApiProxyAuthHeaders(user: IUser) {
private async getApiProxyAuthHeaders(user: IUser, userMessageId: string) {
assert(this.client);
const authResponse = await this.client.getBuilderApiProxyToken(user);
const authResponse = await this.client.getBuilderApiProxyToken(user, { userMessageId });
const authHeaders = {
// eslint-disable-next-line @typescript-eslint/naming-convention
Authorization: `${authResponse.tokenType} ${authResponse.accessToken}`,
@@ -73,7 +73,10 @@ export class AiWorkflowBuilderService {
return authHeaders;
}
private async setupModels(user: IUser): Promise<{
private async setupModels(
user: IUser,
userMessageId: string,
): Promise<{
anthropicClaude: ChatAnthropic;
tracingClient?: TracingClient;
// eslint-disable-next-line @typescript-eslint/naming-convention
@@ -82,7 +85,7 @@ export class AiWorkflowBuilderService {
try {
// If client is provided, use it for API proxy
if (this.client) {
const authHeaders = await this.getApiProxyAuthHeaders(user);
const authHeaders = await this.getApiProxyAuthHeaders(user, userMessageId);
// Extract baseUrl from client configuration
const baseUrl = this.client.getApiProxyBaseUrl();
@@ -158,8 +161,11 @@ export class AiWorkflowBuilderService {
});
}
private async getAgent(user: IUser, featureFlags?: BuilderFeatureFlags) {
const { anthropicClaude, tracingClient, authHeaders } = await this.setupModels(user);
private async getAgent(user: IUser, userMessageId: string, featureFlags?: BuilderFeatureFlags) {
const { anthropicClaude, tracingClient, authHeaders } = await this.setupModels(
user,
userMessageId,
);
const agent = new WorkflowBuilderAgent({
parsedNodeTypes: this.parsedNodeTypes,
@@ -208,7 +214,7 @@ export class AiWorkflowBuilderService {
}
async *chat(payload: ChatPayload, user: IUser, abortSignal?: AbortSignal) {
const agent = await this.getAgent(user, payload.featureFlags);
const agent = await this.getAgent(user, payload.id, payload.featureFlags);
const userId = user?.id?.toString();
const workflowId = payload.workflowContext?.currentWorkflow?.id;
@@ -219,7 +225,7 @@ export class AiWorkflowBuilderService {
// After the stream completes, track telemetry
if (this.onTelemetryEvent && userId) {
try {
await this.trackBuilderReplyTelemetry(agent, workflowId, userId);
await this.trackBuilderReplyTelemetry(agent, workflowId, userId, payload.id);
} catch (error) {
this.logger?.error('Failed to track builder reply telemetry', { error });
}
@@ -230,6 +236,7 @@ export class AiWorkflowBuilderService {
agent: WorkflowBuilderAgent,
workflowId: string | undefined,
userId: string,
userMessageId: string,
): Promise<void> {
if (!this.onTelemetryEvent) return;
@@ -270,6 +277,7 @@ export class AiWorkflowBuilderService {
...(state.values.templateIds.length > 0 && {
templates_selected: state.values.templateIds,
}),
user_message_id: userMessageId,
};
this.onTelemetryEvent('Builder replied to user message', properties);
@@ -151,9 +151,6 @@ describe('AiWorkflowBuilderService', () => {
// Mock AI assistant client
mockClient = mock<AiAssistantClient>();
(mockClient.generateApiProxyCredentials as jest.Mock).mockResolvedValue({
apiKey: 'test-api-key',
});
(mockClient.getBuilderApiProxyToken as jest.Mock).mockResolvedValue({
tokenType: 'Bearer',
accessToken: 'test-access-token',
@@ -301,6 +298,7 @@ describe('AiWorkflowBuilderService', () => {
beforeEach(() => {
mockPayload = {
id: '12345',
message: 'Create a simple workflow',
workflowContext: {
currentWorkflow: { id: 'test-workflow' },
@@ -571,6 +569,7 @@ describe('AiWorkflowBuilderService', () => {
it('should handle complete workflow from chat to session retrieval', async () => {
const workflowId = 'integration-test-workflow';
const mockPayload: ChatPayload = {
id: '545623',
message: 'Create a workflow with HTTP request',
workflowContext: {
currentWorkflow: { id: workflowId },
@@ -146,6 +146,7 @@ describe('WorkflowBuilderAgent', () => {
beforeEach(() => {
mockPayload = {
id: '12345',
message: 'Create a workflow',
workflowContext: {
currentWorkflow: { id: 'workflow-123' },
@@ -156,6 +157,7 @@ describe('WorkflowBuilderAgent', () => {
it('should throw ValidationError when message exceeds maximum length', async () => {
const longMessage = 'x'.repeat(MAX_AI_BUILDER_PROMPT_LENGTH + 1);
const payload: ChatPayload = {
id: '12345',
message: longMessage,
};
@@ -173,6 +175,7 @@ describe('WorkflowBuilderAgent', () => {
it('should handle valid message length', async () => {
const validMessage = 'Create a simple workflow';
const payload: ChatPayload = {
id: '12345',
message: validMessage,
};
@@ -491,6 +494,7 @@ describe('WorkflowBuilderAgent', () => {
);
const generator = agent.chat({
id: '12345',
message: 'Create a workflow',
featureFlags: { templateExamples: true },
});
@@ -511,6 +515,7 @@ describe('WorkflowBuilderAgent', () => {
);
const generator = agent.chat({
id: '12345',
message: 'Create a workflow',
featureFlags: { templateExamples: false },
});
@@ -530,7 +535,10 @@ describe('WorkflowBuilderAgent', () => {
mockAsyncGenerator,
);
const generator = agent.chat({ message: 'Create a workflow' });
const generator = agent.chat({
id: '12345',
message: 'Create a workflow',
});
await generator.next();
expect(mockCreateMainAgentPrompt).toHaveBeenCalledWith({ includeExamplesPhase: false });
@@ -159,6 +159,7 @@ export interface BuilderFeatureFlags {
}
export interface ChatPayload {
id: string;
message: string;
workflowContext?: {
executionSchema?: NodeExecutionSchema[];
@@ -3,6 +3,7 @@ import { AiBuilderChatRequestDto } from '../ai-build-request.dto';
describe('AiBuilderChatRequestDto', () => {
const validBasePayload = {
payload: {
id: '12345',
role: 'user' as const,
type: 'message' as const,
text: 'Build me a workflow',
@@ -305,6 +306,7 @@ describe('AiBuilderChatRequestDto', () => {
const invalidRequest = {
...validBasePayload,
payload: {
id: '12345',
role: 'user' as const,
type: 'message' as const,
workflowContext: validBasePayload.payload.workflowContext,
@@ -315,5 +317,21 @@ describe('AiBuilderChatRequestDto', () => {
expect(result.success).toBe(false);
});
it('should fail when id is missing', () => {
const invalidRequest = {
...validBasePayload,
payload: {
role: 'user' as const,
type: 'message' as const,
text: 'text',
workflowContext: validBasePayload.payload.workflowContext,
},
};
const result = AiBuilderChatRequestDto.safeParse(invalidRequest);
expect(result.success).toBe(false);
});
});
});
@@ -10,6 +10,7 @@ export interface ExpressionValue {
export class AiBuilderChatRequestDto extends Z.class({
payload: z.object({
id: z.string(),
role: z.literal('user'),
type: z.literal('message'),
text: z.string(),
@@ -118,6 +118,7 @@ describe('AiController', () => {
describe('build', () => {
const payload: AiBuilderChatRequestDto = {
payload: {
id: '12345',
text: 'Create a workflow',
type: 'message',
role: 'user',
@@ -146,6 +147,8 @@ describe('AiController', () => {
expect(workflowBuilderService.chat).toHaveBeenCalledWith(
{
id: '12345',
featureFlags: undefined,
message: 'Create a workflow',
workflowContext: {
currentWorkflow: { id: 'workflow123' },
@@ -55,9 +55,10 @@ export class AiController {
res.on('close', handleClose);
const { text, workflowContext, featureFlags } = payload.payload;
const { id, text, workflowContext, featureFlags } = payload.payload;
const aiResponse = this.workflowBuilderService.chat(
{
id,
message: text,
workflowContext: {
currentWorkflow: workflowContext.currentWorkflow,
@@ -100,6 +100,7 @@ describe('WorkflowBuilderService', () => {
describe('chat', () => {
it('should create AiWorkflowBuilderService on first chat call without AI assistant client', async () => {
const mockPayload = {
id: '12345',
message: 'test message',
workflowContext: {},
};
@@ -134,6 +135,7 @@ describe('WorkflowBuilderService', () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -170,6 +172,7 @@ describe('WorkflowBuilderService', () => {
it('should reuse the same service instance on subsequent calls', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -201,6 +204,7 @@ describe('WorkflowBuilderService', () => {
it('should pass abort signal to underlying service', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -265,6 +269,7 @@ describe('WorkflowBuilderService', () => {
it('should send push notification when credits are updated', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -313,6 +318,7 @@ describe('WorkflowBuilderService', () => {
it('should handle multiple credit updates', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -373,6 +379,7 @@ describe('WorkflowBuilderService', () => {
it('should call telemetry.track when telemetry event is triggered', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -419,6 +426,7 @@ describe('WorkflowBuilderService', () => {
it('should handle multiple telemetry events', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -463,6 +471,7 @@ describe('WorkflowBuilderService', () => {
it('should handle telemetry events with empty properties', async () => {
const mockPayload = {
message: 'test message',
id: '12345',
workflowContext: {},
};
@@ -4,9 +4,11 @@ import {
continueEvaluationLoop,
executionFinished,
getRunExecutionData,
handleExecutionFinishedWithSuccessOrOther,
handleExecutionFinishedWithErrorOrCanceled,
type SimplifiedExecution,
} from './executionFinished';
import type { ITaskData } from 'n8n-workflow';
import type { IRunExecutionData, ITaskData } from 'n8n-workflow';
import { EVALUATION_TRIGGER_NODE_TYPE } from 'n8n-workflow';
import type { INodeUi, IWorkflowDb } from '@/Interface';
import type { Router } from 'vue-router';
@@ -17,6 +19,7 @@ import { useWorkflowsStore } from '@/app/stores/workflows.store';
import { useUIStore } from '@/app/stores/ui.store';
import { mockedStore } from '@/__tests__/utils';
import { useReadyToRunStore } from '@/features/workflows/readyToRun/stores/readyToRun.store';
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
const opts = {
workflowState: mock<WorkflowState>(),
@@ -524,3 +527,92 @@ describe('executionFinished', () => {
expect(runWorkflow).not.toHaveBeenCalled();
});
});
describe('manual execution stats tracking', () => {
beforeEach(() => {
vi.resetAllMocks();
});
describe('handleExecutionFinishedWithSuccessOrOther', () => {
it('increments success stats on successful execution', () => {
const pinia = createTestingPinia();
setActivePinia(pinia);
const builderStore = mockedStore(useBuilderStore);
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
handleExecutionFinishedWithSuccessOrOther(mock<WorkflowState>(), 'success', false);
expect(incrementSpy).toHaveBeenCalledWith('success');
});
it('does not increment success stats when successToastAlreadyShown is true', () => {
const pinia = createTestingPinia();
setActivePinia(pinia);
const builderStore = mockedStore(useBuilderStore);
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
handleExecutionFinishedWithSuccessOrOther(mock<WorkflowState>(), 'success', true);
expect(incrementSpy).not.toHaveBeenCalled();
});
it('does not increment stats for non-success status', () => {
const pinia = createTestingPinia();
setActivePinia(pinia);
const builderStore = mockedStore(useBuilderStore);
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
handleExecutionFinishedWithSuccessOrOther(mock<WorkflowState>(), 'error', false);
expect(incrementSpy).not.toHaveBeenCalled();
});
});
describe('handleExecutionFinishedWithErrorOrCanceled', () => {
it('increments error stats on execution error', () => {
const pinia = createTestingPinia();
setActivePinia(pinia);
const builderStore = mockedStore(useBuilderStore);
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
const execution = mock<SimplifiedExecution>({
status: 'error',
data: {
resultData: {
error: { message: 'test error', name: 'Error' },
},
},
});
handleExecutionFinishedWithErrorOrCanceled(
execution,
mock<IRunExecutionData>({ resultData: { error: { message: 'test', name: 'Error' } } }),
);
expect(incrementSpy).toHaveBeenCalledWith('error');
});
it('does not increment stats for canceled executions', () => {
const pinia = createTestingPinia();
setActivePinia(pinia);
const builderStore = mockedStore(useBuilderStore);
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
const execution = mock<SimplifiedExecution>({
status: 'canceled',
});
handleExecutionFinishedWithErrorOrCanceled(
execution,
mock<IRunExecutionData>({ resultData: {} }),
);
expect(incrementSpy).not.toHaveBeenCalled();
});
});
});
@@ -14,6 +14,7 @@ import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
import { useSettingsStore } from '@/app/stores/settings.store';
import { useUIStore } from '@/app/stores/ui.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
import {
SampleTemplates,
isPrebuiltAgentTemplateId,
@@ -28,7 +29,13 @@ import { getTriggerNodeServiceName } from '@/app/utils/nodeTypesUtils';
import type { ExecutionFinished } from '@n8n/api-types/push/execution';
import { useI18n } from '@n8n/i18n';
import { parse } from 'flatted';
import type { ExpressionError, IDataObject, IRunExecutionData, IWorkflowBase } from 'n8n-workflow';
import type {
ExecutionStatus,
ExpressionError,
IDataObject,
IRunExecutionData,
IWorkflowBase,
} from 'n8n-workflow';
import {
EVALUATION_TRIGGER_NODE_TYPE,
TelemetryHelpers,
@@ -115,7 +122,11 @@ export async function executionFinished(
let successToastAlreadyShown = false;
if (data.status === 'success') {
handleExecutionFinishedWithSuccessOrOther(options.workflowState, successToastAlreadyShown);
handleExecutionFinishedWithSuccessOrOther(
options.workflowState,
data.status,
successToastAlreadyShown,
);
successToastAlreadyShown = true;
}
@@ -140,7 +151,11 @@ export async function executionFinished(
} else if (execution.status === 'error' || execution.status === 'canceled') {
handleExecutionFinishedWithErrorOrCanceled(execution, runExecutionData);
} else {
handleExecutionFinishedWithSuccessOrOther(options.workflowState, successToastAlreadyShown);
handleExecutionFinishedWithSuccessOrOther(
options.workflowState,
execution.status,
successToastAlreadyShown,
);
}
setRunExecutionData(execution, runExecutionData, options.workflowState);
@@ -351,6 +366,8 @@ export function handleExecutionFinishedWithErrorOrCanceled(
});
toast.showMessage({ title, message, type: 'error', duration: 0 });
useBuilderStore().incrementManualExecutionStats('error');
}
}
@@ -381,6 +398,7 @@ function handleExecutionFinishedSuccessfully(
*/
export function handleExecutionFinishedWithSuccessOrOther(
workflowState: WorkflowState,
executionStatus: ExecutionStatus,
successToastAlreadyShown: boolean,
) {
const workflowsStore = useWorkflowsStore();
@@ -427,6 +445,12 @@ export function handleExecutionFinishedWithSuccessOrOther(
workflowState,
);
}
// Execution finished is triggered multiple times
// use "successToastAlreadyShown" flag to avoid double counting executions
if (executionStatus === 'success' && !successToastAlreadyShown) {
useBuilderStore().incrementManualExecutionStats('success');
}
}
export function setRunExecutionData(
@@ -40,7 +40,7 @@ export async function executionRecovered(
} else if (execution.status === 'error' || execution.status === 'canceled') {
handleExecutionFinishedWithErrorOrCanceled(execution, runExecutionData);
} else {
handleExecutionFinishedWithSuccessOrOther(options.workflowState, false);
handleExecutionFinishedWithSuccessOrOther(options.workflowState, execution.status, false);
}
setRunExecutionData(execution, runExecutionData, options.workflowState);
@@ -15,6 +15,7 @@ import type {
} from '@/features/execution/executions/executions.types';
import { useUIStore } from '@/app/stores/ui.store';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
import { getPairedItemsMapping } from '@/app/utils/pairedItemUtils';
import {
type INodeIssueData,
@@ -233,6 +234,7 @@ export function useWorkflowState() {
setActiveExecutionId(undefined);
workflowStateStore.executingNode.executingNode.length = 0;
ws.executionWaitingForWebhook = false;
useBuilderStore().resetManualExecutionStats();
}
////
@@ -2149,7 +2149,7 @@ onBeforeUnmount(() => {
v-if="builderStore.streaming"
:class="$style.thinkingPill"
show-stop
@stop="builderStore.stopStreaming"
@stop="builderStore.abortStreaming"
/>
<Suspense>
@@ -38,6 +38,7 @@ describe('API: ai', () => {
it('should call streamRequest with the correct parameters', () => {
const payload: ChatRequest.RequestPayload = {
payload: {
id: '12345',
role: 'user',
type: 'message',
text: 'Build me a workflow',
@@ -62,6 +63,7 @@ describe('API: ai', () => {
it('should pass abort signal when provided', () => {
const payload: ChatRequest.RequestPayload = {
payload: {
id: '12345',
role: 'user',
type: 'message',
text: 'Build me a workflow',
@@ -177,6 +179,7 @@ describe('API: ai', () => {
const payload: ChatRequest.RequestPayload = {
payload: {
id: '12345',
role: 'user',
type: 'message',
text: 'Build me a workflow',
@@ -211,6 +214,7 @@ describe('API: ai', () => {
const payload: ChatRequest.RequestPayload = {
payload: {
id: '12345',
role: 'user',
type: 'message',
text: 'Build me a workflow',
@@ -228,6 +232,7 @@ describe('API: ai', () => {
it('should handle complex workflow context in payload', () => {
const payload: ChatRequest.RequestPayload = {
payload: {
id: '12345',
role: 'user',
type: 'message',
text: 'Improve my workflow',
@@ -271,6 +276,7 @@ describe('API: ai', () => {
it('should handle undefined parameters correctly', () => {
const payload: ChatRequest.RequestPayload = {
payload: {
id: '12345',
role: 'user',
type: 'message',
text: 'Build me a workflow',
@@ -29,6 +29,7 @@ import { useChatPanelStateStore } from './chatPanelState.store';
import { useCredentialsStore } from '@/features/credentials/credentials.store';
import { useAIAssistantHelpers } from '@/features/ai/assistant/composables/useAIAssistantHelpers';
import type { WorkflowState } from '@/app/composables/useWorkflowState';
import { v4 as uuid } from 'uuid';
export const ENABLED_VIEWS = ASSISTANT_ENABLED_VIEWS;
const READABLE_TYPES = ['code-diff', 'text', 'block'];
@@ -555,6 +556,7 @@ export const useAssistantStore = defineStore(STORES.ASSISTANT, () => {
rootStore.restApiContext,
{
payload: {
id: uuid(),
role: 'user',
type: 'message',
text: chatMessage.text,
@@ -102,6 +102,7 @@ export namespace ChatRequest {
role: 'user';
type: 'message';
text: string;
id: string;
quickReplyType?: string;
context?: UserContext;
workflowContext?: WorkflowContext;
@@ -606,7 +606,7 @@ describe('AI Builder store', () => {
expect(builderStore.streamingAbortController).toBeInstanceOf(AbortController);
});
it('should call abort on existing controller when stopStreaming is called', () => {
it('should call abort on existing controller when abortStreaming is called', () => {
const builderStore = useBuilderStore();
// First start a request to create an abort controller
@@ -620,8 +620,8 @@ describe('AI Builder store', () => {
// Spy on the abort method
const abortSpy = vi.spyOn(controller!, 'abort');
// Call stopStreaming
builderStore.stopStreaming();
// Call abortStreaming
builderStore.abortStreaming();
// Verify abort was called
expect(abortSpy).toHaveBeenCalled();
@@ -698,7 +698,7 @@ describe('AI Builder store', () => {
expect(builderStore.streamingAbortController).toBe(firstController);
// Now properly stop streaming first
builderStore.stopStreaming();
builderStore.abortStreaming();
// Verify abort was called and controller was cleared
expect(abortSpy).toHaveBeenCalled();
@@ -1708,4 +1708,570 @@ describe('AI Builder store', () => {
expect(builderStore.creditsRemaining).toBe(80);
});
});
describe('abortStreaming telemetry', () => {
it('tracks end of response with aborted flag when aborting', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.sendChatMessage({ text: 'test' });
track.mockClear();
builderStore.abortStreaming();
expect(track).toHaveBeenCalledWith(
'End of response from builder',
expect.objectContaining({
aborted: true,
user_message_id: expect.any(String),
workflow_id: 'test-workflow-id',
}),
);
});
it('includes workflow modifications in abort telemetry', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.sendChatMessage({ text: 'test' });
track.mockClear();
builderStore.abortStreaming();
expect(track).toHaveBeenCalledWith(
'End of response from builder',
expect.objectContaining({
tools_called: expect.any(Array),
start_workflow_json: expect.any(String),
end_workflow_json: expect.any(String),
}),
);
});
it('includes todos count in abort telemetry', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.sendChatMessage({ text: 'test' });
track.mockClear();
builderStore.abortStreaming();
expect(track).toHaveBeenCalledWith(
'End of response from builder',
expect.objectContaining({
credentials_todo_count: expect.any(Number),
placeholders_todo_count: expect.any(Number),
todos: expect.any(Array),
}),
);
});
it('does not track telemetry if no streaming message in progress', () => {
const builderStore = useBuilderStore();
// Don't start any streaming
track.mockClear();
builderStore.abortStreaming();
expect(track).not.toHaveBeenCalledWith('End of response from builder', expect.anything());
});
});
describe('workflowTodos', () => {
it('returns empty array when no validation issues exist', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toEqual([]);
});
it('includes credential validation issues', () => {
workflowsStore.workflowValidationIssues = [
{ node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' },
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toContainEqual(
expect.objectContaining({ type: 'credentials' }),
);
});
it('includes placeholder issues from node parameters', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
},
},
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toContainEqual(
expect.objectContaining({ type: 'parameters', node: 'HTTP Request' }),
);
});
it('combines credential and placeholder issues', () => {
workflowsStore.workflowValidationIssues = [
{ node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' },
];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
},
},
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos.length).toBeGreaterThanOrEqual(2);
expect(builderStore.workflowTodos).toContainEqual(
expect.objectContaining({ type: 'credentials' }),
);
expect(builderStore.workflowTodos).toContainEqual(
expect.objectContaining({ type: 'parameters' }),
);
});
});
describe('placeholderIssues', () => {
it('returns empty array when nodes have no parameters', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'Start',
type: 'n8n-nodes-base.start',
typeVersion: 1,
position: [0, 0],
parameters: {},
},
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toEqual([]);
});
it('returns empty array when node has undefined parameters', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'Start',
type: 'n8n-nodes-base.start',
typeVersion: 1,
position: [0, 0],
} as Parameters<typeof workflowsStore.workflow.nodes.push>[0],
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toEqual([]);
});
it('detects placeholders in nested object parameters', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
options: {
headers: {
authorization: '<__PLACEHOLDER_VALUE__Enter API Key__>',
},
},
},
},
];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
expect(placeholderIssues).toHaveLength(1);
expect(placeholderIssues[0]).toMatchObject({
node: 'HTTP Request',
type: 'parameters',
});
});
it('detects placeholders in array parameters', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
headers: [
{ name: 'Content-Type', value: 'application/json' },
{ name: 'Authorization', value: '<__PLACEHOLDER_VALUE__Enter Token__>' },
],
},
},
];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
expect(placeholderIssues).toHaveLength(1);
expect(placeholderIssues[0]).toMatchObject({
node: 'HTTP Request',
type: 'parameters',
});
});
it('detects multiple placeholders in the same node', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
body: '<__PLACEHOLDER_VALUE__Enter Body__>',
},
},
];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
expect(placeholderIssues).toHaveLength(2);
});
it('detects placeholders across multiple nodes', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
},
},
{
id: 'node-2',
name: 'Slack',
type: 'n8n-nodes-base.slack',
typeVersion: 1,
position: [200, 0],
parameters: {
channel: '<__PLACEHOLDER_VALUE__Enter Channel__>',
},
},
];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
expect(placeholderIssues).toHaveLength(2);
expect(placeholderIssues).toContainEqual(expect.objectContaining({ node: 'HTTP Request' }));
expect(placeholderIssues).toContainEqual(expect.objectContaining({ node: 'Slack' }));
});
it('deduplicates identical placeholder issues (same node, path, and label)', () => {
workflowsStore.workflowValidationIssues = [];
// Simulate a scenario where the same placeholder appears twice
// (which shouldn't happen in practice but tests the deduplication)
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
},
},
];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
// Should only have 1 issue, not duplicates
expect(placeholderIssues).toHaveLength(1);
});
it('skips placeholder when existing parameter issue already has the same message', () => {
const placeholderLabel = 'Enter URL';
// The message format from the store uses i18n which is mocked to return the key
const expectedMessage = 'aiAssistant.builder.executeMessage.fillParameter';
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: `<__PLACEHOLDER_VALUE__${placeholderLabel}__>`,
},
issues: {
parameters: {
url: [expectedMessage],
},
},
},
];
workflowsStore.workflowValidationIssues = [];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
// Should be skipped because the message already exists
expect(placeholderIssues).toHaveLength(0);
});
it('does not skip placeholder when existing parameter issue has different message', () => {
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
},
issues: {
parameters: {
url: ['Some other validation error'],
},
},
},
];
workflowsStore.workflowValidationIssues = [];
const builderStore = useBuilderStore();
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
// Should still create the placeholder issue
expect(placeholderIssues).toHaveLength(1);
});
it('ignores non-string parameter values', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
timeout: 5000,
enabled: true,
config: null,
},
},
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toEqual([]);
});
it('ignores strings that do not match placeholder format', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: 'https://example.com',
body: 'regular string',
partial: '<__PLACEHOLDER_VALUE__missing end',
wrongPrefix: 'PLACEHOLDER__test__>',
},
},
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toEqual([]);
});
it('ignores placeholder with empty label', () => {
workflowsStore.workflowValidationIssues = [];
workflowsStore.workflow.nodes = [
{
id: 'node-1',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [0, 0],
parameters: {
url: '<__PLACEHOLDER_VALUE____>', // empty label
body: '<__PLACEHOLDER_VALUE__ __>', // whitespace-only label
},
},
];
const builderStore = useBuilderStore();
expect(builderStore.workflowTodos).toEqual([]);
});
it('filters out non-credential and non-parameter validation issues', () => {
workflowsStore.workflowValidationIssues = [
{ node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' },
{ node: 'HTTP Request', type: 'parameters', value: 'Missing parameter' },
{ node: 'HTTP Request', type: 'execution', value: 'Execution error' },
{ node: 'HTTP Request', type: 'unknown' as 'parameters', value: 'Unknown issue' },
];
workflowsStore.workflow.nodes = [];
const builderStore = useBuilderStore();
// Should only include credentials and parameters types
expect(builderStore.workflowTodos).toHaveLength(2);
expect(builderStore.workflowTodos).toContainEqual(
expect.objectContaining({ type: 'credentials' }),
);
expect(builderStore.workflowTodos).toContainEqual(
expect.objectContaining({ type: 'parameters' }),
);
});
});
describe('manual execution stats telemetry', () => {
it('should include success count in telemetry when sending message', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.incrementManualExecutionStats('success');
builderStore.sendChatMessage({ text: 'test' });
expect(track).toHaveBeenCalledWith(
'User submitted builder message',
expect.objectContaining({
manual_exec_success_count_since_prev_msg: 1,
manual_exec_error_count_since_prev_msg: 0,
}),
);
});
it('should include error count in telemetry when sending message', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.incrementManualExecutionStats('error');
builderStore.sendChatMessage({ text: 'test' });
expect(track).toHaveBeenCalledWith(
'User submitted builder message',
expect.objectContaining({
manual_exec_success_count_since_prev_msg: 0,
manual_exec_error_count_since_prev_msg: 1,
}),
);
});
it('should include multiple incremented counts in telemetry', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.incrementManualExecutionStats('success');
builderStore.incrementManualExecutionStats('success');
builderStore.incrementManualExecutionStats('error');
builderStore.sendChatMessage({ text: 'test' });
expect(track).toHaveBeenCalledWith(
'User submitted builder message',
expect.objectContaining({
manual_exec_success_count_since_prev_msg: 2,
manual_exec_error_count_since_prev_msg: 1,
}),
);
});
it('should reset stats after sending message', async () => {
const builderStore = useBuilderStore();
// First message with some stats
apiSpy.mockImplementationOnce((_ctx, _payload, onMessage, onDone) => {
onMessage({
messages: [{ type: 'message', role: 'assistant', text: 'Hello!' }],
sessionId: 'test-session',
});
onDone();
});
builderStore.incrementManualExecutionStats('success');
builderStore.incrementManualExecutionStats('error');
builderStore.sendChatMessage({ text: 'first message' });
await vi.waitFor(() => expect(builderStore.streaming).toBe(false));
// Verify first message had the stats
expect(track).toHaveBeenCalledWith(
'User submitted builder message',
expect.objectContaining({
manual_exec_success_count_since_prev_msg: 1,
manual_exec_error_count_since_prev_msg: 1,
}),
);
track.mockClear();
// Second message should have reset stats (zero counts)
apiSpy.mockImplementationOnce((_ctx, _payload, onMessage, onDone) => {
onMessage({
messages: [{ type: 'message', role: 'assistant', text: 'Hello again!' }],
sessionId: 'test-session',
});
onDone();
});
builderStore.sendChatMessage({ text: 'second message' });
await vi.waitFor(() => expect(builderStore.streaming).toBe(false));
expect(track).toHaveBeenCalledWith(
'User submitted builder message',
expect.objectContaining({
manual_exec_success_count_since_prev_msg: 0,
manual_exec_error_count_since_prev_msg: 0,
}),
);
});
it('should include zero counts when no manual executions occurred', () => {
const builderStore = useBuilderStore();
apiSpy.mockImplementationOnce(() => {});
builderStore.sendChatMessage({ text: 'test' });
expect(track).toHaveBeenCalledWith(
'User submitted builder message',
expect.objectContaining({
manual_exec_success_count_since_prev_msg: 0,
manual_exec_error_count_since_prev_msg: 0,
}),
);
});
});
});
@@ -20,10 +20,11 @@ import {
getSessionsMetadata,
} from '@/features/ai/assistant/assistant.api';
import { generateMessageId, createBuilderPayload } from './builder.utils';
import { useBuilderTodos, type TodosTrackingPayload } from './composables/useBuilderTodos';
import { useRootStore } from '@n8n/stores/useRootStore';
import type { WorkflowDataUpdate } from '@n8n/rest-api-client/api/workflows';
import pick from 'lodash/pick';
import { type INodeExecutionData, jsonParse } from 'n8n-workflow';
import { type ITelemetryTrackProperties, type INodeExecutionData, jsonParse } from 'n8n-workflow';
import { useToast } from '@/app/composables/useToast';
import { injectWorkflowState } from '@/app/composables/useWorkflowState';
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
@@ -31,10 +32,34 @@ import { useCredentialsStore } from '@/features/credentials/credentials.store';
import { getAuthTypeForNodeCredential, getMainAuthField } from '@/app/utils/nodeTypesUtils';
import { stringSizeInBytes } from '@/app/utils/typesUtils';
import { useNDVStore } from '@/features/ndv/shared/ndv.store';
import { dedupe } from 'n8n-workflow';
const INFINITE_CREDITS = -1;
export const ENABLED_VIEWS = BUILDER_ENABLED_VIEWS;
interface EndOfStreamingTrackingPayload {
userMessageId: string;
startWorkflowJson: string;
}
interface UserSubmittedBuilderMessageTrackingPayload
extends ITelemetryTrackProperties,
TodosTrackingPayload {
source: 'chat' | 'canvas';
message: string;
session_id: string;
start_workflow_json: string;
workflow_id: string;
type: 'message' | 'execution';
manual_exec_success_count_since_prev_msg: number;
manual_exec_error_count_since_prev_msg: number;
user_message_id: string;
execution_data?: string;
execution_status?: string;
error_message?: string;
error_node_type?: string;
}
export const useBuilderStore = defineStore(STORES.BUILDER, () => {
// Core state
const chatMessages = ref<ChatUI.AssistantMessage[]>([]);
@@ -45,6 +70,12 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
const creditsQuota = ref<number | undefined>();
const creditsClaimed = ref<number | undefined>();
const hasMessages = ref<boolean>(false);
const manualExecStatsInBetweenMessages = ref<{ success: number; error: number }>({
success: 0,
error: 0,
});
const currentStreamingMessage = ref<EndOfStreamingTrackingPayload | undefined>();
// Store dependencies
const settings = useSettingsStore();
@@ -70,6 +101,8 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
getRunningTools,
} = useBuilderMessages();
const { workflowTodos, getTodosToTrack } = useBuilderTodos();
const trackingSessionId = computed(() => rootStore.pushRef);
const workflowPrompt = computed(() => {
@@ -124,17 +157,74 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
initialGeneration.value = false;
}
function incrementManualExecutionStats(type: 'success' | 'error') {
manualExecStatsInBetweenMessages.value[type]++;
}
function resetManualExecutionStats() {
manualExecStatsInBetweenMessages.value = {
success: 0,
error: 0,
};
}
// Message handling functions
function addLoadingAssistantMessage(message: string) {
builderThinkingMessage.value = message;
}
function stopStreaming() {
function getWorkflowModifications({
userMessageId,
startWorkflowJson,
}: EndOfStreamingTrackingPayload) {
const newToolMessages = toolMessages.value.filter((toolMsg) =>
toolMsg.id?.startsWith(userMessageId),
);
const endWorkflowJson = getWorkflowSnapshot();
return {
tools_called: dedupe(newToolMessages.map((toolMsg) => toolMsg.toolName)),
start_workflow_json: startWorkflowJson,
end_workflow_json: endWorkflowJson,
};
}
type StopStreamingPayload =
| {
error: string;
}
| { aborted: true };
function trackEndBuilderResponse(payload?: StopStreamingPayload) {
if (!currentStreamingMessage.value) {
return;
}
const { userMessageId } = currentStreamingMessage.value;
telemetry.track('End of response from builder', {
// todo user_id available as trait already?
user_message_id: userMessageId,
workflow_id: workflowsStore.workflowId,
session_id: trackingSessionId.value,
...getWorkflowModifications(currentStreamingMessage.value),
...payload,
...getTodosToTrack(),
});
}
function stopStreaming(payload?: StopStreamingPayload) {
streaming.value = false;
if (streamingAbortController.value) {
streamingAbortController.value.abort();
streamingAbortController.value = null;
}
trackEndBuilderResponse(payload);
currentStreamingMessage.value = undefined;
}
function abortStreaming() {
stopStreaming({ aborted: true });
}
// Error handling
@@ -144,10 +234,12 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
* The retry function, if provided, will remove the error message before retrying.
* Tracks error telemetry
*/
function handleServiceError(e: unknown, id: string, retry?: () => Promise<void>) {
function handleServiceError(e: unknown, userMessageId: string, retry?: () => Promise<void>) {
assert(e instanceof Error);
stopStreaming();
stopStreaming({
error: e.message,
});
builderThinkingMessage.value = undefined;
if (e.name === 'AbortError') {
@@ -163,17 +255,11 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
const errorMessage = createErrorMessage(
locale.baseText('aiAssistant.serviceError.message', { interpolate: { message: e.message } }),
id,
userMessageId,
retry,
);
chatMessages.value = [...chatMessages.value, errorMessage];
telemetry.track('Workflow generation errored', {
error: e.message,
session_id: trackingSessionId.value,
workflow_id: workflowsStore.workflowId,
});
}
// Helper functions
@@ -203,6 +289,68 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
};
}
// Telemetry functions
/**
* Tracks when a user submits a message to the builder.
* Captures workflow state, execution data, and todo counts for analytics.
*/
function trackUserSubmittedBuilderMessage(options: {
text: string;
source: 'chat' | 'canvas';
type: 'message' | 'execution';
userMessageId: string;
currentWorkflowJson: string;
errorMessage?: string;
errorNodeType?: string;
executionStatus?: string;
}) {
const {
text,
source,
type,
userMessageId,
currentWorkflowJson,
errorMessage,
errorNodeType,
executionStatus,
} = options;
const trackingPayload: UserSubmittedBuilderMessageTrackingPayload = {
source,
message: text,
session_id: trackingSessionId.value,
start_workflow_json: currentWorkflowJson,
workflow_id: workflowsStore.workflowId,
type,
manual_exec_success_count_since_prev_msg: manualExecStatsInBetweenMessages.value.success,
manual_exec_error_count_since_prev_msg: manualExecStatsInBetweenMessages.value.error,
user_message_id: userMessageId,
...getTodosToTrack(),
};
if (type === 'execution') {
let resultData = '{}';
let resultDataSizeKb = 0;
try {
resultData = JSON.stringify(workflowsStore.workflowExecutionData ?? {});
resultDataSizeKb = stringSizeInBytes(resultData) / 1024;
} catch (error) {
// Handle circular structure errors gracefully
console.warn('Failed to stringify execution data for telemetry:', error);
}
trackingPayload.execution_data = resultDataSizeKb > 512 ? '{}' : resultData;
trackingPayload.execution_status = executionStatus ?? '';
if (executionStatus === 'error') {
trackingPayload.error_message = errorMessage ?? '';
trackingPayload.error_node_type = errorNodeType ?? '';
}
}
telemetry.track('User submitted builder message', trackingPayload);
}
// Core API functions
/**
* Sends a message to the AI builder service and handles the streaming response.
@@ -242,51 +390,38 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
if (options.initialGeneration !== undefined) {
initialGeneration.value = options.initialGeneration;
}
const messageId = generateMessageId();
const userMessageId = generateMessageId();
const currentWorkflowJson = getWorkflowSnapshot();
const trackingPayload: Record<string, string> = {
source,
message: text,
session_id: trackingSessionId.value,
start_workflow_json: currentWorkflowJson,
workflow_id: workflowsStore.workflowId,
type,
currentStreamingMessage.value = {
userMessageId,
startWorkflowJson: currentWorkflowJson,
};
if (type === 'execution') {
let resultData = '{}';
let resultDataSizeKb = 0;
trackUserSubmittedBuilderMessage({
text,
source,
type,
userMessageId,
currentWorkflowJson,
errorMessage,
errorNodeType,
executionStatus,
});
try {
resultData = JSON.stringify(workflowsStore.workflowExecutionData ?? {});
resultDataSizeKb = stringSizeInBytes(resultData) / 1024;
} catch (error) {
// Handle circular structure errors gracefully
console.warn('Failed to stringify execution data for telemetry:', error);
}
resetManualExecutionStats();
trackingPayload.execution_data = resultDataSizeKb > 512 ? '{}' : resultData;
trackingPayload.execution_status = executionStatus ?? '';
if (executionStatus === 'error') {
trackingPayload.error_message = errorMessage ?? '';
trackingPayload.error_node_type = errorNodeType ?? '';
}
}
telemetry.track('User submitted builder message', trackingPayload);
prepareForStreaming(text, messageId);
prepareForStreaming(text, userMessageId);
const executionResult = workflowsStore.workflowExecutionData?.data?.resultData;
const payload = createBuilderPayload(text, {
const payload = createBuilderPayload(text, userMessageId, {
quickReplyType,
workflow: workflowsStore.workflow,
executionData: executionResult,
nodesForSchema: Object.keys(workflowsStore.nodesByName),
});
const retry = createRetryHandler(messageId, async () => sendChatMessage(options));
const retry = createRetryHandler(userMessageId, async () => sendChatMessage(options));
// Abort previous streaming request if any
if (streamingAbortController.value) {
@@ -302,7 +437,7 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
const result = processAssistantMessages(
chatMessages.value,
response.messages,
generateMessageId(),
userMessageId,
retry,
);
chatMessages.value = result.messages;
@@ -315,11 +450,11 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
}
},
() => stopStreaming(),
(e) => handleServiceError(e, messageId, retry),
(e) => handleServiceError(e, userMessageId, retry),
streamingAbortController.value?.signal,
);
} catch (e: unknown) {
handleServiceError(e, messageId, retry);
handleServiceError(e, userMessageId, retry);
}
}
@@ -574,9 +709,10 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
creditsRemaining,
hasNoCreditsRemaining,
hasMessages: computed(() => hasMessages.value),
workflowTodos,
// Methods
stopStreaming,
abortStreaming,
resetBuilderChat,
sendChatMessage,
loadSessions,
@@ -586,5 +722,7 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
updateBuilderCredits,
getRunningTools,
fetchSessionsMetadata,
incrementManualExecutionStats,
resetManualExecutionStats,
};
});
@@ -8,12 +8,17 @@ import {
import type { IRunExecutionData } from 'n8n-workflow';
import type { IWorkflowDb } from '@/Interface';
export function generateShortId() {
return Math.random().toString(36).substr(2, 9);
}
export function generateMessageId(): string {
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
return `${Date.now()}-${generateShortId()}`;
}
export function createBuilderPayload(
text: string,
id: string,
options: {
quickReplyType?: string;
executionData?: IRunExecutionData['resultData'];
@@ -67,6 +72,7 @@ export function createBuilderPayload(
return {
role: 'user',
type: 'message',
id,
text,
quickReplyType: options.quickReplyType,
workflowContext,
@@ -1354,249 +1354,6 @@ describe('AskAssistantBuild', () => {
});
});
it('should track categorization telemetry when categorize_prompt tool completes', async () => {
renderComponent();
// Simulate streaming starts
builderStore.$patch({ streaming: true });
await flushPromises();
// Add categorization tool message
builderStore.toolMessages = [
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'categorize_prompt',
toolCallId: faker.string.uuid(),
status: 'completed',
updates: [
{
type: 'output',
data: {
categorization: {
techniques: ['chatbot', 'notification'],
confidence: 0.85,
},
},
},
],
},
];
// Simulate streaming stops (this triggers trackWorkflowModifications)
builderStore.$patch({ streaming: false });
await flushPromises();
expect(trackMock).toHaveBeenCalledWith('Classifier labels user prompt', {
user_id: undefined,
workflow_id: 'abc123',
classifier_labels: ['chatbot', 'notification'],
confidence: 0.85,
session_id: 'app_session_id',
timestamp: expect.any(String),
});
});
it('should handle multiple canvas generations correctly', async () => {
const originalWorkflow = {
nodes: [],
connections: {},
};
builderStore.getWorkflowSnapshot.mockReturnValue(JSON.stringify(originalWorkflow));
const intermediaryWorkflow = {
nodes: [
{
id: 'node1',
name: 'Start',
type: 'n8n-nodes-base.start',
position: [0, 0],
typeVersion: 1,
parameters: {},
} as INodeUi,
],
connections: {},
};
const finalWorkflow = {
nodes: [
{
id: 'node1',
name: 'Start',
type: 'n8n-nodes-base.start',
position: [0, 0],
typeVersion: 1,
parameters: {},
} as INodeUi,
{
id: 'node2',
name: 'HttpReuqest',
type: 'n8n-nodes-base.httpRequest',
position: [0, 0],
typeVersion: 1,
parameters: {},
} as INodeUi,
],
connections: {},
};
workflowsStore.$patch({
workflow: originalWorkflow,
});
renderComponent();
builderStore.$patch({ streaming: true });
await flushPromises();
// Trigger the watcher by updating workflowMessages
builderStore.workflowMessages = [
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'workflow-updated' as const,
codeSnippet: JSON.stringify(intermediaryWorkflow),
},
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'workflow-updated' as const,
codeSnippet: JSON.stringify(finalWorkflow),
},
];
const toolCallId1_1 = '1234';
const toolCallId1_2 = '3333';
const toolCallId2 = '4567';
const toolCallId3 = '8901';
builderStore.toolMessages = [
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'first-tool',
toolCallId: toolCallId1_1,
status: 'completed',
updates: [],
},
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'first-tool',
toolCallId: toolCallId1_2,
status: 'completed',
updates: [],
},
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'second-tool',
toolCallId: toolCallId2,
status: 'running',
updates: [],
},
];
builderStore.$patch({ streaming: false });
await flushPromises();
expect(trackMock).toHaveBeenCalledOnce();
expect(trackMock).toHaveBeenCalledWith('Workflow modified by builder', {
end_workflow_json: JSON.stringify(finalWorkflow),
session_id: 'app_session_id',
start_workflow_json: JSON.stringify(originalWorkflow),
// first-tool is added once, even though it completed twice
// second-tool is ignored because it's running
tools_called: ['first-tool'],
workflow_id: 'abc123',
});
trackMock.mockClear();
builderStore.$patch({ streaming: true });
await flushPromises();
// second run after new messages
const updatedWorkflow2 = {
...finalWorkflow,
nodes: [
...finalWorkflow.nodes,
{
id: 'node1',
name: 'Updated',
type: 'n8n-nodes-base.updated',
position: [0, 0],
typeVersion: 1,
parameters: {},
},
],
};
builderStore.workflowMessages = [
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'workflow-updated' as const,
codeSnippet: JSON.stringify(updatedWorkflow2),
},
];
builderStore.toolMessages = [
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'first-tool',
toolCallId: toolCallId1_1,
status: 'completed',
updates: [],
},
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'first-tool',
toolCallId: toolCallId1_2,
status: 'completed',
updates: [],
},
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'second-tool',
toolCallId: toolCallId2,
status: 'completed',
updates: [],
},
{
id: faker.string.uuid(),
role: 'assistant' as const,
type: 'tool' as const,
toolName: 'third-tool',
toolCallId: toolCallId3,
status: 'completed',
updates: [],
},
];
builderStore.$patch({ streaming: false });
await flushPromises();
expect(trackMock).toHaveBeenCalledOnce();
expect(trackMock).toHaveBeenCalledWith('Workflow modified by builder', {
end_workflow_json: JSON.stringify(updatedWorkflow2),
session_id: 'app_session_id',
start_workflow_json: JSON.stringify(originalWorkflow),
// first-tool is ignored, because it was tracked in first run (same tool call id)
tools_called: ['second-tool', 'third-tool'],
workflow_id: 'abc123',
});
});
describe('shouldTidyUp logic', () => {
it('should set tidyUp to true when new nodes are added', async () => {
const originalWorkflow = { nodes: [], connections: {} };
@@ -33,9 +33,6 @@ const { goToUpgrade } = usePageRedirectionHelper();
// Track processed workflow updates
const processedWorkflowUpdates = ref(new Set<string>());
const trackedTools = ref(new Set<string>());
const trackedCategorizations = ref(new Set<string>());
const workflowUpdated = ref<{ start: string; end: string } | undefined>();
const shouldTidyUp = ref(false);
const n8nChatRef = ref<InstanceType<typeof N8nAskAssistantChat>>();
@@ -107,9 +104,6 @@ async function onUserMessage(content: string) {
function onNewWorkflow() {
builderStore.resetBuilderChat();
processedWorkflowUpdates.value.clear();
trackedTools.value.clear();
trackedCategorizations.value.clear();
workflowUpdated.value = undefined;
shouldTidyUp.value = false;
}
@@ -130,71 +124,6 @@ function onFeedback(feedback: RatingFeedback) {
}
}
function dedupeToolNames(toolNames: string[]): string[] {
return [...new Set(toolNames)];
}
function isCategorizationData(
data: unknown,
): data is { techniques: string[]; confidence?: number } {
return (
typeof data === 'object' &&
data !== null &&
'techniques' in data &&
Array.isArray(data.techniques) &&
data.techniques.every((t) => typeof t === 'string')
);
}
function trackWorkflowCategorization() {
// Track categorization telemetry
builderStore.toolMessages.forEach((toolMsg) => {
if (toolMsg.toolName !== 'categorize_prompt') return;
if (toolMsg.status !== 'completed') return;
if (!toolMsg.toolCallId) return;
if (trackedCategorizations.value.has(toolMsg.toolCallId)) return;
const outputUpdate = toolMsg.updates.find((u) => u.type === 'output');
const categorizationData = outputUpdate?.data?.categorization;
if (!isCategorizationData(categorizationData)) return;
trackedCategorizations.value.add(toolMsg.toolCallId);
telemetry.track('Classifier labels user prompt', {
user_id: usersStore.currentUserId ?? undefined,
workflow_id: workflowsStore.workflowId,
classifier_labels: categorizationData.techniques,
confidence: categorizationData.confidence,
session_id: builderStore.trackingSessionId,
timestamp: new Date().toISOString(),
});
});
}
function trackWorkflowModifications() {
if (workflowUpdated.value) {
// Track tool usage for telemetry
const newToolMessages = builderStore.toolMessages.filter(
(toolMsg) =>
toolMsg.status !== 'running' &&
toolMsg.toolCallId &&
!trackedTools.value.has(toolMsg.toolCallId),
);
newToolMessages.forEach((toolMsg) => trackedTools.value.add(toolMsg.toolCallId ?? ''));
telemetry.track('Workflow modified by builder', {
tools_called: dedupeToolNames(newToolMessages.map((toolMsg) => toolMsg.toolName)),
session_id: builderStore.trackingSessionId,
start_workflow_json: workflowUpdated.value.start,
end_workflow_json: workflowUpdated.value.end,
workflow_id: workflowsStore.workflowId,
});
workflowUpdated.value = undefined;
}
}
function onWorkflowExecuted() {
const executionData = workflowsStore.workflowExecutionData;
const executionStatus = executionData?.status ?? 'unknown';
@@ -257,8 +186,6 @@ watch(
if (msg.id && isWorkflowUpdatedMessage(msg)) {
processedWorkflowUpdates.value.add(msg.id);
const originalWorkflowJson =
workflowUpdated.value?.start ?? builderStore.getWorkflowSnapshot();
const result = builderStore.applyWorkflowUpdate(msg.codeSnippet);
if (result.success) {
@@ -274,11 +201,6 @@ watch(
regenerateIds: false,
trackEvents: false,
});
workflowUpdated.value = {
start: originalWorkflowJson,
end: msg.codeSnippet,
};
}
}
});
@@ -291,11 +213,6 @@ watch(
watch(
() => builderStore.streaming,
async (isStreaming) => {
if (!isStreaming) {
trackWorkflowModifications();
trackWorkflowCategorization();
}
if (
builderStore.initialGeneration &&
!isStreaming &&
@@ -348,7 +265,7 @@ defineExpose({
@message="onUserMessage"
@upgrade-click="() => goToUpgrade('ai-builder-sidebar', 'upgrade-builder')"
@feedback="onFeedback"
@stop="builderStore.stopStreaming"
@stop="builderStore.abortStreaming"
>
<template #header>
<slot name="header" />
@@ -16,7 +16,6 @@ import { useToast } from '@/app/composables/useToast';
import { N8nTooltip } from '@n8n/design-system';
import { nextTick } from 'vue';
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
import type { WorkflowValidationIssue } from '@/Interface';
interface Emits {
/** Emitted when workflow execution completes */
@@ -37,49 +36,6 @@ const builderStore = useBuilderStore();
// Workflow execution composable
const { runWorkflow } = useRunWorkflow({ router });
const PLACEHOLDER_PREFIX = '<__PLACEHOLDER_VALUE__';
const PLACEHOLDER_SUFFIX = '__>';
interface PlaceholderDetail {
path: string[];
label: string;
}
function extractPlaceholderLabel(value: unknown): string | null {
if (typeof value !== 'string') return null;
if (!value.startsWith(PLACEHOLDER_PREFIX) || !value.endsWith(PLACEHOLDER_SUFFIX)) return null;
const label = value
.slice(PLACEHOLDER_PREFIX.length, value.length - PLACEHOLDER_SUFFIX.length)
.trim();
return label.length > 0 ? label : null;
}
function findPlaceholderDetails(value: unknown, path: string[] = []): PlaceholderDetail[] {
const label = extractPlaceholderLabel(value);
if (label) return [{ path, label }];
if (Array.isArray(value)) {
return value.flatMap((item, index) => findPlaceholderDetails(item, [...path, `[${index}]`]));
}
if (value !== null && typeof value === 'object') {
return Object.entries(value).flatMap(([key, nested]) =>
findPlaceholderDetails(nested, [...path, key]),
);
}
return [];
}
function formatPlaceholderPath(path: string[]): string {
if (path.length === 0) return 'parameters';
return path
.map((segment, index) => (segment.startsWith('[') || index === 0 ? segment : `.${segment}`))
.join('');
}
let executionWatcherStop: WatchStopHandle | undefined;
const containerRef = ref<HTMLElement>();
@@ -115,56 +71,7 @@ const ensureExecutionWatcher = () => {
);
};
// Workflow validation from store
const baseWorkflowIssues = computed(() =>
workflowsStore.workflowValidationIssues.filter((issue) =>
['credentials', 'parameters'].includes(issue.type),
),
);
const placeholderIssues = computed(() => {
const issues: WorkflowValidationIssue[] = [];
const seen = new Set<string>();
for (const node of workflowsStore.workflow.nodes) {
if (!node?.parameters) continue;
const placeholders = findPlaceholderDetails(node.parameters);
if (placeholders.length === 0) continue;
const existingParameterIssues = node.issues?.parameters ?? {};
for (const placeholder of placeholders) {
const path = formatPlaceholderPath(placeholder.path);
const message = i18n.baseText('aiAssistant.builder.executeMessage.fillParameter', {
interpolate: { label: placeholder.label },
});
const rawMessages = existingParameterIssues[path];
const existingMessages = rawMessages
? Array.isArray(rawMessages)
? rawMessages
: [rawMessages]
: [];
if (existingMessages.includes(message)) continue;
const key = `${node.name}|${path}|${placeholder.label}`;
if (seen.has(key)) continue;
seen.add(key);
issues.push({
node: node.name,
type: 'parameters',
value: message,
});
}
}
return issues;
});
const workflowIssues = computed(() => [...baseWorkflowIssues.value, ...placeholderIssues.value]);
const hasValidationIssues = computed(() => workflowIssues.value.length > 0);
const hasValidationIssues = computed(() => builderStore.workflowTodos.length > 0);
const formatIssueMessage = workflowsStore.formatIssueMessage;
const triggerNodes = computed(() =>
@@ -261,7 +168,7 @@ onBeforeUnmount(() => {
aria-label="Workflow validation issues"
>
<NodeIssueItem
v-for="issue in workflowIssues"
v-for="issue in builderStore.workflowTodos"
:key="`${formatIssueMessage(issue.value)}_${issue.node}`"
:issue="issue"
:get-node-type="getNodeTypeByName"
@@ -10,6 +10,11 @@ vi.mock('@n8n/i18n', () => ({
}),
}));
// Mock generateShortId to return a predictable value for testing
vi.mock('../builder.utils', () => ({
generateShortId: () => 'abc123',
}));
describe('useBuilderMessages', () => {
let builderMessages: ReturnType<typeof useBuilderMessages>;
@@ -36,7 +41,7 @@ describe('useBuilderMessages', () => {
expect(result.messages).toHaveLength(1);
expect(result.messages[0]).toMatchObject({
id: 'test-id-0',
id: 'test-id--abc123--0',
role: 'assistant',
type: 'text',
content: 'Hello, how can I help?',
@@ -72,7 +77,7 @@ describe('useBuilderMessages', () => {
expect(result.messages).toHaveLength(1);
const toolMessage = result.messages[0] as ChatUI.ToolMessage;
expect(toolMessage).toMatchObject({
id: 'call-123', // Should use toolCallId as ID
id: 'test-id--abc123--0-call-123', // Format is messageId-toolCallId
role: 'assistant',
type: 'tool',
toolName: 'add_nodes',
@@ -164,7 +169,7 @@ describe('useBuilderMessages', () => {
expect(result.messages).toHaveLength(1);
expect(result.messages[0]).toMatchObject({
id: 'test-id-0', // Should fall back to generated ID
id: 'test-id--abc123--0-undefined', // Format is messageId-toolCallId (undefined when missing)
type: 'tool',
toolName: 'some_tool',
});
@@ -224,7 +229,7 @@ describe('useBuilderMessages', () => {
expect(result.messages).toHaveLength(1);
expect(result.messages[0]).toMatchObject({
id: 'test-id-0',
id: 'test-id--abc123--0',
type: 'workflow-updated',
codeSnippet: '{"nodes": [], "connections": {}}',
read: false,
@@ -262,11 +267,11 @@ describe('useBuilderMessages', () => {
expect(result.messages).toHaveLength(3);
expect(result.messages[0].type).toBe('text');
expect(result.messages[0].id).toBe('batch-id-0');
expect(result.messages[0].id).toBe('batch-id--abc123--0');
expect(result.messages[1].type).toBe('tool');
expect(result.messages[1].id).toBe('call-123'); // Uses toolCallId
expect(result.messages[1].id).toBe('batch-id--abc123--1-call-123'); // Format is messageId-toolCallId
expect(result.messages[2].type).toBe('workflow-updated');
expect(result.messages[2].id).toBe('batch-id-2');
expect(result.messages[2].id).toBe('batch-id--abc123--2');
});
it('should show tool name when tool is in progress with displayTitle', () => {
@@ -988,7 +993,7 @@ describe('useBuilderMessages', () => {
);
expect(result.messages).toHaveLength(1);
expect(result.messages[0].id).toBe('test-id-0');
expect(result.messages[0].id).toBe('test-id--abc123--0-undefined'); // Format is messageId-toolCallId (undefined when missing)
});
it('should handle workflow-updated messages with invalid JSON', () => {
@@ -1526,7 +1531,7 @@ describe('useBuilderMessages', () => {
expect(result.messages).toHaveLength(1);
const errorMessage = result.messages[0] as ChatUI.ErrorMessage;
expect(errorMessage).toMatchObject({
id: 'test-id-0',
id: 'test-id--abc123--0',
role: 'assistant',
type: 'error',
content: 'Something went wrong',
@@ -2,6 +2,7 @@ import type { ChatUI } from '@n8n/design-system/types/assistant';
import type { ChatRequest } from '../assistant.types';
import { useI18n } from '@n8n/i18n';
import { isTextMessage, isWorkflowUpdatedMessage, isToolMessage } from '../assistant.types';
import { generateShortId } from '../builder.utils';
export interface MessageProcessingResult {
messages: ChatUI.AssistantMessage[];
@@ -98,9 +99,6 @@ export function useBuilderMessages() {
msg: ChatRequest.ToolMessage,
messageId: string,
): void {
// Use toolCallId as the message ID for consistency across updates
const toolMessageId = msg.toolCallId ?? messageId;
// Check if we already have this tool message
const existingIndex = msg.toolCallId
? messages.findIndex((m) => m.type === 'tool' && m.toolCallId === msg.toolCallId)
@@ -111,6 +109,7 @@ export function useBuilderMessages() {
const existing = messages[existingIndex] as ChatUI.ToolMessage;
const toolMessage: ChatUI.ToolMessage = {
...existing,
id: `${messageId}-${msg.toolCallId}`,
status: msg.status,
updates: [...(existing.updates || []), ...(msg.updates || [])],
};
@@ -118,7 +117,7 @@ export function useBuilderMessages() {
} else {
// Add new tool message
const toolMessage: ChatUI.AssistantMessage = {
id: toolMessageId,
id: `${messageId}-${msg.toolCallId}`,
role: 'assistant',
type: 'tool',
toolName: msg.toolName,
@@ -269,15 +268,18 @@ export function useBuilderMessages() {
function processAssistantMessages(
currentMessages: ChatUI.AssistantMessage[],
newMessages: ChatRequest.MessageResponse[],
baseId: string,
userMessageId: string,
retry?: () => Promise<void>,
): MessageProcessingResult {
const mutableMessages = [...currentMessages];
let shouldClearThinking = false;
const messageGroupId = generateShortId();
newMessages.forEach((msg, index) => {
// Generate unique ID for each message in the batch
const messageId = `${baseId}-${index}`;
// Generate unique ID for each message in the batch, based on original user message id.
// Used in telemetry to track events related to a specific user message
const messageId = `${userMessageId}--${messageGroupId}--${index}`;
const clearThinking = processSingleMessage(mutableMessages, msg, messageId, retry);
shouldClearThinking = shouldClearThinking || clearThinking;
});
@@ -0,0 +1,183 @@
import { describe, it, expect } from 'vitest';
import {
extractPlaceholderLabel,
findPlaceholderDetails,
formatPlaceholderPath,
} from './useBuilderTodos';
describe('useBuilderTodos', () => {
describe('extractPlaceholderLabel', () => {
it('returns null for non-string values', () => {
expect(extractPlaceholderLabel(123)).toBeNull();
expect(extractPlaceholderLabel(true)).toBeNull();
expect(extractPlaceholderLabel(null)).toBeNull();
expect(extractPlaceholderLabel(undefined)).toBeNull();
expect(extractPlaceholderLabel({})).toBeNull();
expect(extractPlaceholderLabel([])).toBeNull();
});
it('returns null for strings without placeholder format', () => {
expect(extractPlaceholderLabel('regular string')).toBeNull();
expect(extractPlaceholderLabel('https://example.com')).toBeNull();
expect(extractPlaceholderLabel('')).toBeNull();
});
it('returns null for partial placeholder format', () => {
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__missing end')).toBeNull();
expect(extractPlaceholderLabel('PLACEHOLDER__test__>')).toBeNull();
expect(extractPlaceholderLabel('__PLACEHOLDER_VALUE__test__>')).toBeNull();
});
it('returns null for empty label', () => {
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE____>')).toBeNull();
});
it('returns null for whitespace-only label', () => {
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__ __>')).toBeNull();
});
it('extracts label from valid placeholder', () => {
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__Enter URL__>')).toBe('Enter URL');
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__API Key__>')).toBe('API Key');
});
it('trims whitespace from label', () => {
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__ Enter URL __>')).toBe('Enter URL');
});
});
describe('findPlaceholderDetails', () => {
it('returns empty array for primitive non-placeholder values', () => {
expect(findPlaceholderDetails('regular string')).toEqual([]);
expect(findPlaceholderDetails(123)).toEqual([]);
expect(findPlaceholderDetails(true)).toEqual([]);
expect(findPlaceholderDetails(null)).toEqual([]);
});
it('returns empty array for empty object', () => {
expect(findPlaceholderDetails({})).toEqual([]);
});
it('returns empty array for empty array', () => {
expect(findPlaceholderDetails([])).toEqual([]);
});
it('finds placeholder at root level', () => {
const result = findPlaceholderDetails('<__PLACEHOLDER_VALUE__Enter URL__>');
expect(result).toEqual([{ path: [], label: 'Enter URL' }]);
});
it('finds placeholder in simple object', () => {
const result = findPlaceholderDetails({
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
});
expect(result).toEqual([{ path: ['url'], label: 'Enter URL' }]);
});
it('finds multiple placeholders in object', () => {
const result = findPlaceholderDetails({
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
body: '<__PLACEHOLDER_VALUE__Enter Body__>',
});
expect(result).toHaveLength(2);
expect(result).toContainEqual({ path: ['url'], label: 'Enter URL' });
expect(result).toContainEqual({ path: ['body'], label: 'Enter Body' });
});
it('finds placeholder in nested object', () => {
const result = findPlaceholderDetails({
options: {
headers: {
authorization: '<__PLACEHOLDER_VALUE__Enter API Key__>',
},
},
});
expect(result).toEqual([
{ path: ['options', 'headers', 'authorization'], label: 'Enter API Key' },
]);
});
it('finds placeholder in array', () => {
const result = findPlaceholderDetails([
'regular value',
'<__PLACEHOLDER_VALUE__Enter Value__>',
]);
expect(result).toEqual([{ path: ['[1]'], label: 'Enter Value' }]);
});
it('finds placeholder in array of objects', () => {
const result = findPlaceholderDetails({
headers: [
{ name: 'Content-Type', value: 'application/json' },
{ name: 'Authorization', value: '<__PLACEHOLDER_VALUE__Enter Token__>' },
],
});
expect(result).toEqual([{ path: ['headers', '[1]', 'value'], label: 'Enter Token' }]);
});
it('finds placeholders in mixed structure', () => {
const result = findPlaceholderDetails({
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
options: {
items: [{ key: '<__PLACEHOLDER_VALUE__Enter Key__>' }, { value: 'static' }],
},
});
expect(result).toHaveLength(2);
expect(result).toContainEqual({ path: ['url'], label: 'Enter URL' });
expect(result).toContainEqual({
path: ['options', 'items', '[0]', 'key'],
label: 'Enter Key',
});
});
it('ignores non-placeholder strings in object', () => {
const result = findPlaceholderDetails({
url: 'https://example.com',
method: 'GET',
placeholder: '<__PLACEHOLDER_VALUE__Enter Value__>',
});
expect(result).toEqual([{ path: ['placeholder'], label: 'Enter Value' }]);
});
it('handles custom starting path', () => {
const result = findPlaceholderDetails({ url: '<__PLACEHOLDER_VALUE__Enter URL__>' }, [
'parameters',
]);
expect(result).toEqual([{ path: ['parameters', 'url'], label: 'Enter URL' }]);
});
});
describe('formatPlaceholderPath', () => {
it('returns "parameters" for empty path', () => {
expect(formatPlaceholderPath([])).toBe('parameters');
});
it('formats single segment path', () => {
expect(formatPlaceholderPath(['url'])).toBe('url');
});
it('formats multi-segment path with dot notation', () => {
expect(formatPlaceholderPath(['options', 'headers', 'authorization'])).toBe(
'options.headers.authorization',
);
});
it('formats path with array indices without leading dot', () => {
expect(formatPlaceholderPath(['headers', '[0]', 'value'])).toBe('headers[0].value');
});
it('formats path starting with array index', () => {
expect(formatPlaceholderPath(['[0]', 'key'])).toBe('[0].key');
});
it('formats path with multiple array indices', () => {
expect(formatPlaceholderPath(['items', '[0]', 'options', '[1]', 'value'])).toBe(
'items[0].options[1].value',
);
});
it('formats path with consecutive array indices', () => {
expect(formatPlaceholderPath(['matrix', '[0]', '[1]'])).toBe('matrix[0][1]');
});
});
});
@@ -0,0 +1,164 @@
import { computed } from 'vue';
import { useI18n } from '@n8n/i18n';
import { useWorkflowsStore } from '@/app/stores/workflows.store';
import type { WorkflowValidationIssue } from '@/Interface';
const PLACEHOLDER_PREFIX = '<__PLACEHOLDER_VALUE__';
const PLACEHOLDER_SUFFIX = '__>';
export interface PlaceholderDetail {
path: string[];
label: string;
}
export interface TodoTrackingItem {
type: string;
node_type: string | undefined;
label: string | string[];
}
export interface TodosTrackingPayload {
credentials_todo_count: number;
placeholders_todo_count: number;
todos: TodoTrackingItem[];
}
/**
* Extracts the label from a placeholder value string.
* Returns null if the value is not a valid placeholder.
*/
export function extractPlaceholderLabel(value: unknown): string | null {
if (typeof value !== 'string') return null;
if (!value.startsWith(PLACEHOLDER_PREFIX) || !value.endsWith(PLACEHOLDER_SUFFIX)) return null;
const label = value
.slice(PLACEHOLDER_PREFIX.length, value.length - PLACEHOLDER_SUFFIX.length)
.trim();
return label.length > 0 ? label : null;
}
/**
* Recursively searches through a value (object, array, or primitive) to find
* all placeholder values and their paths.
*/
export function findPlaceholderDetails(value: unknown, path: string[] = []): PlaceholderDetail[] {
const label = extractPlaceholderLabel(value);
if (label) return [{ path, label }];
if (Array.isArray(value)) {
return value.flatMap((item, index) => findPlaceholderDetails(item, [...path, `[${index}]`]));
}
if (value !== null && typeof value === 'object') {
return Object.entries(value).flatMap(([key, nested]) =>
findPlaceholderDetails(nested, [...path, key]),
);
}
return [];
}
/**
* Formats a path array into a dot-notation string for display.
* Array indices are preserved as [N] without leading dots.
*/
export function formatPlaceholderPath(path: string[]): string {
if (path.length === 0) return 'parameters';
return path
.map((segment, index) => (segment.startsWith('[') || index === 0 ? segment : `.${segment}`))
.join('');
}
/**
* Composable for managing workflow todos (validation issues and placeholders)
* used by the AI builder.
*/
export function useBuilderTodos() {
const workflowsStore = useWorkflowsStore();
const locale = useI18n();
/**
* Base workflow validation issues filtered to only credentials and parameters types.
*/
const baseWorkflowIssues = computed(() =>
workflowsStore.workflowValidationIssues.filter((issue) =>
['credentials', 'parameters'].includes(issue.type),
),
);
/**
* Placeholder issues detected in workflow node parameters.
* These are values with the format <__PLACEHOLDER_VALUE__label__>.
*/
const placeholderIssues = computed(() => {
const issues: WorkflowValidationIssue[] = [];
const seen = new Set<string>();
for (const node of workflowsStore.workflow.nodes) {
if (!node?.parameters) continue;
const placeholders = findPlaceholderDetails(node.parameters);
if (placeholders.length === 0) continue;
const existingParameterIssues = node.issues?.parameters ?? {};
for (const placeholder of placeholders) {
const path = formatPlaceholderPath(placeholder.path);
const message = locale.baseText('aiAssistant.builder.executeMessage.fillParameter', {
interpolate: { label: placeholder.label },
});
const rawMessages = existingParameterIssues[path];
const existingMessages = rawMessages
? Array.isArray(rawMessages)
? rawMessages
: [rawMessages]
: [];
if (existingMessages.includes(message)) continue;
const key = `${node.name}|${path}|${placeholder.label}`;
if (seen.has(key)) continue;
seen.add(key);
issues.push({
node: node.name,
type: 'parameters',
value: message,
});
}
}
return issues;
});
/**
* Combined list of all workflow todos (base issues + placeholder issues).
*/
const workflowTodos = computed(() => [...baseWorkflowIssues.value, ...placeholderIssues.value]);
/**
* Returns todos data formatted for telemetry tracking.
*/
function getTodosToTrack(): TodosTrackingPayload {
const credentials_todo_count = workflowsStore.workflowValidationIssues.filter(
(issue) => issue.type === 'credentials',
).length;
const placeholders_todo_count = placeholderIssues.value.length;
return {
credentials_todo_count,
placeholders_todo_count,
todos: workflowTodos.value.map((todo) => ({
type: todo.type,
node_type: workflowsStore.getNodeByName(todo.node)?.type,
label: todo.value,
})),
};
}
return {
workflowTodos,
placeholderIssues,
getTodosToTrack,
};
}
+1
View File
@@ -51,6 +51,7 @@ export {
setSafeObjectProperty,
isDomainAllowed,
isCommunityPackageName,
dedupe,
} from './utils';
export {
isINodeProperties,
+4
View File
@@ -438,3 +438,7 @@ export function isCommunityPackageName(packageName: string): boolean {
return !!nameMatch;
}
export function dedupe<T>(arr: T[]): T[] {
return [...new Set(arr)];
}
+1 -4
View File
@@ -42,10 +42,7 @@ import * as NodeHelpers from './node-helpers';
import { renameFormFields } from './node-parameters/rename-node-utils';
import { applyAccessPatterns } from './node-reference-parser-utils';
import * as ObservableObject from './observable-object';
function dedupe<T>(arr: T[]): T[] {
return [...new Set(arr)];
}
import { dedupe } from './utils';
export interface WorkflowParameters {
id?: string;
+7 -7
View File
@@ -22,8 +22,8 @@ catalogs:
specifier: 0.3.20-15
version: 0.3.20-15
'@n8n_io/ai-assistant-sdk':
specifier: 1.17.0
version: 1.17.0
specifier: 1.18.0
version: 1.18.0
'@sentry/node':
specifier: ^9.42.1
version: 9.42.1
@@ -396,7 +396,7 @@ importers:
version: link:../utils
'@n8n_io/ai-assistant-sdk':
specifier: 'catalog:'
version: 1.17.0
version: 1.18.0
csv-parse:
specifier: 5.5.0
version: 5.5.0
@@ -1534,7 +1534,7 @@ importers:
version: link:../@n8n/utils
'@n8n_io/ai-assistant-sdk':
specifier: 'catalog:'
version: 1.17.0
version: 1.18.0
'@n8n_io/license-sdk':
specifier: 2.24.1
version: 2.24.1
@@ -6339,8 +6339,8 @@ packages:
engines: {node: '>=18.10', pnpm: '>=9.6'}
hasBin: true
'@n8n_io/ai-assistant-sdk@1.17.0':
resolution: {integrity: sha512-Zwfgf9N4aK9klCVC15xHL8R5ID8h9f6OAlW6fPJRV00cmBjX2gD8ZYaX92A9iGiKpmW5YG3mxPU7XTFVexB7wQ==}
'@n8n_io/ai-assistant-sdk@1.18.0':
resolution: {integrity: sha512-lWsGJ5AniMd/s6FqtQuCS5YJ/iWIA53GjdLieQWgpjwmc52zOAIhfpPGbI5tmm0kpvExYpGDvVRH8UzOHlt9QA==}
engines: {node: '>=20.15', pnpm: '>=8.14'}
'@n8n_io/license-sdk@2.24.1':
@@ -22728,7 +22728,7 @@ snapshots:
acorn: 8.12.1
acorn-walk: 8.3.4
'@n8n_io/ai-assistant-sdk@1.17.0': {}
'@n8n_io/ai-assistant-sdk@1.18.0': {}
'@n8n_io/license-sdk@2.24.1':
dependencies:
+1 -1
View File
@@ -11,7 +11,7 @@ catalog:
'@langchain/core': 1.1.0
'@langchain/openai': 1.1.3
'@n8n/typeorm': 0.3.20-15
'@n8n_io/ai-assistant-sdk': 1.17.0
'@n8n_io/ai-assistant-sdk': 1.18.0
'@sentry/node': ^9.42.1
'@types/basic-auth': ^1.1.3
'@types/express': ^5.0.1