mirror of
https://github.com/n8n-io/n8n.git
synced 2026-09-24 23:22:38 +08:00
feat(ai-builder): Consolidate builder telemetry (#22644)
This commit is contained in:
@@ -99,7 +99,10 @@ export async function runSingleTest(
|
||||
// Generate workflow
|
||||
const startTime = Date.now();
|
||||
await consumeGenerator(
|
||||
agent.chat(getChatPayload(testCase.prompt, testCase.id, opts?.featureFlags), userId),
|
||||
agent.chat(
|
||||
getChatPayload('single-eval', testCase.prompt, testCase.id, opts?.featureFlags),
|
||||
userId,
|
||||
),
|
||||
);
|
||||
const generationTime = Date.now() - startTime;
|
||||
|
||||
|
||||
@@ -356,7 +356,10 @@ async function runSingleGeneration(
|
||||
|
||||
// Generate workflow
|
||||
await consumeGenerator(
|
||||
agent.chat(getChatPayload(inputs.prompt, runId, featureFlags), `pairwise-gen-${generationIndex}`),
|
||||
agent.chat(
|
||||
getChatPayload('pairewise-gen', inputs.prompt, runId, featureFlags),
|
||||
`pairwise-gen-${generationIndex}`,
|
||||
),
|
||||
);
|
||||
|
||||
const state = await agent.getState(runId, `pairwise-gen-${generationIndex}`);
|
||||
@@ -441,7 +444,16 @@ function createPairwiseWorkflowGenerator(
|
||||
// Run all generations in parallel
|
||||
const generationResults = await Promise.all(
|
||||
Array.from({ length: numGenerations }, async (_, i) => {
|
||||
return await runSingleGeneration(parsedNodeTypes, llm, numJudges, inputs, i, log, featureFlags, tracer);
|
||||
return await runSingleGeneration(
|
||||
parsedNodeTypes,
|
||||
llm,
|
||||
numJudges,
|
||||
inputs,
|
||||
i,
|
||||
log,
|
||||
featureFlags,
|
||||
tracer,
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -838,7 +850,12 @@ export async function runLocalPairwiseEvaluation(options: LocalPairwiseOptions):
|
||||
const genStartTime = Date.now();
|
||||
const runId = generateRunId();
|
||||
const agent = createAgent(parsedNodeTypes, llm, undefined, featureFlags);
|
||||
await consumeGenerator(agent.chat(getChatPayload(prompt, runId, featureFlags), `local-gen-${genIndex}`));
|
||||
await consumeGenerator(
|
||||
agent.chat(
|
||||
getChatPayload('pairwise-local', prompt, runId, featureFlags),
|
||||
`local-gen-${genIndex}`,
|
||||
),
|
||||
);
|
||||
const state = await agent.getState(runId, `local-gen-${genIndex}`);
|
||||
|
||||
if (!state.values || !isWorkflowStateValues(state.values)) {
|
||||
|
||||
@@ -46,7 +46,10 @@ function createWorkflowGenerator(
|
||||
// Create agent for this run
|
||||
const agent = createAgent(parsedNodeTypes, llm, tracer);
|
||||
await consumeGenerator(
|
||||
agent.chat(getChatPayload(messageContent, runId, featureFlags), 'langsmith-eval-user'),
|
||||
agent.chat(
|
||||
getChatPayload('langsmith-evals', messageContent, runId, featureFlags),
|
||||
'langsmith-eval-user',
|
||||
),
|
||||
);
|
||||
|
||||
// Get generated workflow with validation
|
||||
|
||||
@@ -6,6 +6,7 @@ import { Client } from 'langsmith';
|
||||
import type { INodeTypeDescription } from 'n8n-workflow';
|
||||
import { join } from 'path';
|
||||
import pc from 'picocolors';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
import { anthropicClaudeSonnet45 } from '../../src/llm-config';
|
||||
import type { BuilderFeatureFlags, ChatPayload } from '../../src/workflow-builder-agent';
|
||||
@@ -278,18 +279,20 @@ export async function consumeGenerator<T>(gen: AsyncGenerator<T>) {
|
||||
}
|
||||
|
||||
export function getChatPayload(
|
||||
evalType: string,
|
||||
message: string,
|
||||
id: string,
|
||||
workflowId: string,
|
||||
featureFlags?: BuilderFeatureFlags,
|
||||
): ChatPayload {
|
||||
return {
|
||||
id: `${evalType}-${uuid()}`,
|
||||
featureFlags: featureFlags ?? {
|
||||
multiAgent: true,
|
||||
templateExamples: false,
|
||||
},
|
||||
message,
|
||||
workflowContext: {
|
||||
currentWorkflow: { id, nodes: [], connections: {} },
|
||||
currentWorkflow: { id: workflowId, nodes: [], connections: {} },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -61,10 +61,10 @@ export class AiWorkflowBuilderService {
|
||||
});
|
||||
}
|
||||
|
||||
private async getApiProxyAuthHeaders(user: IUser) {
|
||||
private async getApiProxyAuthHeaders(user: IUser, userMessageId: string) {
|
||||
assert(this.client);
|
||||
|
||||
const authResponse = await this.client.getBuilderApiProxyToken(user);
|
||||
const authResponse = await this.client.getBuilderApiProxyToken(user, { userMessageId });
|
||||
const authHeaders = {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
Authorization: `${authResponse.tokenType} ${authResponse.accessToken}`,
|
||||
@@ -73,7 +73,10 @@ export class AiWorkflowBuilderService {
|
||||
return authHeaders;
|
||||
}
|
||||
|
||||
private async setupModels(user: IUser): Promise<{
|
||||
private async setupModels(
|
||||
user: IUser,
|
||||
userMessageId: string,
|
||||
): Promise<{
|
||||
anthropicClaude: ChatAnthropic;
|
||||
tracingClient?: TracingClient;
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
@@ -82,7 +85,7 @@ export class AiWorkflowBuilderService {
|
||||
try {
|
||||
// If client is provided, use it for API proxy
|
||||
if (this.client) {
|
||||
const authHeaders = await this.getApiProxyAuthHeaders(user);
|
||||
const authHeaders = await this.getApiProxyAuthHeaders(user, userMessageId);
|
||||
|
||||
// Extract baseUrl from client configuration
|
||||
const baseUrl = this.client.getApiProxyBaseUrl();
|
||||
@@ -158,8 +161,11 @@ export class AiWorkflowBuilderService {
|
||||
});
|
||||
}
|
||||
|
||||
private async getAgent(user: IUser, featureFlags?: BuilderFeatureFlags) {
|
||||
const { anthropicClaude, tracingClient, authHeaders } = await this.setupModels(user);
|
||||
private async getAgent(user: IUser, userMessageId: string, featureFlags?: BuilderFeatureFlags) {
|
||||
const { anthropicClaude, tracingClient, authHeaders } = await this.setupModels(
|
||||
user,
|
||||
userMessageId,
|
||||
);
|
||||
|
||||
const agent = new WorkflowBuilderAgent({
|
||||
parsedNodeTypes: this.parsedNodeTypes,
|
||||
@@ -208,7 +214,7 @@ export class AiWorkflowBuilderService {
|
||||
}
|
||||
|
||||
async *chat(payload: ChatPayload, user: IUser, abortSignal?: AbortSignal) {
|
||||
const agent = await this.getAgent(user, payload.featureFlags);
|
||||
const agent = await this.getAgent(user, payload.id, payload.featureFlags);
|
||||
const userId = user?.id?.toString();
|
||||
const workflowId = payload.workflowContext?.currentWorkflow?.id;
|
||||
|
||||
@@ -219,7 +225,7 @@ export class AiWorkflowBuilderService {
|
||||
// After the stream completes, track telemetry
|
||||
if (this.onTelemetryEvent && userId) {
|
||||
try {
|
||||
await this.trackBuilderReplyTelemetry(agent, workflowId, userId);
|
||||
await this.trackBuilderReplyTelemetry(agent, workflowId, userId, payload.id);
|
||||
} catch (error) {
|
||||
this.logger?.error('Failed to track builder reply telemetry', { error });
|
||||
}
|
||||
@@ -230,6 +236,7 @@ export class AiWorkflowBuilderService {
|
||||
agent: WorkflowBuilderAgent,
|
||||
workflowId: string | undefined,
|
||||
userId: string,
|
||||
userMessageId: string,
|
||||
): Promise<void> {
|
||||
if (!this.onTelemetryEvent) return;
|
||||
|
||||
@@ -270,6 +277,7 @@ export class AiWorkflowBuilderService {
|
||||
...(state.values.templateIds.length > 0 && {
|
||||
templates_selected: state.values.templateIds,
|
||||
}),
|
||||
user_message_id: userMessageId,
|
||||
};
|
||||
|
||||
this.onTelemetryEvent('Builder replied to user message', properties);
|
||||
|
||||
+2
-3
@@ -151,9 +151,6 @@ describe('AiWorkflowBuilderService', () => {
|
||||
|
||||
// Mock AI assistant client
|
||||
mockClient = mock<AiAssistantClient>();
|
||||
(mockClient.generateApiProxyCredentials as jest.Mock).mockResolvedValue({
|
||||
apiKey: 'test-api-key',
|
||||
});
|
||||
(mockClient.getBuilderApiProxyToken as jest.Mock).mockResolvedValue({
|
||||
tokenType: 'Bearer',
|
||||
accessToken: 'test-access-token',
|
||||
@@ -301,6 +298,7 @@ describe('AiWorkflowBuilderService', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
mockPayload = {
|
||||
id: '12345',
|
||||
message: 'Create a simple workflow',
|
||||
workflowContext: {
|
||||
currentWorkflow: { id: 'test-workflow' },
|
||||
@@ -571,6 +569,7 @@ describe('AiWorkflowBuilderService', () => {
|
||||
it('should handle complete workflow from chat to session retrieval', async () => {
|
||||
const workflowId = 'integration-test-workflow';
|
||||
const mockPayload: ChatPayload = {
|
||||
id: '545623',
|
||||
message: 'Create a workflow with HTTP request',
|
||||
workflowContext: {
|
||||
currentWorkflow: { id: workflowId },
|
||||
|
||||
@@ -146,6 +146,7 @@ describe('WorkflowBuilderAgent', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
mockPayload = {
|
||||
id: '12345',
|
||||
message: 'Create a workflow',
|
||||
workflowContext: {
|
||||
currentWorkflow: { id: 'workflow-123' },
|
||||
@@ -156,6 +157,7 @@ describe('WorkflowBuilderAgent', () => {
|
||||
it('should throw ValidationError when message exceeds maximum length', async () => {
|
||||
const longMessage = 'x'.repeat(MAX_AI_BUILDER_PROMPT_LENGTH + 1);
|
||||
const payload: ChatPayload = {
|
||||
id: '12345',
|
||||
message: longMessage,
|
||||
};
|
||||
|
||||
@@ -173,6 +175,7 @@ describe('WorkflowBuilderAgent', () => {
|
||||
it('should handle valid message length', async () => {
|
||||
const validMessage = 'Create a simple workflow';
|
||||
const payload: ChatPayload = {
|
||||
id: '12345',
|
||||
message: validMessage,
|
||||
};
|
||||
|
||||
@@ -491,6 +494,7 @@ describe('WorkflowBuilderAgent', () => {
|
||||
);
|
||||
|
||||
const generator = agent.chat({
|
||||
id: '12345',
|
||||
message: 'Create a workflow',
|
||||
featureFlags: { templateExamples: true },
|
||||
});
|
||||
@@ -511,6 +515,7 @@ describe('WorkflowBuilderAgent', () => {
|
||||
);
|
||||
|
||||
const generator = agent.chat({
|
||||
id: '12345',
|
||||
message: 'Create a workflow',
|
||||
featureFlags: { templateExamples: false },
|
||||
});
|
||||
@@ -530,7 +535,10 @@ describe('WorkflowBuilderAgent', () => {
|
||||
mockAsyncGenerator,
|
||||
);
|
||||
|
||||
const generator = agent.chat({ message: 'Create a workflow' });
|
||||
const generator = agent.chat({
|
||||
id: '12345',
|
||||
message: 'Create a workflow',
|
||||
});
|
||||
await generator.next();
|
||||
|
||||
expect(mockCreateMainAgentPrompt).toHaveBeenCalledWith({ includeExamplesPhase: false });
|
||||
|
||||
@@ -159,6 +159,7 @@ export interface BuilderFeatureFlags {
|
||||
}
|
||||
|
||||
export interface ChatPayload {
|
||||
id: string;
|
||||
message: string;
|
||||
workflowContext?: {
|
||||
executionSchema?: NodeExecutionSchema[];
|
||||
|
||||
@@ -3,6 +3,7 @@ import { AiBuilderChatRequestDto } from '../ai-build-request.dto';
|
||||
describe('AiBuilderChatRequestDto', () => {
|
||||
const validBasePayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user' as const,
|
||||
type: 'message' as const,
|
||||
text: 'Build me a workflow',
|
||||
@@ -305,6 +306,7 @@ describe('AiBuilderChatRequestDto', () => {
|
||||
const invalidRequest = {
|
||||
...validBasePayload,
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user' as const,
|
||||
type: 'message' as const,
|
||||
workflowContext: validBasePayload.payload.workflowContext,
|
||||
@@ -315,5 +317,21 @@ describe('AiBuilderChatRequestDto', () => {
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it('should fail when id is missing', () => {
|
||||
const invalidRequest = {
|
||||
...validBasePayload,
|
||||
payload: {
|
||||
role: 'user' as const,
|
||||
type: 'message' as const,
|
||||
text: 'text',
|
||||
workflowContext: validBasePayload.payload.workflowContext,
|
||||
},
|
||||
};
|
||||
|
||||
const result = AiBuilderChatRequestDto.safeParse(invalidRequest);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ export interface ExpressionValue {
|
||||
|
||||
export class AiBuilderChatRequestDto extends Z.class({
|
||||
payload: z.object({
|
||||
id: z.string(),
|
||||
role: z.literal('user'),
|
||||
type: z.literal('message'),
|
||||
text: z.string(),
|
||||
|
||||
@@ -118,6 +118,7 @@ describe('AiController', () => {
|
||||
describe('build', () => {
|
||||
const payload: AiBuilderChatRequestDto = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
text: 'Create a workflow',
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
@@ -146,6 +147,8 @@ describe('AiController', () => {
|
||||
|
||||
expect(workflowBuilderService.chat).toHaveBeenCalledWith(
|
||||
{
|
||||
id: '12345',
|
||||
featureFlags: undefined,
|
||||
message: 'Create a workflow',
|
||||
workflowContext: {
|
||||
currentWorkflow: { id: 'workflow123' },
|
||||
|
||||
@@ -55,9 +55,10 @@ export class AiController {
|
||||
|
||||
res.on('close', handleClose);
|
||||
|
||||
const { text, workflowContext, featureFlags } = payload.payload;
|
||||
const { id, text, workflowContext, featureFlags } = payload.payload;
|
||||
const aiResponse = this.workflowBuilderService.chat(
|
||||
{
|
||||
id,
|
||||
message: text,
|
||||
workflowContext: {
|
||||
currentWorkflow: workflowContext.currentWorkflow,
|
||||
|
||||
@@ -100,6 +100,7 @@ describe('WorkflowBuilderService', () => {
|
||||
describe('chat', () => {
|
||||
it('should create AiWorkflowBuilderService on first chat call without AI assistant client', async () => {
|
||||
const mockPayload = {
|
||||
id: '12345',
|
||||
message: 'test message',
|
||||
workflowContext: {},
|
||||
};
|
||||
@@ -134,6 +135,7 @@ describe('WorkflowBuilderService', () => {
|
||||
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -170,6 +172,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should reuse the same service instance on subsequent calls', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -201,6 +204,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should pass abort signal to underlying service', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -265,6 +269,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should send push notification when credits are updated', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -313,6 +318,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should handle multiple credit updates', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -373,6 +379,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should call telemetry.track when telemetry event is triggered', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -419,6 +426,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should handle multiple telemetry events', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
@@ -463,6 +471,7 @@ describe('WorkflowBuilderService', () => {
|
||||
it('should handle telemetry events with empty properties', async () => {
|
||||
const mockPayload = {
|
||||
message: 'test message',
|
||||
id: '12345',
|
||||
workflowContext: {},
|
||||
};
|
||||
|
||||
|
||||
+93
-1
@@ -4,9 +4,11 @@ import {
|
||||
continueEvaluationLoop,
|
||||
executionFinished,
|
||||
getRunExecutionData,
|
||||
handleExecutionFinishedWithSuccessOrOther,
|
||||
handleExecutionFinishedWithErrorOrCanceled,
|
||||
type SimplifiedExecution,
|
||||
} from './executionFinished';
|
||||
import type { ITaskData } from 'n8n-workflow';
|
||||
import type { IRunExecutionData, ITaskData } from 'n8n-workflow';
|
||||
import { EVALUATION_TRIGGER_NODE_TYPE } from 'n8n-workflow';
|
||||
import type { INodeUi, IWorkflowDb } from '@/Interface';
|
||||
import type { Router } from 'vue-router';
|
||||
@@ -17,6 +19,7 @@ import { useWorkflowsStore } from '@/app/stores/workflows.store';
|
||||
import { useUIStore } from '@/app/stores/ui.store';
|
||||
import { mockedStore } from '@/__tests__/utils';
|
||||
import { useReadyToRunStore } from '@/features/workflows/readyToRun/stores/readyToRun.store';
|
||||
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
|
||||
|
||||
const opts = {
|
||||
workflowState: mock<WorkflowState>(),
|
||||
@@ -524,3 +527,92 @@ describe('executionFinished', () => {
|
||||
expect(runWorkflow).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('manual execution stats tracking', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
describe('handleExecutionFinishedWithSuccessOrOther', () => {
|
||||
it('increments success stats on successful execution', () => {
|
||||
const pinia = createTestingPinia();
|
||||
setActivePinia(pinia);
|
||||
|
||||
const builderStore = mockedStore(useBuilderStore);
|
||||
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
|
||||
|
||||
handleExecutionFinishedWithSuccessOrOther(mock<WorkflowState>(), 'success', false);
|
||||
|
||||
expect(incrementSpy).toHaveBeenCalledWith('success');
|
||||
});
|
||||
|
||||
it('does not increment success stats when successToastAlreadyShown is true', () => {
|
||||
const pinia = createTestingPinia();
|
||||
setActivePinia(pinia);
|
||||
|
||||
const builderStore = mockedStore(useBuilderStore);
|
||||
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
|
||||
|
||||
handleExecutionFinishedWithSuccessOrOther(mock<WorkflowState>(), 'success', true);
|
||||
|
||||
expect(incrementSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not increment stats for non-success status', () => {
|
||||
const pinia = createTestingPinia();
|
||||
setActivePinia(pinia);
|
||||
|
||||
const builderStore = mockedStore(useBuilderStore);
|
||||
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
|
||||
|
||||
handleExecutionFinishedWithSuccessOrOther(mock<WorkflowState>(), 'error', false);
|
||||
|
||||
expect(incrementSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleExecutionFinishedWithErrorOrCanceled', () => {
|
||||
it('increments error stats on execution error', () => {
|
||||
const pinia = createTestingPinia();
|
||||
setActivePinia(pinia);
|
||||
|
||||
const builderStore = mockedStore(useBuilderStore);
|
||||
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
|
||||
|
||||
const execution = mock<SimplifiedExecution>({
|
||||
status: 'error',
|
||||
data: {
|
||||
resultData: {
|
||||
error: { message: 'test error', name: 'Error' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
handleExecutionFinishedWithErrorOrCanceled(
|
||||
execution,
|
||||
mock<IRunExecutionData>({ resultData: { error: { message: 'test', name: 'Error' } } }),
|
||||
);
|
||||
|
||||
expect(incrementSpy).toHaveBeenCalledWith('error');
|
||||
});
|
||||
|
||||
it('does not increment stats for canceled executions', () => {
|
||||
const pinia = createTestingPinia();
|
||||
setActivePinia(pinia);
|
||||
|
||||
const builderStore = mockedStore(useBuilderStore);
|
||||
const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats');
|
||||
|
||||
const execution = mock<SimplifiedExecution>({
|
||||
status: 'canceled',
|
||||
});
|
||||
|
||||
handleExecutionFinishedWithErrorOrCanceled(
|
||||
execution,
|
||||
mock<IRunExecutionData>({ resultData: {} }),
|
||||
);
|
||||
|
||||
expect(incrementSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
+27
-3
@@ -14,6 +14,7 @@ import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
|
||||
import { useSettingsStore } from '@/app/stores/settings.store';
|
||||
import { useUIStore } from '@/app/stores/ui.store';
|
||||
import { useWorkflowsStore } from '@/app/stores/workflows.store';
|
||||
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
|
||||
import {
|
||||
SampleTemplates,
|
||||
isPrebuiltAgentTemplateId,
|
||||
@@ -28,7 +29,13 @@ import { getTriggerNodeServiceName } from '@/app/utils/nodeTypesUtils';
|
||||
import type { ExecutionFinished } from '@n8n/api-types/push/execution';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
import { parse } from 'flatted';
|
||||
import type { ExpressionError, IDataObject, IRunExecutionData, IWorkflowBase } from 'n8n-workflow';
|
||||
import type {
|
||||
ExecutionStatus,
|
||||
ExpressionError,
|
||||
IDataObject,
|
||||
IRunExecutionData,
|
||||
IWorkflowBase,
|
||||
} from 'n8n-workflow';
|
||||
import {
|
||||
EVALUATION_TRIGGER_NODE_TYPE,
|
||||
TelemetryHelpers,
|
||||
@@ -115,7 +122,11 @@ export async function executionFinished(
|
||||
let successToastAlreadyShown = false;
|
||||
|
||||
if (data.status === 'success') {
|
||||
handleExecutionFinishedWithSuccessOrOther(options.workflowState, successToastAlreadyShown);
|
||||
handleExecutionFinishedWithSuccessOrOther(
|
||||
options.workflowState,
|
||||
data.status,
|
||||
successToastAlreadyShown,
|
||||
);
|
||||
successToastAlreadyShown = true;
|
||||
}
|
||||
|
||||
@@ -140,7 +151,11 @@ export async function executionFinished(
|
||||
} else if (execution.status === 'error' || execution.status === 'canceled') {
|
||||
handleExecutionFinishedWithErrorOrCanceled(execution, runExecutionData);
|
||||
} else {
|
||||
handleExecutionFinishedWithSuccessOrOther(options.workflowState, successToastAlreadyShown);
|
||||
handleExecutionFinishedWithSuccessOrOther(
|
||||
options.workflowState,
|
||||
execution.status,
|
||||
successToastAlreadyShown,
|
||||
);
|
||||
}
|
||||
|
||||
setRunExecutionData(execution, runExecutionData, options.workflowState);
|
||||
@@ -351,6 +366,8 @@ export function handleExecutionFinishedWithErrorOrCanceled(
|
||||
});
|
||||
|
||||
toast.showMessage({ title, message, type: 'error', duration: 0 });
|
||||
|
||||
useBuilderStore().incrementManualExecutionStats('error');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,6 +398,7 @@ function handleExecutionFinishedSuccessfully(
|
||||
*/
|
||||
export function handleExecutionFinishedWithSuccessOrOther(
|
||||
workflowState: WorkflowState,
|
||||
executionStatus: ExecutionStatus,
|
||||
successToastAlreadyShown: boolean,
|
||||
) {
|
||||
const workflowsStore = useWorkflowsStore();
|
||||
@@ -427,6 +445,12 @@ export function handleExecutionFinishedWithSuccessOrOther(
|
||||
workflowState,
|
||||
);
|
||||
}
|
||||
|
||||
// Execution finished is triggered multiple times
|
||||
// use "successToastAlreadyShown" flag to avoid double counting executions
|
||||
if (executionStatus === 'success' && !successToastAlreadyShown) {
|
||||
useBuilderStore().incrementManualExecutionStats('success');
|
||||
}
|
||||
}
|
||||
|
||||
export function setRunExecutionData(
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ export async function executionRecovered(
|
||||
} else if (execution.status === 'error' || execution.status === 'canceled') {
|
||||
handleExecutionFinishedWithErrorOrCanceled(execution, runExecutionData);
|
||||
} else {
|
||||
handleExecutionFinishedWithSuccessOrOther(options.workflowState, false);
|
||||
handleExecutionFinishedWithSuccessOrOther(options.workflowState, execution.status, false);
|
||||
}
|
||||
|
||||
setRunExecutionData(execution, runExecutionData, options.workflowState);
|
||||
|
||||
@@ -15,6 +15,7 @@ import type {
|
||||
} from '@/features/execution/executions/executions.types';
|
||||
import { useUIStore } from '@/app/stores/ui.store';
|
||||
import { useWorkflowsStore } from '@/app/stores/workflows.store';
|
||||
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
|
||||
import { getPairedItemsMapping } from '@/app/utils/pairedItemUtils';
|
||||
import {
|
||||
type INodeIssueData,
|
||||
@@ -233,6 +234,7 @@ export function useWorkflowState() {
|
||||
setActiveExecutionId(undefined);
|
||||
workflowStateStore.executingNode.executingNode.length = 0;
|
||||
ws.executionWaitingForWebhook = false;
|
||||
useBuilderStore().resetManualExecutionStats();
|
||||
}
|
||||
|
||||
////
|
||||
|
||||
@@ -2149,7 +2149,7 @@ onBeforeUnmount(() => {
|
||||
v-if="builderStore.streaming"
|
||||
:class="$style.thinkingPill"
|
||||
show-stop
|
||||
@stop="builderStore.stopStreaming"
|
||||
@stop="builderStore.abortStreaming"
|
||||
/>
|
||||
|
||||
<Suspense>
|
||||
|
||||
@@ -38,6 +38,7 @@ describe('API: ai', () => {
|
||||
it('should call streamRequest with the correct parameters', () => {
|
||||
const payload: ChatRequest.RequestPayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: 'Build me a workflow',
|
||||
@@ -62,6 +63,7 @@ describe('API: ai', () => {
|
||||
it('should pass abort signal when provided', () => {
|
||||
const payload: ChatRequest.RequestPayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: 'Build me a workflow',
|
||||
@@ -177,6 +179,7 @@ describe('API: ai', () => {
|
||||
|
||||
const payload: ChatRequest.RequestPayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: 'Build me a workflow',
|
||||
@@ -211,6 +214,7 @@ describe('API: ai', () => {
|
||||
|
||||
const payload: ChatRequest.RequestPayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: 'Build me a workflow',
|
||||
@@ -228,6 +232,7 @@ describe('API: ai', () => {
|
||||
it('should handle complex workflow context in payload', () => {
|
||||
const payload: ChatRequest.RequestPayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: 'Improve my workflow',
|
||||
@@ -271,6 +276,7 @@ describe('API: ai', () => {
|
||||
it('should handle undefined parameters correctly', () => {
|
||||
const payload: ChatRequest.RequestPayload = {
|
||||
payload: {
|
||||
id: '12345',
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: 'Build me a workflow',
|
||||
|
||||
@@ -29,6 +29,7 @@ import { useChatPanelStateStore } from './chatPanelState.store';
|
||||
import { useCredentialsStore } from '@/features/credentials/credentials.store';
|
||||
import { useAIAssistantHelpers } from '@/features/ai/assistant/composables/useAIAssistantHelpers';
|
||||
import type { WorkflowState } from '@/app/composables/useWorkflowState';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
export const ENABLED_VIEWS = ASSISTANT_ENABLED_VIEWS;
|
||||
const READABLE_TYPES = ['code-diff', 'text', 'block'];
|
||||
@@ -555,6 +556,7 @@ export const useAssistantStore = defineStore(STORES.ASSISTANT, () => {
|
||||
rootStore.restApiContext,
|
||||
{
|
||||
payload: {
|
||||
id: uuid(),
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
text: chatMessage.text,
|
||||
|
||||
@@ -102,6 +102,7 @@ export namespace ChatRequest {
|
||||
role: 'user';
|
||||
type: 'message';
|
||||
text: string;
|
||||
id: string;
|
||||
quickReplyType?: string;
|
||||
context?: UserContext;
|
||||
workflowContext?: WorkflowContext;
|
||||
|
||||
@@ -606,7 +606,7 @@ describe('AI Builder store', () => {
|
||||
expect(builderStore.streamingAbortController).toBeInstanceOf(AbortController);
|
||||
});
|
||||
|
||||
it('should call abort on existing controller when stopStreaming is called', () => {
|
||||
it('should call abort on existing controller when abortStreaming is called', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
// First start a request to create an abort controller
|
||||
@@ -620,8 +620,8 @@ describe('AI Builder store', () => {
|
||||
// Spy on the abort method
|
||||
const abortSpy = vi.spyOn(controller!, 'abort');
|
||||
|
||||
// Call stopStreaming
|
||||
builderStore.stopStreaming();
|
||||
// Call abortStreaming
|
||||
builderStore.abortStreaming();
|
||||
|
||||
// Verify abort was called
|
||||
expect(abortSpy).toHaveBeenCalled();
|
||||
@@ -698,7 +698,7 @@ describe('AI Builder store', () => {
|
||||
expect(builderStore.streamingAbortController).toBe(firstController);
|
||||
|
||||
// Now properly stop streaming first
|
||||
builderStore.stopStreaming();
|
||||
builderStore.abortStreaming();
|
||||
|
||||
// Verify abort was called and controller was cleared
|
||||
expect(abortSpy).toHaveBeenCalled();
|
||||
@@ -1708,4 +1708,570 @@ describe('AI Builder store', () => {
|
||||
expect(builderStore.creditsRemaining).toBe(80);
|
||||
});
|
||||
});
|
||||
|
||||
describe('abortStreaming telemetry', () => {
|
||||
it('tracks end of response with aborted flag when aborting', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
track.mockClear();
|
||||
builderStore.abortStreaming();
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'End of response from builder',
|
||||
expect.objectContaining({
|
||||
aborted: true,
|
||||
user_message_id: expect.any(String),
|
||||
workflow_id: 'test-workflow-id',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('includes workflow modifications in abort telemetry', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
track.mockClear();
|
||||
builderStore.abortStreaming();
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'End of response from builder',
|
||||
expect.objectContaining({
|
||||
tools_called: expect.any(Array),
|
||||
start_workflow_json: expect.any(String),
|
||||
end_workflow_json: expect.any(String),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('includes todos count in abort telemetry', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
track.mockClear();
|
||||
builderStore.abortStreaming();
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'End of response from builder',
|
||||
expect.objectContaining({
|
||||
credentials_todo_count: expect.any(Number),
|
||||
placeholders_todo_count: expect.any(Number),
|
||||
todos: expect.any(Array),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not track telemetry if no streaming message in progress', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
// Don't start any streaming
|
||||
track.mockClear();
|
||||
|
||||
builderStore.abortStreaming();
|
||||
|
||||
expect(track).not.toHaveBeenCalledWith('End of response from builder', expect.anything());
|
||||
});
|
||||
});
|
||||
|
||||
describe('workflowTodos', () => {
|
||||
it('returns empty array when no validation issues exist', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toEqual([]);
|
||||
});
|
||||
|
||||
it('includes credential validation issues', () => {
|
||||
workflowsStore.workflowValidationIssues = [
|
||||
{ node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' },
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toContainEqual(
|
||||
expect.objectContaining({ type: 'credentials' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('includes placeholder issues from node parameters', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toContainEqual(
|
||||
expect.objectContaining({ type: 'parameters', node: 'HTTP Request' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('combines credential and placeholder issues', () => {
|
||||
workflowsStore.workflowValidationIssues = [
|
||||
{ node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' },
|
||||
];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos.length).toBeGreaterThanOrEqual(2);
|
||||
expect(builderStore.workflowTodos).toContainEqual(
|
||||
expect.objectContaining({ type: 'credentials' }),
|
||||
);
|
||||
expect(builderStore.workflowTodos).toContainEqual(
|
||||
expect.objectContaining({ type: 'parameters' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('placeholderIssues', () => {
|
||||
it('returns empty array when nodes have no parameters', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'Start',
|
||||
type: 'n8n-nodes-base.start',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array when node has undefined parameters', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'Start',
|
||||
type: 'n8n-nodes-base.start',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
} as Parameters<typeof workflowsStore.workflow.nodes.push>[0],
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toEqual([]);
|
||||
});
|
||||
|
||||
it('detects placeholders in nested object parameters', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
options: {
|
||||
headers: {
|
||||
authorization: '<__PLACEHOLDER_VALUE__Enter API Key__>',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
expect(placeholderIssues).toHaveLength(1);
|
||||
expect(placeholderIssues[0]).toMatchObject({
|
||||
node: 'HTTP Request',
|
||||
type: 'parameters',
|
||||
});
|
||||
});
|
||||
|
||||
it('detects placeholders in array parameters', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
headers: [
|
||||
{ name: 'Content-Type', value: 'application/json' },
|
||||
{ name: 'Authorization', value: '<__PLACEHOLDER_VALUE__Enter Token__>' },
|
||||
],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
expect(placeholderIssues).toHaveLength(1);
|
||||
expect(placeholderIssues[0]).toMatchObject({
|
||||
node: 'HTTP Request',
|
||||
type: 'parameters',
|
||||
});
|
||||
});
|
||||
|
||||
it('detects multiple placeholders in the same node', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
body: '<__PLACEHOLDER_VALUE__Enter Body__>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
expect(placeholderIssues).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('detects placeholders across multiple nodes', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'node-2',
|
||||
name: 'Slack',
|
||||
type: 'n8n-nodes-base.slack',
|
||||
typeVersion: 1,
|
||||
position: [200, 0],
|
||||
parameters: {
|
||||
channel: '<__PLACEHOLDER_VALUE__Enter Channel__>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
expect(placeholderIssues).toHaveLength(2);
|
||||
expect(placeholderIssues).toContainEqual(expect.objectContaining({ node: 'HTTP Request' }));
|
||||
expect(placeholderIssues).toContainEqual(expect.objectContaining({ node: 'Slack' }));
|
||||
});
|
||||
|
||||
it('deduplicates identical placeholder issues (same node, path, and label)', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
// Simulate a scenario where the same placeholder appears twice
|
||||
// (which shouldn't happen in practice but tests the deduplication)
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
// Should only have 1 issue, not duplicates
|
||||
expect(placeholderIssues).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('skips placeholder when existing parameter issue already has the same message', () => {
|
||||
const placeholderLabel = 'Enter URL';
|
||||
// The message format from the store uses i18n which is mocked to return the key
|
||||
const expectedMessage = 'aiAssistant.builder.executeMessage.fillParameter';
|
||||
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: `<__PLACEHOLDER_VALUE__${placeholderLabel}__>`,
|
||||
},
|
||||
issues: {
|
||||
parameters: {
|
||||
url: [expectedMessage],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
// Should be skipped because the message already exists
|
||||
expect(placeholderIssues).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('does not skip placeholder when existing parameter issue has different message', () => {
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
},
|
||||
issues: {
|
||||
parameters: {
|
||||
url: ['Some other validation error'],
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters');
|
||||
// Should still create the placeholder issue
|
||||
expect(placeholderIssues).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('ignores non-string parameter values', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
timeout: 5000,
|
||||
enabled: true,
|
||||
config: null,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toEqual([]);
|
||||
});
|
||||
|
||||
it('ignores strings that do not match placeholder format', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: 'https://example.com',
|
||||
body: 'regular string',
|
||||
partial: '<__PLACEHOLDER_VALUE__missing end',
|
||||
wrongPrefix: 'PLACEHOLDER__test__>',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toEqual([]);
|
||||
});
|
||||
|
||||
it('ignores placeholder with empty label', () => {
|
||||
workflowsStore.workflowValidationIssues = [];
|
||||
workflowsStore.workflow.nodes = [
|
||||
{
|
||||
id: 'node-1',
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
typeVersion: 1,
|
||||
position: [0, 0],
|
||||
parameters: {
|
||||
url: '<__PLACEHOLDER_VALUE____>', // empty label
|
||||
body: '<__PLACEHOLDER_VALUE__ __>', // whitespace-only label
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
expect(builderStore.workflowTodos).toEqual([]);
|
||||
});
|
||||
|
||||
it('filters out non-credential and non-parameter validation issues', () => {
|
||||
workflowsStore.workflowValidationIssues = [
|
||||
{ node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' },
|
||||
{ node: 'HTTP Request', type: 'parameters', value: 'Missing parameter' },
|
||||
{ node: 'HTTP Request', type: 'execution', value: 'Execution error' },
|
||||
{ node: 'HTTP Request', type: 'unknown' as 'parameters', value: 'Unknown issue' },
|
||||
];
|
||||
workflowsStore.workflow.nodes = [];
|
||||
|
||||
const builderStore = useBuilderStore();
|
||||
// Should only include credentials and parameters types
|
||||
expect(builderStore.workflowTodos).toHaveLength(2);
|
||||
expect(builderStore.workflowTodos).toContainEqual(
|
||||
expect.objectContaining({ type: 'credentials' }),
|
||||
);
|
||||
expect(builderStore.workflowTodos).toContainEqual(
|
||||
expect.objectContaining({ type: 'parameters' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('manual execution stats telemetry', () => {
|
||||
it('should include success count in telemetry when sending message', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
|
||||
builderStore.incrementManualExecutionStats('success');
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'User submitted builder message',
|
||||
expect.objectContaining({
|
||||
manual_exec_success_count_since_prev_msg: 1,
|
||||
manual_exec_error_count_since_prev_msg: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should include error count in telemetry when sending message', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
|
||||
builderStore.incrementManualExecutionStats('error');
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'User submitted builder message',
|
||||
expect.objectContaining({
|
||||
manual_exec_success_count_since_prev_msg: 0,
|
||||
manual_exec_error_count_since_prev_msg: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should include multiple incremented counts in telemetry', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
|
||||
builderStore.incrementManualExecutionStats('success');
|
||||
builderStore.incrementManualExecutionStats('success');
|
||||
builderStore.incrementManualExecutionStats('error');
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'User submitted builder message',
|
||||
expect.objectContaining({
|
||||
manual_exec_success_count_since_prev_msg: 2,
|
||||
manual_exec_error_count_since_prev_msg: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should reset stats after sending message', async () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
// First message with some stats
|
||||
apiSpy.mockImplementationOnce((_ctx, _payload, onMessage, onDone) => {
|
||||
onMessage({
|
||||
messages: [{ type: 'message', role: 'assistant', text: 'Hello!' }],
|
||||
sessionId: 'test-session',
|
||||
});
|
||||
onDone();
|
||||
});
|
||||
|
||||
builderStore.incrementManualExecutionStats('success');
|
||||
builderStore.incrementManualExecutionStats('error');
|
||||
builderStore.sendChatMessage({ text: 'first message' });
|
||||
|
||||
await vi.waitFor(() => expect(builderStore.streaming).toBe(false));
|
||||
|
||||
// Verify first message had the stats
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'User submitted builder message',
|
||||
expect.objectContaining({
|
||||
manual_exec_success_count_since_prev_msg: 1,
|
||||
manual_exec_error_count_since_prev_msg: 1,
|
||||
}),
|
||||
);
|
||||
|
||||
track.mockClear();
|
||||
|
||||
// Second message should have reset stats (zero counts)
|
||||
apiSpy.mockImplementationOnce((_ctx, _payload, onMessage, onDone) => {
|
||||
onMessage({
|
||||
messages: [{ type: 'message', role: 'assistant', text: 'Hello again!' }],
|
||||
sessionId: 'test-session',
|
||||
});
|
||||
onDone();
|
||||
});
|
||||
|
||||
builderStore.sendChatMessage({ text: 'second message' });
|
||||
|
||||
await vi.waitFor(() => expect(builderStore.streaming).toBe(false));
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'User submitted builder message',
|
||||
expect.objectContaining({
|
||||
manual_exec_success_count_since_prev_msg: 0,
|
||||
manual_exec_error_count_since_prev_msg: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should include zero counts when no manual executions occurred', () => {
|
||||
const builderStore = useBuilderStore();
|
||||
|
||||
apiSpy.mockImplementationOnce(() => {});
|
||||
|
||||
builderStore.sendChatMessage({ text: 'test' });
|
||||
|
||||
expect(track).toHaveBeenCalledWith(
|
||||
'User submitted builder message',
|
||||
expect.objectContaining({
|
||||
manual_exec_success_count_since_prev_msg: 0,
|
||||
manual_exec_error_count_since_prev_msg: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,10 +20,11 @@ import {
|
||||
getSessionsMetadata,
|
||||
} from '@/features/ai/assistant/assistant.api';
|
||||
import { generateMessageId, createBuilderPayload } from './builder.utils';
|
||||
import { useBuilderTodos, type TodosTrackingPayload } from './composables/useBuilderTodos';
|
||||
import { useRootStore } from '@n8n/stores/useRootStore';
|
||||
import type { WorkflowDataUpdate } from '@n8n/rest-api-client/api/workflows';
|
||||
import pick from 'lodash/pick';
|
||||
import { type INodeExecutionData, jsonParse } from 'n8n-workflow';
|
||||
import { type ITelemetryTrackProperties, type INodeExecutionData, jsonParse } from 'n8n-workflow';
|
||||
import { useToast } from '@/app/composables/useToast';
|
||||
import { injectWorkflowState } from '@/app/composables/useWorkflowState';
|
||||
import { useNodeTypesStore } from '@/app/stores/nodeTypes.store';
|
||||
@@ -31,10 +32,34 @@ import { useCredentialsStore } from '@/features/credentials/credentials.store';
|
||||
import { getAuthTypeForNodeCredential, getMainAuthField } from '@/app/utils/nodeTypesUtils';
|
||||
import { stringSizeInBytes } from '@/app/utils/typesUtils';
|
||||
import { useNDVStore } from '@/features/ndv/shared/ndv.store';
|
||||
import { dedupe } from 'n8n-workflow';
|
||||
|
||||
const INFINITE_CREDITS = -1;
|
||||
export const ENABLED_VIEWS = BUILDER_ENABLED_VIEWS;
|
||||
|
||||
interface EndOfStreamingTrackingPayload {
|
||||
userMessageId: string;
|
||||
startWorkflowJson: string;
|
||||
}
|
||||
|
||||
interface UserSubmittedBuilderMessageTrackingPayload
|
||||
extends ITelemetryTrackProperties,
|
||||
TodosTrackingPayload {
|
||||
source: 'chat' | 'canvas';
|
||||
message: string;
|
||||
session_id: string;
|
||||
start_workflow_json: string;
|
||||
workflow_id: string;
|
||||
type: 'message' | 'execution';
|
||||
manual_exec_success_count_since_prev_msg: number;
|
||||
manual_exec_error_count_since_prev_msg: number;
|
||||
user_message_id: string;
|
||||
execution_data?: string;
|
||||
execution_status?: string;
|
||||
error_message?: string;
|
||||
error_node_type?: string;
|
||||
}
|
||||
|
||||
export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
// Core state
|
||||
const chatMessages = ref<ChatUI.AssistantMessage[]>([]);
|
||||
@@ -45,6 +70,12 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
const creditsQuota = ref<number | undefined>();
|
||||
const creditsClaimed = ref<number | undefined>();
|
||||
const hasMessages = ref<boolean>(false);
|
||||
const manualExecStatsInBetweenMessages = ref<{ success: number; error: number }>({
|
||||
success: 0,
|
||||
error: 0,
|
||||
});
|
||||
|
||||
const currentStreamingMessage = ref<EndOfStreamingTrackingPayload | undefined>();
|
||||
|
||||
// Store dependencies
|
||||
const settings = useSettingsStore();
|
||||
@@ -70,6 +101,8 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
getRunningTools,
|
||||
} = useBuilderMessages();
|
||||
|
||||
const { workflowTodos, getTodosToTrack } = useBuilderTodos();
|
||||
|
||||
const trackingSessionId = computed(() => rootStore.pushRef);
|
||||
|
||||
const workflowPrompt = computed(() => {
|
||||
@@ -124,17 +157,74 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
initialGeneration.value = false;
|
||||
}
|
||||
|
||||
function incrementManualExecutionStats(type: 'success' | 'error') {
|
||||
manualExecStatsInBetweenMessages.value[type]++;
|
||||
}
|
||||
|
||||
function resetManualExecutionStats() {
|
||||
manualExecStatsInBetweenMessages.value = {
|
||||
success: 0,
|
||||
error: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Message handling functions
|
||||
function addLoadingAssistantMessage(message: string) {
|
||||
builderThinkingMessage.value = message;
|
||||
}
|
||||
|
||||
function stopStreaming() {
|
||||
function getWorkflowModifications({
|
||||
userMessageId,
|
||||
startWorkflowJson,
|
||||
}: EndOfStreamingTrackingPayload) {
|
||||
const newToolMessages = toolMessages.value.filter((toolMsg) =>
|
||||
toolMsg.id?.startsWith(userMessageId),
|
||||
);
|
||||
const endWorkflowJson = getWorkflowSnapshot();
|
||||
|
||||
return {
|
||||
tools_called: dedupe(newToolMessages.map((toolMsg) => toolMsg.toolName)),
|
||||
start_workflow_json: startWorkflowJson,
|
||||
end_workflow_json: endWorkflowJson,
|
||||
};
|
||||
}
|
||||
|
||||
type StopStreamingPayload =
|
||||
| {
|
||||
error: string;
|
||||
}
|
||||
| { aborted: true };
|
||||
function trackEndBuilderResponse(payload?: StopStreamingPayload) {
|
||||
if (!currentStreamingMessage.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { userMessageId } = currentStreamingMessage.value;
|
||||
|
||||
telemetry.track('End of response from builder', {
|
||||
// todo user_id available as trait already?
|
||||
user_message_id: userMessageId,
|
||||
workflow_id: workflowsStore.workflowId,
|
||||
session_id: trackingSessionId.value,
|
||||
...getWorkflowModifications(currentStreamingMessage.value),
|
||||
...payload,
|
||||
...getTodosToTrack(),
|
||||
});
|
||||
}
|
||||
|
||||
function stopStreaming(payload?: StopStreamingPayload) {
|
||||
streaming.value = false;
|
||||
if (streamingAbortController.value) {
|
||||
streamingAbortController.value.abort();
|
||||
streamingAbortController.value = null;
|
||||
}
|
||||
|
||||
trackEndBuilderResponse(payload);
|
||||
currentStreamingMessage.value = undefined;
|
||||
}
|
||||
|
||||
function abortStreaming() {
|
||||
stopStreaming({ aborted: true });
|
||||
}
|
||||
|
||||
// Error handling
|
||||
@@ -144,10 +234,12 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
* The retry function, if provided, will remove the error message before retrying.
|
||||
* Tracks error telemetry
|
||||
*/
|
||||
function handleServiceError(e: unknown, id: string, retry?: () => Promise<void>) {
|
||||
function handleServiceError(e: unknown, userMessageId: string, retry?: () => Promise<void>) {
|
||||
assert(e instanceof Error);
|
||||
|
||||
stopStreaming();
|
||||
stopStreaming({
|
||||
error: e.message,
|
||||
});
|
||||
builderThinkingMessage.value = undefined;
|
||||
|
||||
if (e.name === 'AbortError') {
|
||||
@@ -163,17 +255,11 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
|
||||
const errorMessage = createErrorMessage(
|
||||
locale.baseText('aiAssistant.serviceError.message', { interpolate: { message: e.message } }),
|
||||
id,
|
||||
userMessageId,
|
||||
retry,
|
||||
);
|
||||
|
||||
chatMessages.value = [...chatMessages.value, errorMessage];
|
||||
|
||||
telemetry.track('Workflow generation errored', {
|
||||
error: e.message,
|
||||
session_id: trackingSessionId.value,
|
||||
workflow_id: workflowsStore.workflowId,
|
||||
});
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
@@ -203,6 +289,68 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
};
|
||||
}
|
||||
|
||||
// Telemetry functions
|
||||
/**
|
||||
* Tracks when a user submits a message to the builder.
|
||||
* Captures workflow state, execution data, and todo counts for analytics.
|
||||
*/
|
||||
function trackUserSubmittedBuilderMessage(options: {
|
||||
text: string;
|
||||
source: 'chat' | 'canvas';
|
||||
type: 'message' | 'execution';
|
||||
userMessageId: string;
|
||||
currentWorkflowJson: string;
|
||||
errorMessage?: string;
|
||||
errorNodeType?: string;
|
||||
executionStatus?: string;
|
||||
}) {
|
||||
const {
|
||||
text,
|
||||
source,
|
||||
type,
|
||||
userMessageId,
|
||||
currentWorkflowJson,
|
||||
errorMessage,
|
||||
errorNodeType,
|
||||
executionStatus,
|
||||
} = options;
|
||||
|
||||
const trackingPayload: UserSubmittedBuilderMessageTrackingPayload = {
|
||||
source,
|
||||
message: text,
|
||||
session_id: trackingSessionId.value,
|
||||
start_workflow_json: currentWorkflowJson,
|
||||
workflow_id: workflowsStore.workflowId,
|
||||
type,
|
||||
manual_exec_success_count_since_prev_msg: manualExecStatsInBetweenMessages.value.success,
|
||||
manual_exec_error_count_since_prev_msg: manualExecStatsInBetweenMessages.value.error,
|
||||
user_message_id: userMessageId,
|
||||
...getTodosToTrack(),
|
||||
};
|
||||
|
||||
if (type === 'execution') {
|
||||
let resultData = '{}';
|
||||
let resultDataSizeKb = 0;
|
||||
|
||||
try {
|
||||
resultData = JSON.stringify(workflowsStore.workflowExecutionData ?? {});
|
||||
resultDataSizeKb = stringSizeInBytes(resultData) / 1024;
|
||||
} catch (error) {
|
||||
// Handle circular structure errors gracefully
|
||||
console.warn('Failed to stringify execution data for telemetry:', error);
|
||||
}
|
||||
|
||||
trackingPayload.execution_data = resultDataSizeKb > 512 ? '{}' : resultData;
|
||||
trackingPayload.execution_status = executionStatus ?? '';
|
||||
if (executionStatus === 'error') {
|
||||
trackingPayload.error_message = errorMessage ?? '';
|
||||
trackingPayload.error_node_type = errorNodeType ?? '';
|
||||
}
|
||||
}
|
||||
|
||||
telemetry.track('User submitted builder message', trackingPayload);
|
||||
}
|
||||
|
||||
// Core API functions
|
||||
/**
|
||||
* Sends a message to the AI builder service and handles the streaming response.
|
||||
@@ -242,51 +390,38 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
if (options.initialGeneration !== undefined) {
|
||||
initialGeneration.value = options.initialGeneration;
|
||||
}
|
||||
const messageId = generateMessageId();
|
||||
|
||||
const userMessageId = generateMessageId();
|
||||
const currentWorkflowJson = getWorkflowSnapshot();
|
||||
const trackingPayload: Record<string, string> = {
|
||||
source,
|
||||
message: text,
|
||||
session_id: trackingSessionId.value,
|
||||
start_workflow_json: currentWorkflowJson,
|
||||
workflow_id: workflowsStore.workflowId,
|
||||
type,
|
||||
|
||||
currentStreamingMessage.value = {
|
||||
userMessageId,
|
||||
startWorkflowJson: currentWorkflowJson,
|
||||
};
|
||||
|
||||
if (type === 'execution') {
|
||||
let resultData = '{}';
|
||||
let resultDataSizeKb = 0;
|
||||
trackUserSubmittedBuilderMessage({
|
||||
text,
|
||||
source,
|
||||
type,
|
||||
userMessageId,
|
||||
currentWorkflowJson,
|
||||
errorMessage,
|
||||
errorNodeType,
|
||||
executionStatus,
|
||||
});
|
||||
|
||||
try {
|
||||
resultData = JSON.stringify(workflowsStore.workflowExecutionData ?? {});
|
||||
resultDataSizeKb = stringSizeInBytes(resultData) / 1024;
|
||||
} catch (error) {
|
||||
// Handle circular structure errors gracefully
|
||||
console.warn('Failed to stringify execution data for telemetry:', error);
|
||||
}
|
||||
resetManualExecutionStats();
|
||||
|
||||
trackingPayload.execution_data = resultDataSizeKb > 512 ? '{}' : resultData;
|
||||
trackingPayload.execution_status = executionStatus ?? '';
|
||||
if (executionStatus === 'error') {
|
||||
trackingPayload.error_message = errorMessage ?? '';
|
||||
trackingPayload.error_node_type = errorNodeType ?? '';
|
||||
}
|
||||
}
|
||||
|
||||
telemetry.track('User submitted builder message', trackingPayload);
|
||||
|
||||
prepareForStreaming(text, messageId);
|
||||
prepareForStreaming(text, userMessageId);
|
||||
|
||||
const executionResult = workflowsStore.workflowExecutionData?.data?.resultData;
|
||||
const payload = createBuilderPayload(text, {
|
||||
const payload = createBuilderPayload(text, userMessageId, {
|
||||
quickReplyType,
|
||||
workflow: workflowsStore.workflow,
|
||||
executionData: executionResult,
|
||||
nodesForSchema: Object.keys(workflowsStore.nodesByName),
|
||||
});
|
||||
|
||||
const retry = createRetryHandler(messageId, async () => sendChatMessage(options));
|
||||
const retry = createRetryHandler(userMessageId, async () => sendChatMessage(options));
|
||||
|
||||
// Abort previous streaming request if any
|
||||
if (streamingAbortController.value) {
|
||||
@@ -302,7 +437,7 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
const result = processAssistantMessages(
|
||||
chatMessages.value,
|
||||
response.messages,
|
||||
generateMessageId(),
|
||||
userMessageId,
|
||||
retry,
|
||||
);
|
||||
chatMessages.value = result.messages;
|
||||
@@ -315,11 +450,11 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
}
|
||||
},
|
||||
() => stopStreaming(),
|
||||
(e) => handleServiceError(e, messageId, retry),
|
||||
(e) => handleServiceError(e, userMessageId, retry),
|
||||
streamingAbortController.value?.signal,
|
||||
);
|
||||
} catch (e: unknown) {
|
||||
handleServiceError(e, messageId, retry);
|
||||
handleServiceError(e, userMessageId, retry);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,9 +709,10 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
creditsRemaining,
|
||||
hasNoCreditsRemaining,
|
||||
hasMessages: computed(() => hasMessages.value),
|
||||
workflowTodos,
|
||||
|
||||
// Methods
|
||||
stopStreaming,
|
||||
abortStreaming,
|
||||
resetBuilderChat,
|
||||
sendChatMessage,
|
||||
loadSessions,
|
||||
@@ -586,5 +722,7 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => {
|
||||
updateBuilderCredits,
|
||||
getRunningTools,
|
||||
fetchSessionsMetadata,
|
||||
incrementManualExecutionStats,
|
||||
resetManualExecutionStats,
|
||||
};
|
||||
});
|
||||
|
||||
@@ -8,12 +8,17 @@ import {
|
||||
import type { IRunExecutionData } from 'n8n-workflow';
|
||||
import type { IWorkflowDb } from '@/Interface';
|
||||
|
||||
export function generateShortId() {
|
||||
return Math.random().toString(36).substr(2, 9);
|
||||
}
|
||||
|
||||
export function generateMessageId(): string {
|
||||
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
||||
return `${Date.now()}-${generateShortId()}`;
|
||||
}
|
||||
|
||||
export function createBuilderPayload(
|
||||
text: string,
|
||||
id: string,
|
||||
options: {
|
||||
quickReplyType?: string;
|
||||
executionData?: IRunExecutionData['resultData'];
|
||||
@@ -67,6 +72,7 @@ export function createBuilderPayload(
|
||||
return {
|
||||
role: 'user',
|
||||
type: 'message',
|
||||
id,
|
||||
text,
|
||||
quickReplyType: options.quickReplyType,
|
||||
workflowContext,
|
||||
|
||||
-243
@@ -1354,249 +1354,6 @@ describe('AskAssistantBuild', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('should track categorization telemetry when categorize_prompt tool completes', async () => {
|
||||
renderComponent();
|
||||
|
||||
// Simulate streaming starts
|
||||
builderStore.$patch({ streaming: true });
|
||||
await flushPromises();
|
||||
|
||||
// Add categorization tool message
|
||||
builderStore.toolMessages = [
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'categorize_prompt',
|
||||
toolCallId: faker.string.uuid(),
|
||||
status: 'completed',
|
||||
updates: [
|
||||
{
|
||||
type: 'output',
|
||||
data: {
|
||||
categorization: {
|
||||
techniques: ['chatbot', 'notification'],
|
||||
confidence: 0.85,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// Simulate streaming stops (this triggers trackWorkflowModifications)
|
||||
builderStore.$patch({ streaming: false });
|
||||
await flushPromises();
|
||||
|
||||
expect(trackMock).toHaveBeenCalledWith('Classifier labels user prompt', {
|
||||
user_id: undefined,
|
||||
workflow_id: 'abc123',
|
||||
classifier_labels: ['chatbot', 'notification'],
|
||||
confidence: 0.85,
|
||||
session_id: 'app_session_id',
|
||||
timestamp: expect.any(String),
|
||||
});
|
||||
});
|
||||
|
||||
it('should handle multiple canvas generations correctly', async () => {
|
||||
const originalWorkflow = {
|
||||
nodes: [],
|
||||
connections: {},
|
||||
};
|
||||
builderStore.getWorkflowSnapshot.mockReturnValue(JSON.stringify(originalWorkflow));
|
||||
|
||||
const intermediaryWorkflow = {
|
||||
nodes: [
|
||||
{
|
||||
id: 'node1',
|
||||
name: 'Start',
|
||||
type: 'n8n-nodes-base.start',
|
||||
position: [0, 0],
|
||||
typeVersion: 1,
|
||||
parameters: {},
|
||||
} as INodeUi,
|
||||
],
|
||||
connections: {},
|
||||
};
|
||||
|
||||
const finalWorkflow = {
|
||||
nodes: [
|
||||
{
|
||||
id: 'node1',
|
||||
name: 'Start',
|
||||
type: 'n8n-nodes-base.start',
|
||||
position: [0, 0],
|
||||
typeVersion: 1,
|
||||
parameters: {},
|
||||
} as INodeUi,
|
||||
|
||||
{
|
||||
id: 'node2',
|
||||
name: 'HttpReuqest',
|
||||
type: 'n8n-nodes-base.httpRequest',
|
||||
position: [0, 0],
|
||||
typeVersion: 1,
|
||||
parameters: {},
|
||||
} as INodeUi,
|
||||
],
|
||||
connections: {},
|
||||
};
|
||||
workflowsStore.$patch({
|
||||
workflow: originalWorkflow,
|
||||
});
|
||||
|
||||
renderComponent();
|
||||
|
||||
builderStore.$patch({ streaming: true });
|
||||
await flushPromises();
|
||||
|
||||
// Trigger the watcher by updating workflowMessages
|
||||
builderStore.workflowMessages = [
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'workflow-updated' as const,
|
||||
codeSnippet: JSON.stringify(intermediaryWorkflow),
|
||||
},
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'workflow-updated' as const,
|
||||
codeSnippet: JSON.stringify(finalWorkflow),
|
||||
},
|
||||
];
|
||||
|
||||
const toolCallId1_1 = '1234';
|
||||
const toolCallId1_2 = '3333';
|
||||
const toolCallId2 = '4567';
|
||||
const toolCallId3 = '8901';
|
||||
|
||||
builderStore.toolMessages = [
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'first-tool',
|
||||
toolCallId: toolCallId1_1,
|
||||
status: 'completed',
|
||||
updates: [],
|
||||
},
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'first-tool',
|
||||
toolCallId: toolCallId1_2,
|
||||
status: 'completed',
|
||||
updates: [],
|
||||
},
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'second-tool',
|
||||
toolCallId: toolCallId2,
|
||||
status: 'running',
|
||||
updates: [],
|
||||
},
|
||||
];
|
||||
|
||||
builderStore.$patch({ streaming: false });
|
||||
await flushPromises();
|
||||
|
||||
expect(trackMock).toHaveBeenCalledOnce();
|
||||
expect(trackMock).toHaveBeenCalledWith('Workflow modified by builder', {
|
||||
end_workflow_json: JSON.stringify(finalWorkflow),
|
||||
session_id: 'app_session_id',
|
||||
start_workflow_json: JSON.stringify(originalWorkflow),
|
||||
// first-tool is added once, even though it completed twice
|
||||
// second-tool is ignored because it's running
|
||||
tools_called: ['first-tool'],
|
||||
workflow_id: 'abc123',
|
||||
});
|
||||
|
||||
trackMock.mockClear();
|
||||
|
||||
builderStore.$patch({ streaming: true });
|
||||
|
||||
await flushPromises();
|
||||
// second run after new messages
|
||||
const updatedWorkflow2 = {
|
||||
...finalWorkflow,
|
||||
nodes: [
|
||||
...finalWorkflow.nodes,
|
||||
{
|
||||
id: 'node1',
|
||||
name: 'Updated',
|
||||
type: 'n8n-nodes-base.updated',
|
||||
position: [0, 0],
|
||||
typeVersion: 1,
|
||||
parameters: {},
|
||||
},
|
||||
],
|
||||
};
|
||||
builderStore.workflowMessages = [
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'workflow-updated' as const,
|
||||
codeSnippet: JSON.stringify(updatedWorkflow2),
|
||||
},
|
||||
];
|
||||
|
||||
builderStore.toolMessages = [
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'first-tool',
|
||||
toolCallId: toolCallId1_1,
|
||||
status: 'completed',
|
||||
updates: [],
|
||||
},
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'first-tool',
|
||||
toolCallId: toolCallId1_2,
|
||||
status: 'completed',
|
||||
updates: [],
|
||||
},
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'second-tool',
|
||||
toolCallId: toolCallId2,
|
||||
status: 'completed',
|
||||
updates: [],
|
||||
},
|
||||
{
|
||||
id: faker.string.uuid(),
|
||||
role: 'assistant' as const,
|
||||
type: 'tool' as const,
|
||||
toolName: 'third-tool',
|
||||
toolCallId: toolCallId3,
|
||||
status: 'completed',
|
||||
updates: [],
|
||||
},
|
||||
];
|
||||
|
||||
builderStore.$patch({ streaming: false });
|
||||
await flushPromises();
|
||||
|
||||
expect(trackMock).toHaveBeenCalledOnce();
|
||||
expect(trackMock).toHaveBeenCalledWith('Workflow modified by builder', {
|
||||
end_workflow_json: JSON.stringify(updatedWorkflow2),
|
||||
session_id: 'app_session_id',
|
||||
start_workflow_json: JSON.stringify(originalWorkflow),
|
||||
// first-tool is ignored, because it was tracked in first run (same tool call id)
|
||||
tools_called: ['second-tool', 'third-tool'],
|
||||
workflow_id: 'abc123',
|
||||
});
|
||||
});
|
||||
|
||||
describe('shouldTidyUp logic', () => {
|
||||
it('should set tidyUp to true when new nodes are added', async () => {
|
||||
const originalWorkflow = { nodes: [], connections: {} };
|
||||
|
||||
+1
-84
@@ -33,9 +33,6 @@ const { goToUpgrade } = usePageRedirectionHelper();
|
||||
|
||||
// Track processed workflow updates
|
||||
const processedWorkflowUpdates = ref(new Set<string>());
|
||||
const trackedTools = ref(new Set<string>());
|
||||
const trackedCategorizations = ref(new Set<string>());
|
||||
const workflowUpdated = ref<{ start: string; end: string } | undefined>();
|
||||
const shouldTidyUp = ref(false);
|
||||
const n8nChatRef = ref<InstanceType<typeof N8nAskAssistantChat>>();
|
||||
|
||||
@@ -107,9 +104,6 @@ async function onUserMessage(content: string) {
|
||||
function onNewWorkflow() {
|
||||
builderStore.resetBuilderChat();
|
||||
processedWorkflowUpdates.value.clear();
|
||||
trackedTools.value.clear();
|
||||
trackedCategorizations.value.clear();
|
||||
workflowUpdated.value = undefined;
|
||||
shouldTidyUp.value = false;
|
||||
}
|
||||
|
||||
@@ -130,71 +124,6 @@ function onFeedback(feedback: RatingFeedback) {
|
||||
}
|
||||
}
|
||||
|
||||
function dedupeToolNames(toolNames: string[]): string[] {
|
||||
return [...new Set(toolNames)];
|
||||
}
|
||||
|
||||
function isCategorizationData(
|
||||
data: unknown,
|
||||
): data is { techniques: string[]; confidence?: number } {
|
||||
return (
|
||||
typeof data === 'object' &&
|
||||
data !== null &&
|
||||
'techniques' in data &&
|
||||
Array.isArray(data.techniques) &&
|
||||
data.techniques.every((t) => typeof t === 'string')
|
||||
);
|
||||
}
|
||||
|
||||
function trackWorkflowCategorization() {
|
||||
// Track categorization telemetry
|
||||
builderStore.toolMessages.forEach((toolMsg) => {
|
||||
if (toolMsg.toolName !== 'categorize_prompt') return;
|
||||
if (toolMsg.status !== 'completed') return;
|
||||
if (!toolMsg.toolCallId) return;
|
||||
if (trackedCategorizations.value.has(toolMsg.toolCallId)) return;
|
||||
|
||||
const outputUpdate = toolMsg.updates.find((u) => u.type === 'output');
|
||||
const categorizationData = outputUpdate?.data?.categorization;
|
||||
|
||||
if (!isCategorizationData(categorizationData)) return;
|
||||
|
||||
trackedCategorizations.value.add(toolMsg.toolCallId);
|
||||
|
||||
telemetry.track('Classifier labels user prompt', {
|
||||
user_id: usersStore.currentUserId ?? undefined,
|
||||
workflow_id: workflowsStore.workflowId,
|
||||
classifier_labels: categorizationData.techniques,
|
||||
confidence: categorizationData.confidence,
|
||||
session_id: builderStore.trackingSessionId,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function trackWorkflowModifications() {
|
||||
if (workflowUpdated.value) {
|
||||
// Track tool usage for telemetry
|
||||
const newToolMessages = builderStore.toolMessages.filter(
|
||||
(toolMsg) =>
|
||||
toolMsg.status !== 'running' &&
|
||||
toolMsg.toolCallId &&
|
||||
!trackedTools.value.has(toolMsg.toolCallId),
|
||||
);
|
||||
|
||||
newToolMessages.forEach((toolMsg) => trackedTools.value.add(toolMsg.toolCallId ?? ''));
|
||||
telemetry.track('Workflow modified by builder', {
|
||||
tools_called: dedupeToolNames(newToolMessages.map((toolMsg) => toolMsg.toolName)),
|
||||
session_id: builderStore.trackingSessionId,
|
||||
start_workflow_json: workflowUpdated.value.start,
|
||||
end_workflow_json: workflowUpdated.value.end,
|
||||
workflow_id: workflowsStore.workflowId,
|
||||
});
|
||||
|
||||
workflowUpdated.value = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function onWorkflowExecuted() {
|
||||
const executionData = workflowsStore.workflowExecutionData;
|
||||
const executionStatus = executionData?.status ?? 'unknown';
|
||||
@@ -257,8 +186,6 @@ watch(
|
||||
if (msg.id && isWorkflowUpdatedMessage(msg)) {
|
||||
processedWorkflowUpdates.value.add(msg.id);
|
||||
|
||||
const originalWorkflowJson =
|
||||
workflowUpdated.value?.start ?? builderStore.getWorkflowSnapshot();
|
||||
const result = builderStore.applyWorkflowUpdate(msg.codeSnippet);
|
||||
|
||||
if (result.success) {
|
||||
@@ -274,11 +201,6 @@ watch(
|
||||
regenerateIds: false,
|
||||
trackEvents: false,
|
||||
});
|
||||
|
||||
workflowUpdated.value = {
|
||||
start: originalWorkflowJson,
|
||||
end: msg.codeSnippet,
|
||||
};
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -291,11 +213,6 @@ watch(
|
||||
watch(
|
||||
() => builderStore.streaming,
|
||||
async (isStreaming) => {
|
||||
if (!isStreaming) {
|
||||
trackWorkflowModifications();
|
||||
trackWorkflowCategorization();
|
||||
}
|
||||
|
||||
if (
|
||||
builderStore.initialGeneration &&
|
||||
!isStreaming &&
|
||||
@@ -348,7 +265,7 @@ defineExpose({
|
||||
@message="onUserMessage"
|
||||
@upgrade-click="() => goToUpgrade('ai-builder-sidebar', 'upgrade-builder')"
|
||||
@feedback="onFeedback"
|
||||
@stop="builderStore.stopStreaming"
|
||||
@stop="builderStore.abortStreaming"
|
||||
>
|
||||
<template #header>
|
||||
<slot name="header" />
|
||||
|
||||
+2
-95
@@ -16,7 +16,6 @@ import { useToast } from '@/app/composables/useToast';
|
||||
import { N8nTooltip } from '@n8n/design-system';
|
||||
import { nextTick } from 'vue';
|
||||
import { useBuilderStore } from '@/features/ai/assistant/builder.store';
|
||||
import type { WorkflowValidationIssue } from '@/Interface';
|
||||
|
||||
interface Emits {
|
||||
/** Emitted when workflow execution completes */
|
||||
@@ -37,49 +36,6 @@ const builderStore = useBuilderStore();
|
||||
// Workflow execution composable
|
||||
const { runWorkflow } = useRunWorkflow({ router });
|
||||
|
||||
const PLACEHOLDER_PREFIX = '<__PLACEHOLDER_VALUE__';
|
||||
const PLACEHOLDER_SUFFIX = '__>';
|
||||
|
||||
interface PlaceholderDetail {
|
||||
path: string[];
|
||||
label: string;
|
||||
}
|
||||
|
||||
function extractPlaceholderLabel(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
if (!value.startsWith(PLACEHOLDER_PREFIX) || !value.endsWith(PLACEHOLDER_SUFFIX)) return null;
|
||||
|
||||
const label = value
|
||||
.slice(PLACEHOLDER_PREFIX.length, value.length - PLACEHOLDER_SUFFIX.length)
|
||||
.trim();
|
||||
return label.length > 0 ? label : null;
|
||||
}
|
||||
|
||||
function findPlaceholderDetails(value: unknown, path: string[] = []): PlaceholderDetail[] {
|
||||
const label = extractPlaceholderLabel(value);
|
||||
if (label) return [{ path, label }];
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item, index) => findPlaceholderDetails(item, [...path, `[${index}]`]));
|
||||
}
|
||||
|
||||
if (value !== null && typeof value === 'object') {
|
||||
return Object.entries(value).flatMap(([key, nested]) =>
|
||||
findPlaceholderDetails(nested, [...path, key]),
|
||||
);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function formatPlaceholderPath(path: string[]): string {
|
||||
if (path.length === 0) return 'parameters';
|
||||
|
||||
return path
|
||||
.map((segment, index) => (segment.startsWith('[') || index === 0 ? segment : `.${segment}`))
|
||||
.join('');
|
||||
}
|
||||
|
||||
let executionWatcherStop: WatchStopHandle | undefined;
|
||||
|
||||
const containerRef = ref<HTMLElement>();
|
||||
@@ -115,56 +71,7 @@ const ensureExecutionWatcher = () => {
|
||||
);
|
||||
};
|
||||
|
||||
// Workflow validation from store
|
||||
const baseWorkflowIssues = computed(() =>
|
||||
workflowsStore.workflowValidationIssues.filter((issue) =>
|
||||
['credentials', 'parameters'].includes(issue.type),
|
||||
),
|
||||
);
|
||||
|
||||
const placeholderIssues = computed(() => {
|
||||
const issues: WorkflowValidationIssue[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const node of workflowsStore.workflow.nodes) {
|
||||
if (!node?.parameters) continue;
|
||||
|
||||
const placeholders = findPlaceholderDetails(node.parameters);
|
||||
if (placeholders.length === 0) continue;
|
||||
|
||||
const existingParameterIssues = node.issues?.parameters ?? {};
|
||||
|
||||
for (const placeholder of placeholders) {
|
||||
const path = formatPlaceholderPath(placeholder.path);
|
||||
const message = i18n.baseText('aiAssistant.builder.executeMessage.fillParameter', {
|
||||
interpolate: { label: placeholder.label },
|
||||
});
|
||||
const rawMessages = existingParameterIssues[path];
|
||||
const existingMessages = rawMessages
|
||||
? Array.isArray(rawMessages)
|
||||
? rawMessages
|
||||
: [rawMessages]
|
||||
: [];
|
||||
|
||||
if (existingMessages.includes(message)) continue;
|
||||
|
||||
const key = `${node.name}|${path}|${placeholder.label}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
issues.push({
|
||||
node: node.name,
|
||||
type: 'parameters',
|
||||
value: message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
});
|
||||
|
||||
const workflowIssues = computed(() => [...baseWorkflowIssues.value, ...placeholderIssues.value]);
|
||||
const hasValidationIssues = computed(() => workflowIssues.value.length > 0);
|
||||
const hasValidationIssues = computed(() => builderStore.workflowTodos.length > 0);
|
||||
const formatIssueMessage = workflowsStore.formatIssueMessage;
|
||||
|
||||
const triggerNodes = computed(() =>
|
||||
@@ -261,7 +168,7 @@ onBeforeUnmount(() => {
|
||||
aria-label="Workflow validation issues"
|
||||
>
|
||||
<NodeIssueItem
|
||||
v-for="issue in workflowIssues"
|
||||
v-for="issue in builderStore.workflowTodos"
|
||||
:key="`${formatIssueMessage(issue.value)}_${issue.node}`"
|
||||
:issue="issue"
|
||||
:get-node-type="getNodeTypeByName"
|
||||
|
||||
+14
-9
@@ -10,6 +10,11 @@ vi.mock('@n8n/i18n', () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
// Mock generateShortId to return a predictable value for testing
|
||||
vi.mock('../builder.utils', () => ({
|
||||
generateShortId: () => 'abc123',
|
||||
}));
|
||||
|
||||
describe('useBuilderMessages', () => {
|
||||
let builderMessages: ReturnType<typeof useBuilderMessages>;
|
||||
|
||||
@@ -36,7 +41,7 @@ describe('useBuilderMessages', () => {
|
||||
|
||||
expect(result.messages).toHaveLength(1);
|
||||
expect(result.messages[0]).toMatchObject({
|
||||
id: 'test-id-0',
|
||||
id: 'test-id--abc123--0',
|
||||
role: 'assistant',
|
||||
type: 'text',
|
||||
content: 'Hello, how can I help?',
|
||||
@@ -72,7 +77,7 @@ describe('useBuilderMessages', () => {
|
||||
expect(result.messages).toHaveLength(1);
|
||||
const toolMessage = result.messages[0] as ChatUI.ToolMessage;
|
||||
expect(toolMessage).toMatchObject({
|
||||
id: 'call-123', // Should use toolCallId as ID
|
||||
id: 'test-id--abc123--0-call-123', // Format is messageId-toolCallId
|
||||
role: 'assistant',
|
||||
type: 'tool',
|
||||
toolName: 'add_nodes',
|
||||
@@ -164,7 +169,7 @@ describe('useBuilderMessages', () => {
|
||||
|
||||
expect(result.messages).toHaveLength(1);
|
||||
expect(result.messages[0]).toMatchObject({
|
||||
id: 'test-id-0', // Should fall back to generated ID
|
||||
id: 'test-id--abc123--0-undefined', // Format is messageId-toolCallId (undefined when missing)
|
||||
type: 'tool',
|
||||
toolName: 'some_tool',
|
||||
});
|
||||
@@ -224,7 +229,7 @@ describe('useBuilderMessages', () => {
|
||||
|
||||
expect(result.messages).toHaveLength(1);
|
||||
expect(result.messages[0]).toMatchObject({
|
||||
id: 'test-id-0',
|
||||
id: 'test-id--abc123--0',
|
||||
type: 'workflow-updated',
|
||||
codeSnippet: '{"nodes": [], "connections": {}}',
|
||||
read: false,
|
||||
@@ -262,11 +267,11 @@ describe('useBuilderMessages', () => {
|
||||
|
||||
expect(result.messages).toHaveLength(3);
|
||||
expect(result.messages[0].type).toBe('text');
|
||||
expect(result.messages[0].id).toBe('batch-id-0');
|
||||
expect(result.messages[0].id).toBe('batch-id--abc123--0');
|
||||
expect(result.messages[1].type).toBe('tool');
|
||||
expect(result.messages[1].id).toBe('call-123'); // Uses toolCallId
|
||||
expect(result.messages[1].id).toBe('batch-id--abc123--1-call-123'); // Format is messageId-toolCallId
|
||||
expect(result.messages[2].type).toBe('workflow-updated');
|
||||
expect(result.messages[2].id).toBe('batch-id-2');
|
||||
expect(result.messages[2].id).toBe('batch-id--abc123--2');
|
||||
});
|
||||
|
||||
it('should show tool name when tool is in progress with displayTitle', () => {
|
||||
@@ -988,7 +993,7 @@ describe('useBuilderMessages', () => {
|
||||
);
|
||||
|
||||
expect(result.messages).toHaveLength(1);
|
||||
expect(result.messages[0].id).toBe('test-id-0');
|
||||
expect(result.messages[0].id).toBe('test-id--abc123--0-undefined'); // Format is messageId-toolCallId (undefined when missing)
|
||||
});
|
||||
|
||||
it('should handle workflow-updated messages with invalid JSON', () => {
|
||||
@@ -1526,7 +1531,7 @@ describe('useBuilderMessages', () => {
|
||||
expect(result.messages).toHaveLength(1);
|
||||
const errorMessage = result.messages[0] as ChatUI.ErrorMessage;
|
||||
expect(errorMessage).toMatchObject({
|
||||
id: 'test-id-0',
|
||||
id: 'test-id--abc123--0',
|
||||
role: 'assistant',
|
||||
type: 'error',
|
||||
content: 'Something went wrong',
|
||||
|
||||
+9
-7
@@ -2,6 +2,7 @@ import type { ChatUI } from '@n8n/design-system/types/assistant';
|
||||
import type { ChatRequest } from '../assistant.types';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
import { isTextMessage, isWorkflowUpdatedMessage, isToolMessage } from '../assistant.types';
|
||||
import { generateShortId } from '../builder.utils';
|
||||
|
||||
export interface MessageProcessingResult {
|
||||
messages: ChatUI.AssistantMessage[];
|
||||
@@ -98,9 +99,6 @@ export function useBuilderMessages() {
|
||||
msg: ChatRequest.ToolMessage,
|
||||
messageId: string,
|
||||
): void {
|
||||
// Use toolCallId as the message ID for consistency across updates
|
||||
const toolMessageId = msg.toolCallId ?? messageId;
|
||||
|
||||
// Check if we already have this tool message
|
||||
const existingIndex = msg.toolCallId
|
||||
? messages.findIndex((m) => m.type === 'tool' && m.toolCallId === msg.toolCallId)
|
||||
@@ -111,6 +109,7 @@ export function useBuilderMessages() {
|
||||
const existing = messages[existingIndex] as ChatUI.ToolMessage;
|
||||
const toolMessage: ChatUI.ToolMessage = {
|
||||
...existing,
|
||||
id: `${messageId}-${msg.toolCallId}`,
|
||||
status: msg.status,
|
||||
updates: [...(existing.updates || []), ...(msg.updates || [])],
|
||||
};
|
||||
@@ -118,7 +117,7 @@ export function useBuilderMessages() {
|
||||
} else {
|
||||
// Add new tool message
|
||||
const toolMessage: ChatUI.AssistantMessage = {
|
||||
id: toolMessageId,
|
||||
id: `${messageId}-${msg.toolCallId}`,
|
||||
role: 'assistant',
|
||||
type: 'tool',
|
||||
toolName: msg.toolName,
|
||||
@@ -269,15 +268,18 @@ export function useBuilderMessages() {
|
||||
function processAssistantMessages(
|
||||
currentMessages: ChatUI.AssistantMessage[],
|
||||
newMessages: ChatRequest.MessageResponse[],
|
||||
baseId: string,
|
||||
userMessageId: string,
|
||||
retry?: () => Promise<void>,
|
||||
): MessageProcessingResult {
|
||||
const mutableMessages = [...currentMessages];
|
||||
let shouldClearThinking = false;
|
||||
|
||||
const messageGroupId = generateShortId();
|
||||
|
||||
newMessages.forEach((msg, index) => {
|
||||
// Generate unique ID for each message in the batch
|
||||
const messageId = `${baseId}-${index}`;
|
||||
// Generate unique ID for each message in the batch, based on original user message id.
|
||||
// Used in telemetry to track events related to a specific user message
|
||||
const messageId = `${userMessageId}--${messageGroupId}--${index}`;
|
||||
const clearThinking = processSingleMessage(mutableMessages, msg, messageId, retry);
|
||||
shouldClearThinking = shouldClearThinking || clearThinking;
|
||||
});
|
||||
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
extractPlaceholderLabel,
|
||||
findPlaceholderDetails,
|
||||
formatPlaceholderPath,
|
||||
} from './useBuilderTodos';
|
||||
|
||||
describe('useBuilderTodos', () => {
|
||||
describe('extractPlaceholderLabel', () => {
|
||||
it('returns null for non-string values', () => {
|
||||
expect(extractPlaceholderLabel(123)).toBeNull();
|
||||
expect(extractPlaceholderLabel(true)).toBeNull();
|
||||
expect(extractPlaceholderLabel(null)).toBeNull();
|
||||
expect(extractPlaceholderLabel(undefined)).toBeNull();
|
||||
expect(extractPlaceholderLabel({})).toBeNull();
|
||||
expect(extractPlaceholderLabel([])).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for strings without placeholder format', () => {
|
||||
expect(extractPlaceholderLabel('regular string')).toBeNull();
|
||||
expect(extractPlaceholderLabel('https://example.com')).toBeNull();
|
||||
expect(extractPlaceholderLabel('')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for partial placeholder format', () => {
|
||||
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__missing end')).toBeNull();
|
||||
expect(extractPlaceholderLabel('PLACEHOLDER__test__>')).toBeNull();
|
||||
expect(extractPlaceholderLabel('__PLACEHOLDER_VALUE__test__>')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for empty label', () => {
|
||||
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE____>')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for whitespace-only label', () => {
|
||||
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__ __>')).toBeNull();
|
||||
});
|
||||
|
||||
it('extracts label from valid placeholder', () => {
|
||||
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__Enter URL__>')).toBe('Enter URL');
|
||||
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__API Key__>')).toBe('API Key');
|
||||
});
|
||||
|
||||
it('trims whitespace from label', () => {
|
||||
expect(extractPlaceholderLabel('<__PLACEHOLDER_VALUE__ Enter URL __>')).toBe('Enter URL');
|
||||
});
|
||||
});
|
||||
|
||||
describe('findPlaceholderDetails', () => {
|
||||
it('returns empty array for primitive non-placeholder values', () => {
|
||||
expect(findPlaceholderDetails('regular string')).toEqual([]);
|
||||
expect(findPlaceholderDetails(123)).toEqual([]);
|
||||
expect(findPlaceholderDetails(true)).toEqual([]);
|
||||
expect(findPlaceholderDetails(null)).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array for empty object', () => {
|
||||
expect(findPlaceholderDetails({})).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array for empty array', () => {
|
||||
expect(findPlaceholderDetails([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('finds placeholder at root level', () => {
|
||||
const result = findPlaceholderDetails('<__PLACEHOLDER_VALUE__Enter URL__>');
|
||||
expect(result).toEqual([{ path: [], label: 'Enter URL' }]);
|
||||
});
|
||||
|
||||
it('finds placeholder in simple object', () => {
|
||||
const result = findPlaceholderDetails({
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
});
|
||||
expect(result).toEqual([{ path: ['url'], label: 'Enter URL' }]);
|
||||
});
|
||||
|
||||
it('finds multiple placeholders in object', () => {
|
||||
const result = findPlaceholderDetails({
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
body: '<__PLACEHOLDER_VALUE__Enter Body__>',
|
||||
});
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result).toContainEqual({ path: ['url'], label: 'Enter URL' });
|
||||
expect(result).toContainEqual({ path: ['body'], label: 'Enter Body' });
|
||||
});
|
||||
|
||||
it('finds placeholder in nested object', () => {
|
||||
const result = findPlaceholderDetails({
|
||||
options: {
|
||||
headers: {
|
||||
authorization: '<__PLACEHOLDER_VALUE__Enter API Key__>',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result).toEqual([
|
||||
{ path: ['options', 'headers', 'authorization'], label: 'Enter API Key' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('finds placeholder in array', () => {
|
||||
const result = findPlaceholderDetails([
|
||||
'regular value',
|
||||
'<__PLACEHOLDER_VALUE__Enter Value__>',
|
||||
]);
|
||||
expect(result).toEqual([{ path: ['[1]'], label: 'Enter Value' }]);
|
||||
});
|
||||
|
||||
it('finds placeholder in array of objects', () => {
|
||||
const result = findPlaceholderDetails({
|
||||
headers: [
|
||||
{ name: 'Content-Type', value: 'application/json' },
|
||||
{ name: 'Authorization', value: '<__PLACEHOLDER_VALUE__Enter Token__>' },
|
||||
],
|
||||
});
|
||||
expect(result).toEqual([{ path: ['headers', '[1]', 'value'], label: 'Enter Token' }]);
|
||||
});
|
||||
|
||||
it('finds placeholders in mixed structure', () => {
|
||||
const result = findPlaceholderDetails({
|
||||
url: '<__PLACEHOLDER_VALUE__Enter URL__>',
|
||||
options: {
|
||||
items: [{ key: '<__PLACEHOLDER_VALUE__Enter Key__>' }, { value: 'static' }],
|
||||
},
|
||||
});
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result).toContainEqual({ path: ['url'], label: 'Enter URL' });
|
||||
expect(result).toContainEqual({
|
||||
path: ['options', 'items', '[0]', 'key'],
|
||||
label: 'Enter Key',
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores non-placeholder strings in object', () => {
|
||||
const result = findPlaceholderDetails({
|
||||
url: 'https://example.com',
|
||||
method: 'GET',
|
||||
placeholder: '<__PLACEHOLDER_VALUE__Enter Value__>',
|
||||
});
|
||||
expect(result).toEqual([{ path: ['placeholder'], label: 'Enter Value' }]);
|
||||
});
|
||||
|
||||
it('handles custom starting path', () => {
|
||||
const result = findPlaceholderDetails({ url: '<__PLACEHOLDER_VALUE__Enter URL__>' }, [
|
||||
'parameters',
|
||||
]);
|
||||
expect(result).toEqual([{ path: ['parameters', 'url'], label: 'Enter URL' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatPlaceholderPath', () => {
|
||||
it('returns "parameters" for empty path', () => {
|
||||
expect(formatPlaceholderPath([])).toBe('parameters');
|
||||
});
|
||||
|
||||
it('formats single segment path', () => {
|
||||
expect(formatPlaceholderPath(['url'])).toBe('url');
|
||||
});
|
||||
|
||||
it('formats multi-segment path with dot notation', () => {
|
||||
expect(formatPlaceholderPath(['options', 'headers', 'authorization'])).toBe(
|
||||
'options.headers.authorization',
|
||||
);
|
||||
});
|
||||
|
||||
it('formats path with array indices without leading dot', () => {
|
||||
expect(formatPlaceholderPath(['headers', '[0]', 'value'])).toBe('headers[0].value');
|
||||
});
|
||||
|
||||
it('formats path starting with array index', () => {
|
||||
expect(formatPlaceholderPath(['[0]', 'key'])).toBe('[0].key');
|
||||
});
|
||||
|
||||
it('formats path with multiple array indices', () => {
|
||||
expect(formatPlaceholderPath(['items', '[0]', 'options', '[1]', 'value'])).toBe(
|
||||
'items[0].options[1].value',
|
||||
);
|
||||
});
|
||||
|
||||
it('formats path with consecutive array indices', () => {
|
||||
expect(formatPlaceholderPath(['matrix', '[0]', '[1]'])).toBe('matrix[0][1]');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,164 @@
|
||||
import { computed } from 'vue';
|
||||
import { useI18n } from '@n8n/i18n';
|
||||
import { useWorkflowsStore } from '@/app/stores/workflows.store';
|
||||
import type { WorkflowValidationIssue } from '@/Interface';
|
||||
|
||||
const PLACEHOLDER_PREFIX = '<__PLACEHOLDER_VALUE__';
|
||||
const PLACEHOLDER_SUFFIX = '__>';
|
||||
|
||||
export interface PlaceholderDetail {
|
||||
path: string[];
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface TodoTrackingItem {
|
||||
type: string;
|
||||
node_type: string | undefined;
|
||||
label: string | string[];
|
||||
}
|
||||
|
||||
export interface TodosTrackingPayload {
|
||||
credentials_todo_count: number;
|
||||
placeholders_todo_count: number;
|
||||
todos: TodoTrackingItem[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the label from a placeholder value string.
|
||||
* Returns null if the value is not a valid placeholder.
|
||||
*/
|
||||
export function extractPlaceholderLabel(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
if (!value.startsWith(PLACEHOLDER_PREFIX) || !value.endsWith(PLACEHOLDER_SUFFIX)) return null;
|
||||
|
||||
const label = value
|
||||
.slice(PLACEHOLDER_PREFIX.length, value.length - PLACEHOLDER_SUFFIX.length)
|
||||
.trim();
|
||||
return label.length > 0 ? label : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively searches through a value (object, array, or primitive) to find
|
||||
* all placeholder values and their paths.
|
||||
*/
|
||||
export function findPlaceholderDetails(value: unknown, path: string[] = []): PlaceholderDetail[] {
|
||||
const label = extractPlaceholderLabel(value);
|
||||
if (label) return [{ path, label }];
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item, index) => findPlaceholderDetails(item, [...path, `[${index}]`]));
|
||||
}
|
||||
|
||||
if (value !== null && typeof value === 'object') {
|
||||
return Object.entries(value).flatMap(([key, nested]) =>
|
||||
findPlaceholderDetails(nested, [...path, key]),
|
||||
);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a path array into a dot-notation string for display.
|
||||
* Array indices are preserved as [N] without leading dots.
|
||||
*/
|
||||
export function formatPlaceholderPath(path: string[]): string {
|
||||
if (path.length === 0) return 'parameters';
|
||||
|
||||
return path
|
||||
.map((segment, index) => (segment.startsWith('[') || index === 0 ? segment : `.${segment}`))
|
||||
.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Composable for managing workflow todos (validation issues and placeholders)
|
||||
* used by the AI builder.
|
||||
*/
|
||||
export function useBuilderTodos() {
|
||||
const workflowsStore = useWorkflowsStore();
|
||||
const locale = useI18n();
|
||||
|
||||
/**
|
||||
* Base workflow validation issues filtered to only credentials and parameters types.
|
||||
*/
|
||||
const baseWorkflowIssues = computed(() =>
|
||||
workflowsStore.workflowValidationIssues.filter((issue) =>
|
||||
['credentials', 'parameters'].includes(issue.type),
|
||||
),
|
||||
);
|
||||
|
||||
/**
|
||||
* Placeholder issues detected in workflow node parameters.
|
||||
* These are values with the format <__PLACEHOLDER_VALUE__label__>.
|
||||
*/
|
||||
const placeholderIssues = computed(() => {
|
||||
const issues: WorkflowValidationIssue[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const node of workflowsStore.workflow.nodes) {
|
||||
if (!node?.parameters) continue;
|
||||
|
||||
const placeholders = findPlaceholderDetails(node.parameters);
|
||||
if (placeholders.length === 0) continue;
|
||||
|
||||
const existingParameterIssues = node.issues?.parameters ?? {};
|
||||
|
||||
for (const placeholder of placeholders) {
|
||||
const path = formatPlaceholderPath(placeholder.path);
|
||||
const message = locale.baseText('aiAssistant.builder.executeMessage.fillParameter', {
|
||||
interpolate: { label: placeholder.label },
|
||||
});
|
||||
const rawMessages = existingParameterIssues[path];
|
||||
const existingMessages = rawMessages
|
||||
? Array.isArray(rawMessages)
|
||||
? rawMessages
|
||||
: [rawMessages]
|
||||
: [];
|
||||
|
||||
if (existingMessages.includes(message)) continue;
|
||||
|
||||
const key = `${node.name}|${path}|${placeholder.label}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
issues.push({
|
||||
node: node.name,
|
||||
type: 'parameters',
|
||||
value: message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
});
|
||||
|
||||
/**
|
||||
* Combined list of all workflow todos (base issues + placeholder issues).
|
||||
*/
|
||||
const workflowTodos = computed(() => [...baseWorkflowIssues.value, ...placeholderIssues.value]);
|
||||
|
||||
/**
|
||||
* Returns todos data formatted for telemetry tracking.
|
||||
*/
|
||||
function getTodosToTrack(): TodosTrackingPayload {
|
||||
const credentials_todo_count = workflowsStore.workflowValidationIssues.filter(
|
||||
(issue) => issue.type === 'credentials',
|
||||
).length;
|
||||
const placeholders_todo_count = placeholderIssues.value.length;
|
||||
return {
|
||||
credentials_todo_count,
|
||||
placeholders_todo_count,
|
||||
todos: workflowTodos.value.map((todo) => ({
|
||||
type: todo.type,
|
||||
node_type: workflowsStore.getNodeByName(todo.node)?.type,
|
||||
label: todo.value,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
workflowTodos,
|
||||
placeholderIssues,
|
||||
getTodosToTrack,
|
||||
};
|
||||
}
|
||||
@@ -51,6 +51,7 @@ export {
|
||||
setSafeObjectProperty,
|
||||
isDomainAllowed,
|
||||
isCommunityPackageName,
|
||||
dedupe,
|
||||
} from './utils';
|
||||
export {
|
||||
isINodeProperties,
|
||||
|
||||
@@ -438,3 +438,7 @@ export function isCommunityPackageName(packageName: string): boolean {
|
||||
|
||||
return !!nameMatch;
|
||||
}
|
||||
|
||||
export function dedupe<T>(arr: T[]): T[] {
|
||||
return [...new Set(arr)];
|
||||
}
|
||||
|
||||
@@ -42,10 +42,7 @@ import * as NodeHelpers from './node-helpers';
|
||||
import { renameFormFields } from './node-parameters/rename-node-utils';
|
||||
import { applyAccessPatterns } from './node-reference-parser-utils';
|
||||
import * as ObservableObject from './observable-object';
|
||||
|
||||
function dedupe<T>(arr: T[]): T[] {
|
||||
return [...new Set(arr)];
|
||||
}
|
||||
import { dedupe } from './utils';
|
||||
|
||||
export interface WorkflowParameters {
|
||||
id?: string;
|
||||
|
||||
Generated
+7
-7
@@ -22,8 +22,8 @@ catalogs:
|
||||
specifier: 0.3.20-15
|
||||
version: 0.3.20-15
|
||||
'@n8n_io/ai-assistant-sdk':
|
||||
specifier: 1.17.0
|
||||
version: 1.17.0
|
||||
specifier: 1.18.0
|
||||
version: 1.18.0
|
||||
'@sentry/node':
|
||||
specifier: ^9.42.1
|
||||
version: 9.42.1
|
||||
@@ -396,7 +396,7 @@ importers:
|
||||
version: link:../utils
|
||||
'@n8n_io/ai-assistant-sdk':
|
||||
specifier: 'catalog:'
|
||||
version: 1.17.0
|
||||
version: 1.18.0
|
||||
csv-parse:
|
||||
specifier: 5.5.0
|
||||
version: 5.5.0
|
||||
@@ -1534,7 +1534,7 @@ importers:
|
||||
version: link:../@n8n/utils
|
||||
'@n8n_io/ai-assistant-sdk':
|
||||
specifier: 'catalog:'
|
||||
version: 1.17.0
|
||||
version: 1.18.0
|
||||
'@n8n_io/license-sdk':
|
||||
specifier: 2.24.1
|
||||
version: 2.24.1
|
||||
@@ -6339,8 +6339,8 @@ packages:
|
||||
engines: {node: '>=18.10', pnpm: '>=9.6'}
|
||||
hasBin: true
|
||||
|
||||
'@n8n_io/ai-assistant-sdk@1.17.0':
|
||||
resolution: {integrity: sha512-Zwfgf9N4aK9klCVC15xHL8R5ID8h9f6OAlW6fPJRV00cmBjX2gD8ZYaX92A9iGiKpmW5YG3mxPU7XTFVexB7wQ==}
|
||||
'@n8n_io/ai-assistant-sdk@1.18.0':
|
||||
resolution: {integrity: sha512-lWsGJ5AniMd/s6FqtQuCS5YJ/iWIA53GjdLieQWgpjwmc52zOAIhfpPGbI5tmm0kpvExYpGDvVRH8UzOHlt9QA==}
|
||||
engines: {node: '>=20.15', pnpm: '>=8.14'}
|
||||
|
||||
'@n8n_io/license-sdk@2.24.1':
|
||||
@@ -22728,7 +22728,7 @@ snapshots:
|
||||
acorn: 8.12.1
|
||||
acorn-walk: 8.3.4
|
||||
|
||||
'@n8n_io/ai-assistant-sdk@1.17.0': {}
|
||||
'@n8n_io/ai-assistant-sdk@1.18.0': {}
|
||||
|
||||
'@n8n_io/license-sdk@2.24.1':
|
||||
dependencies:
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ catalog:
|
||||
'@langchain/core': 1.1.0
|
||||
'@langchain/openai': 1.1.3
|
||||
'@n8n/typeorm': 0.3.20-15
|
||||
'@n8n_io/ai-assistant-sdk': 1.17.0
|
||||
'@n8n_io/ai-assistant-sdk': 1.18.0
|
||||
'@sentry/node': ^9.42.1
|
||||
'@types/basic-auth': ^1.1.3
|
||||
'@types/express': ^5.0.1
|
||||
|
||||
Reference in New Issue
Block a user