From dcbc24d3f756d5ce4370bf2277118ee31604d863 Mon Sep 17 00:00:00 2001 From: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:56:38 +0100 Subject: [PATCH] feat(ai-builder): Consolidate builder telemetry (#22644) --- .../evaluations/core/test-runner.ts | 5 +- .../evaluations/langsmith/pairwise-runner.ts | 23 +- .../evaluations/langsmith/runner.ts | 5 +- .../evaluations/utils/evaluation-helpers.ts | 7 +- .../src/ai-workflow-builder-agent.service.ts | 24 +- .../ai-workflow-builder-agent.service.test.ts | 5 +- .../src/test/workflow-builder-agent.test.ts | 10 +- .../src/workflow-builder-agent.ts | 1 + .../ai/__tests__/ai-build-request.dto.test.ts | 18 + .../src/dto/ai/ai-build-request.dto.ts | 1 + .../__tests__/ai.controller.test.ts | 3 + packages/cli/src/controllers/ai.controller.ts | 3 +- .../ai-workflow-builder.service.test.ts | 9 + .../handlers/executionFinished.test.ts | 94 ++- .../handlers/executionFinished.ts | 30 +- .../handlers/executionRecovered.ts | 2 +- .../src/app/composables/useWorkflowState.ts | 2 + .../editor-ui/src/app/views/NodeView.vue | 2 +- .../ai/assistant/assistant.api.test.ts | 6 + .../features/ai/assistant/assistant.store.ts | 2 + .../features/ai/assistant/assistant.types.ts | 1 + .../ai/assistant/builder.store.test.ts | 574 +++++++++++++++++- .../features/ai/assistant/builder.store.ts | 232 +++++-- .../features/ai/assistant/builder.utils.ts | 8 +- .../Agent/AskAssistantBuild.test.ts | 243 -------- .../components/Agent/AskAssistantBuild.vue | 85 +-- .../components/Agent/ExecuteMessage.vue | 97 +-- .../composables/useBuilderMessages.test.ts | 23 +- .../composables/useBuilderMessages.ts | 16 +- .../composables/useBuilderTodos.test.ts | 183 ++++++ .../assistant/composables/useBuilderTodos.ts | 164 +++++ packages/workflow/src/index.ts | 1 + packages/workflow/src/utils.ts | 4 + packages/workflow/src/workflow.ts | 5 +- pnpm-lock.yaml | 14 +- pnpm-workspace.yaml | 2 +- 36 files changed, 1376 insertions(+), 528 deletions(-) create mode 100644 packages/frontend/editor-ui/src/features/ai/assistant/composables/useBuilderTodos.test.ts create mode 100644 packages/frontend/editor-ui/src/features/ai/assistant/composables/useBuilderTodos.ts diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/core/test-runner.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/core/test-runner.ts index aa814eedf82..742f059d259 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/core/test-runner.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/core/test-runner.ts @@ -99,7 +99,10 @@ export async function runSingleTest( // Generate workflow const startTime = Date.now(); await consumeGenerator( - agent.chat(getChatPayload(testCase.prompt, testCase.id, opts?.featureFlags), userId), + agent.chat( + getChatPayload('single-eval', testCase.prompt, testCase.id, opts?.featureFlags), + userId, + ), ); const generationTime = Date.now() - startTime; diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/pairwise-runner.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/pairwise-runner.ts index b9b11e4e273..3053ebc8d8c 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/pairwise-runner.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/pairwise-runner.ts @@ -356,7 +356,10 @@ async function runSingleGeneration( // Generate workflow await consumeGenerator( - agent.chat(getChatPayload(inputs.prompt, runId, featureFlags), `pairwise-gen-${generationIndex}`), + agent.chat( + getChatPayload('pairewise-gen', inputs.prompt, runId, featureFlags), + `pairwise-gen-${generationIndex}`, + ), ); const state = await agent.getState(runId, `pairwise-gen-${generationIndex}`); @@ -441,7 +444,16 @@ function createPairwiseWorkflowGenerator( // Run all generations in parallel const generationResults = await Promise.all( Array.from({ length: numGenerations }, async (_, i) => { - return await runSingleGeneration(parsedNodeTypes, llm, numJudges, inputs, i, log, featureFlags, tracer); + return await runSingleGeneration( + parsedNodeTypes, + llm, + numJudges, + inputs, + i, + log, + featureFlags, + tracer, + ); }), ); @@ -838,7 +850,12 @@ export async function runLocalPairwiseEvaluation(options: LocalPairwiseOptions): const genStartTime = Date.now(); const runId = generateRunId(); const agent = createAgent(parsedNodeTypes, llm, undefined, featureFlags); - await consumeGenerator(agent.chat(getChatPayload(prompt, runId, featureFlags), `local-gen-${genIndex}`)); + await consumeGenerator( + agent.chat( + getChatPayload('pairwise-local', prompt, runId, featureFlags), + `local-gen-${genIndex}`, + ), + ); const state = await agent.getState(runId, `local-gen-${genIndex}`); if (!state.values || !isWorkflowStateValues(state.values)) { diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/runner.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/runner.ts index f0c6d91e35c..f40902d592b 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/runner.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/langsmith/runner.ts @@ -46,7 +46,10 @@ function createWorkflowGenerator( // Create agent for this run const agent = createAgent(parsedNodeTypes, llm, tracer); await consumeGenerator( - agent.chat(getChatPayload(messageContent, runId, featureFlags), 'langsmith-eval-user'), + agent.chat( + getChatPayload('langsmith-evals', messageContent, runId, featureFlags), + 'langsmith-eval-user', + ), ); // Get generated workflow with validation diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/utils/evaluation-helpers.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/utils/evaluation-helpers.ts index 016a2c47592..0ff3141c999 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/utils/evaluation-helpers.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/utils/evaluation-helpers.ts @@ -6,6 +6,7 @@ import { Client } from 'langsmith'; import type { INodeTypeDescription } from 'n8n-workflow'; import { join } from 'path'; import pc from 'picocolors'; +import { v4 as uuid } from 'uuid'; import { anthropicClaudeSonnet45 } from '../../src/llm-config'; import type { BuilderFeatureFlags, ChatPayload } from '../../src/workflow-builder-agent'; @@ -278,18 +279,20 @@ export async function consumeGenerator(gen: AsyncGenerator) { } export function getChatPayload( + evalType: string, message: string, - id: string, + workflowId: string, featureFlags?: BuilderFeatureFlags, ): ChatPayload { return { + id: `${evalType}-${uuid()}`, featureFlags: featureFlags ?? { multiAgent: true, templateExamples: false, }, message, workflowContext: { - currentWorkflow: { id, nodes: [], connections: {} }, + currentWorkflow: { id: workflowId, nodes: [], connections: {} }, }, }; } diff --git a/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts b/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts index db80d6f58f4..8d273c0256f 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/ai-workflow-builder-agent.service.ts @@ -61,10 +61,10 @@ export class AiWorkflowBuilderService { }); } - private async getApiProxyAuthHeaders(user: IUser) { + private async getApiProxyAuthHeaders(user: IUser, userMessageId: string) { assert(this.client); - const authResponse = await this.client.getBuilderApiProxyToken(user); + const authResponse = await this.client.getBuilderApiProxyToken(user, { userMessageId }); const authHeaders = { // eslint-disable-next-line @typescript-eslint/naming-convention Authorization: `${authResponse.tokenType} ${authResponse.accessToken}`, @@ -73,7 +73,10 @@ export class AiWorkflowBuilderService { return authHeaders; } - private async setupModels(user: IUser): Promise<{ + private async setupModels( + user: IUser, + userMessageId: string, + ): Promise<{ anthropicClaude: ChatAnthropic; tracingClient?: TracingClient; // eslint-disable-next-line @typescript-eslint/naming-convention @@ -82,7 +85,7 @@ export class AiWorkflowBuilderService { try { // If client is provided, use it for API proxy if (this.client) { - const authHeaders = await this.getApiProxyAuthHeaders(user); + const authHeaders = await this.getApiProxyAuthHeaders(user, userMessageId); // Extract baseUrl from client configuration const baseUrl = this.client.getApiProxyBaseUrl(); @@ -158,8 +161,11 @@ export class AiWorkflowBuilderService { }); } - private async getAgent(user: IUser, featureFlags?: BuilderFeatureFlags) { - const { anthropicClaude, tracingClient, authHeaders } = await this.setupModels(user); + private async getAgent(user: IUser, userMessageId: string, featureFlags?: BuilderFeatureFlags) { + const { anthropicClaude, tracingClient, authHeaders } = await this.setupModels( + user, + userMessageId, + ); const agent = new WorkflowBuilderAgent({ parsedNodeTypes: this.parsedNodeTypes, @@ -208,7 +214,7 @@ export class AiWorkflowBuilderService { } async *chat(payload: ChatPayload, user: IUser, abortSignal?: AbortSignal) { - const agent = await this.getAgent(user, payload.featureFlags); + const agent = await this.getAgent(user, payload.id, payload.featureFlags); const userId = user?.id?.toString(); const workflowId = payload.workflowContext?.currentWorkflow?.id; @@ -219,7 +225,7 @@ export class AiWorkflowBuilderService { // After the stream completes, track telemetry if (this.onTelemetryEvent && userId) { try { - await this.trackBuilderReplyTelemetry(agent, workflowId, userId); + await this.trackBuilderReplyTelemetry(agent, workflowId, userId, payload.id); } catch (error) { this.logger?.error('Failed to track builder reply telemetry', { error }); } @@ -230,6 +236,7 @@ export class AiWorkflowBuilderService { agent: WorkflowBuilderAgent, workflowId: string | undefined, userId: string, + userMessageId: string, ): Promise { if (!this.onTelemetryEvent) return; @@ -270,6 +277,7 @@ export class AiWorkflowBuilderService { ...(state.values.templateIds.length > 0 && { templates_selected: state.values.templateIds, }), + user_message_id: userMessageId, }; this.onTelemetryEvent('Builder replied to user message', properties); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts index e78864ab8c6..85a15f4446b 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/test/ai-workflow-builder-agent.service.test.ts @@ -151,9 +151,6 @@ describe('AiWorkflowBuilderService', () => { // Mock AI assistant client mockClient = mock(); - (mockClient.generateApiProxyCredentials as jest.Mock).mockResolvedValue({ - apiKey: 'test-api-key', - }); (mockClient.getBuilderApiProxyToken as jest.Mock).mockResolvedValue({ tokenType: 'Bearer', accessToken: 'test-access-token', @@ -301,6 +298,7 @@ describe('AiWorkflowBuilderService', () => { beforeEach(() => { mockPayload = { + id: '12345', message: 'Create a simple workflow', workflowContext: { currentWorkflow: { id: 'test-workflow' }, @@ -571,6 +569,7 @@ describe('AiWorkflowBuilderService', () => { it('should handle complete workflow from chat to session retrieval', async () => { const workflowId = 'integration-test-workflow'; const mockPayload: ChatPayload = { + id: '545623', message: 'Create a workflow with HTTP request', workflowContext: { currentWorkflow: { id: workflowId }, diff --git a/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts b/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts index 22092414300..793679c9fef 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/test/workflow-builder-agent.test.ts @@ -146,6 +146,7 @@ describe('WorkflowBuilderAgent', () => { beforeEach(() => { mockPayload = { + id: '12345', message: 'Create a workflow', workflowContext: { currentWorkflow: { id: 'workflow-123' }, @@ -156,6 +157,7 @@ describe('WorkflowBuilderAgent', () => { it('should throw ValidationError when message exceeds maximum length', async () => { const longMessage = 'x'.repeat(MAX_AI_BUILDER_PROMPT_LENGTH + 1); const payload: ChatPayload = { + id: '12345', message: longMessage, }; @@ -173,6 +175,7 @@ describe('WorkflowBuilderAgent', () => { it('should handle valid message length', async () => { const validMessage = 'Create a simple workflow'; const payload: ChatPayload = { + id: '12345', message: validMessage, }; @@ -491,6 +494,7 @@ describe('WorkflowBuilderAgent', () => { ); const generator = agent.chat({ + id: '12345', message: 'Create a workflow', featureFlags: { templateExamples: true }, }); @@ -511,6 +515,7 @@ describe('WorkflowBuilderAgent', () => { ); const generator = agent.chat({ + id: '12345', message: 'Create a workflow', featureFlags: { templateExamples: false }, }); @@ -530,7 +535,10 @@ describe('WorkflowBuilderAgent', () => { mockAsyncGenerator, ); - const generator = agent.chat({ message: 'Create a workflow' }); + const generator = agent.chat({ + id: '12345', + message: 'Create a workflow', + }); await generator.next(); expect(mockCreateMainAgentPrompt).toHaveBeenCalledWith({ includeExamplesPhase: false }); diff --git a/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts b/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts index b871bcd8b9f..e1557a0ca72 100644 --- a/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts +++ b/packages/@n8n/ai-workflow-builder.ee/src/workflow-builder-agent.ts @@ -159,6 +159,7 @@ export interface BuilderFeatureFlags { } export interface ChatPayload { + id: string; message: string; workflowContext?: { executionSchema?: NodeExecutionSchema[]; diff --git a/packages/@n8n/api-types/src/dto/ai/__tests__/ai-build-request.dto.test.ts b/packages/@n8n/api-types/src/dto/ai/__tests__/ai-build-request.dto.test.ts index e31edc8ff7b..e8b2be09f15 100644 --- a/packages/@n8n/api-types/src/dto/ai/__tests__/ai-build-request.dto.test.ts +++ b/packages/@n8n/api-types/src/dto/ai/__tests__/ai-build-request.dto.test.ts @@ -3,6 +3,7 @@ import { AiBuilderChatRequestDto } from '../ai-build-request.dto'; describe('AiBuilderChatRequestDto', () => { const validBasePayload = { payload: { + id: '12345', role: 'user' as const, type: 'message' as const, text: 'Build me a workflow', @@ -305,6 +306,7 @@ describe('AiBuilderChatRequestDto', () => { const invalidRequest = { ...validBasePayload, payload: { + id: '12345', role: 'user' as const, type: 'message' as const, workflowContext: validBasePayload.payload.workflowContext, @@ -315,5 +317,21 @@ describe('AiBuilderChatRequestDto', () => { expect(result.success).toBe(false); }); + + it('should fail when id is missing', () => { + const invalidRequest = { + ...validBasePayload, + payload: { + role: 'user' as const, + type: 'message' as const, + text: 'text', + workflowContext: validBasePayload.payload.workflowContext, + }, + }; + + const result = AiBuilderChatRequestDto.safeParse(invalidRequest); + + expect(result.success).toBe(false); + }); }); }); diff --git a/packages/@n8n/api-types/src/dto/ai/ai-build-request.dto.ts b/packages/@n8n/api-types/src/dto/ai/ai-build-request.dto.ts index 5ad8fc703b7..bb6cde31c10 100644 --- a/packages/@n8n/api-types/src/dto/ai/ai-build-request.dto.ts +++ b/packages/@n8n/api-types/src/dto/ai/ai-build-request.dto.ts @@ -10,6 +10,7 @@ export interface ExpressionValue { export class AiBuilderChatRequestDto extends Z.class({ payload: z.object({ + id: z.string(), role: z.literal('user'), type: z.literal('message'), text: z.string(), diff --git a/packages/cli/src/controllers/__tests__/ai.controller.test.ts b/packages/cli/src/controllers/__tests__/ai.controller.test.ts index 7dd17f2a83c..a78e9fd003c 100644 --- a/packages/cli/src/controllers/__tests__/ai.controller.test.ts +++ b/packages/cli/src/controllers/__tests__/ai.controller.test.ts @@ -118,6 +118,7 @@ describe('AiController', () => { describe('build', () => { const payload: AiBuilderChatRequestDto = { payload: { + id: '12345', text: 'Create a workflow', type: 'message', role: 'user', @@ -146,6 +147,8 @@ describe('AiController', () => { expect(workflowBuilderService.chat).toHaveBeenCalledWith( { + id: '12345', + featureFlags: undefined, message: 'Create a workflow', workflowContext: { currentWorkflow: { id: 'workflow123' }, diff --git a/packages/cli/src/controllers/ai.controller.ts b/packages/cli/src/controllers/ai.controller.ts index c513b8492e6..7dba4e368bc 100644 --- a/packages/cli/src/controllers/ai.controller.ts +++ b/packages/cli/src/controllers/ai.controller.ts @@ -55,9 +55,10 @@ export class AiController { res.on('close', handleClose); - const { text, workflowContext, featureFlags } = payload.payload; + const { id, text, workflowContext, featureFlags } = payload.payload; const aiResponse = this.workflowBuilderService.chat( { + id, message: text, workflowContext: { currentWorkflow: workflowContext.currentWorkflow, diff --git a/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts b/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts index f24e3a9afe8..c3e6824bf35 100644 --- a/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts +++ b/packages/cli/src/services/__tests__/ai-workflow-builder.service.test.ts @@ -100,6 +100,7 @@ describe('WorkflowBuilderService', () => { describe('chat', () => { it('should create AiWorkflowBuilderService on first chat call without AI assistant client', async () => { const mockPayload = { + id: '12345', message: 'test message', workflowContext: {}, }; @@ -134,6 +135,7 @@ describe('WorkflowBuilderService', () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -170,6 +172,7 @@ describe('WorkflowBuilderService', () => { it('should reuse the same service instance on subsequent calls', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -201,6 +204,7 @@ describe('WorkflowBuilderService', () => { it('should pass abort signal to underlying service', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -265,6 +269,7 @@ describe('WorkflowBuilderService', () => { it('should send push notification when credits are updated', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -313,6 +318,7 @@ describe('WorkflowBuilderService', () => { it('should handle multiple credit updates', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -373,6 +379,7 @@ describe('WorkflowBuilderService', () => { it('should call telemetry.track when telemetry event is triggered', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -419,6 +426,7 @@ describe('WorkflowBuilderService', () => { it('should handle multiple telemetry events', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; @@ -463,6 +471,7 @@ describe('WorkflowBuilderService', () => { it('should handle telemetry events with empty properties', async () => { const mockPayload = { message: 'test message', + id: '12345', workflowContext: {}, }; diff --git a/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.test.ts b/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.test.ts index d3d29e7b7ae..fbfbb49b83b 100644 --- a/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.test.ts +++ b/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.test.ts @@ -4,9 +4,11 @@ import { continueEvaluationLoop, executionFinished, getRunExecutionData, + handleExecutionFinishedWithSuccessOrOther, + handleExecutionFinishedWithErrorOrCanceled, type SimplifiedExecution, } from './executionFinished'; -import type { ITaskData } from 'n8n-workflow'; +import type { IRunExecutionData, ITaskData } from 'n8n-workflow'; import { EVALUATION_TRIGGER_NODE_TYPE } from 'n8n-workflow'; import type { INodeUi, IWorkflowDb } from '@/Interface'; import type { Router } from 'vue-router'; @@ -17,6 +19,7 @@ import { useWorkflowsStore } from '@/app/stores/workflows.store'; import { useUIStore } from '@/app/stores/ui.store'; import { mockedStore } from '@/__tests__/utils'; import { useReadyToRunStore } from '@/features/workflows/readyToRun/stores/readyToRun.store'; +import { useBuilderStore } from '@/features/ai/assistant/builder.store'; const opts = { workflowState: mock(), @@ -524,3 +527,92 @@ describe('executionFinished', () => { expect(runWorkflow).not.toHaveBeenCalled(); }); }); + +describe('manual execution stats tracking', () => { + beforeEach(() => { + vi.resetAllMocks(); + }); + + describe('handleExecutionFinishedWithSuccessOrOther', () => { + it('increments success stats on successful execution', () => { + const pinia = createTestingPinia(); + setActivePinia(pinia); + + const builderStore = mockedStore(useBuilderStore); + const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats'); + + handleExecutionFinishedWithSuccessOrOther(mock(), 'success', false); + + expect(incrementSpy).toHaveBeenCalledWith('success'); + }); + + it('does not increment success stats when successToastAlreadyShown is true', () => { + const pinia = createTestingPinia(); + setActivePinia(pinia); + + const builderStore = mockedStore(useBuilderStore); + const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats'); + + handleExecutionFinishedWithSuccessOrOther(mock(), 'success', true); + + expect(incrementSpy).not.toHaveBeenCalled(); + }); + + it('does not increment stats for non-success status', () => { + const pinia = createTestingPinia(); + setActivePinia(pinia); + + const builderStore = mockedStore(useBuilderStore); + const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats'); + + handleExecutionFinishedWithSuccessOrOther(mock(), 'error', false); + + expect(incrementSpy).not.toHaveBeenCalled(); + }); + }); + + describe('handleExecutionFinishedWithErrorOrCanceled', () => { + it('increments error stats on execution error', () => { + const pinia = createTestingPinia(); + setActivePinia(pinia); + + const builderStore = mockedStore(useBuilderStore); + const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats'); + + const execution = mock({ + status: 'error', + data: { + resultData: { + error: { message: 'test error', name: 'Error' }, + }, + }, + }); + + handleExecutionFinishedWithErrorOrCanceled( + execution, + mock({ resultData: { error: { message: 'test', name: 'Error' } } }), + ); + + expect(incrementSpy).toHaveBeenCalledWith('error'); + }); + + it('does not increment stats for canceled executions', () => { + const pinia = createTestingPinia(); + setActivePinia(pinia); + + const builderStore = mockedStore(useBuilderStore); + const incrementSpy = vi.spyOn(builderStore, 'incrementManualExecutionStats'); + + const execution = mock({ + status: 'canceled', + }); + + handleExecutionFinishedWithErrorOrCanceled( + execution, + mock({ resultData: {} }), + ); + + expect(incrementSpy).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.ts b/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.ts index 420e71a5209..85edd57c046 100644 --- a/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.ts +++ b/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionFinished.ts @@ -14,6 +14,7 @@ import { useNodeTypesStore } from '@/app/stores/nodeTypes.store'; import { useSettingsStore } from '@/app/stores/settings.store'; import { useUIStore } from '@/app/stores/ui.store'; import { useWorkflowsStore } from '@/app/stores/workflows.store'; +import { useBuilderStore } from '@/features/ai/assistant/builder.store'; import { SampleTemplates, isPrebuiltAgentTemplateId, @@ -28,7 +29,13 @@ import { getTriggerNodeServiceName } from '@/app/utils/nodeTypesUtils'; import type { ExecutionFinished } from '@n8n/api-types/push/execution'; import { useI18n } from '@n8n/i18n'; import { parse } from 'flatted'; -import type { ExpressionError, IDataObject, IRunExecutionData, IWorkflowBase } from 'n8n-workflow'; +import type { + ExecutionStatus, + ExpressionError, + IDataObject, + IRunExecutionData, + IWorkflowBase, +} from 'n8n-workflow'; import { EVALUATION_TRIGGER_NODE_TYPE, TelemetryHelpers, @@ -115,7 +122,11 @@ export async function executionFinished( let successToastAlreadyShown = false; if (data.status === 'success') { - handleExecutionFinishedWithSuccessOrOther(options.workflowState, successToastAlreadyShown); + handleExecutionFinishedWithSuccessOrOther( + options.workflowState, + data.status, + successToastAlreadyShown, + ); successToastAlreadyShown = true; } @@ -140,7 +151,11 @@ export async function executionFinished( } else if (execution.status === 'error' || execution.status === 'canceled') { handleExecutionFinishedWithErrorOrCanceled(execution, runExecutionData); } else { - handleExecutionFinishedWithSuccessOrOther(options.workflowState, successToastAlreadyShown); + handleExecutionFinishedWithSuccessOrOther( + options.workflowState, + execution.status, + successToastAlreadyShown, + ); } setRunExecutionData(execution, runExecutionData, options.workflowState); @@ -351,6 +366,8 @@ export function handleExecutionFinishedWithErrorOrCanceled( }); toast.showMessage({ title, message, type: 'error', duration: 0 }); + + useBuilderStore().incrementManualExecutionStats('error'); } } @@ -381,6 +398,7 @@ function handleExecutionFinishedSuccessfully( */ export function handleExecutionFinishedWithSuccessOrOther( workflowState: WorkflowState, + executionStatus: ExecutionStatus, successToastAlreadyShown: boolean, ) { const workflowsStore = useWorkflowsStore(); @@ -427,6 +445,12 @@ export function handleExecutionFinishedWithSuccessOrOther( workflowState, ); } + + // Execution finished is triggered multiple times + // use "successToastAlreadyShown" flag to avoid double counting executions + if (executionStatus === 'success' && !successToastAlreadyShown) { + useBuilderStore().incrementManualExecutionStats('success'); + } } export function setRunExecutionData( diff --git a/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionRecovered.ts b/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionRecovered.ts index 63e8942764c..924af739305 100644 --- a/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionRecovered.ts +++ b/packages/frontend/editor-ui/src/app/composables/usePushConnection/handlers/executionRecovered.ts @@ -40,7 +40,7 @@ export async function executionRecovered( } else if (execution.status === 'error' || execution.status === 'canceled') { handleExecutionFinishedWithErrorOrCanceled(execution, runExecutionData); } else { - handleExecutionFinishedWithSuccessOrOther(options.workflowState, false); + handleExecutionFinishedWithSuccessOrOther(options.workflowState, execution.status, false); } setRunExecutionData(execution, runExecutionData, options.workflowState); diff --git a/packages/frontend/editor-ui/src/app/composables/useWorkflowState.ts b/packages/frontend/editor-ui/src/app/composables/useWorkflowState.ts index fe30b66bdf2..0482f6948ec 100644 --- a/packages/frontend/editor-ui/src/app/composables/useWorkflowState.ts +++ b/packages/frontend/editor-ui/src/app/composables/useWorkflowState.ts @@ -15,6 +15,7 @@ import type { } from '@/features/execution/executions/executions.types'; import { useUIStore } from '@/app/stores/ui.store'; import { useWorkflowsStore } from '@/app/stores/workflows.store'; +import { useBuilderStore } from '@/features/ai/assistant/builder.store'; import { getPairedItemsMapping } from '@/app/utils/pairedItemUtils'; import { type INodeIssueData, @@ -233,6 +234,7 @@ export function useWorkflowState() { setActiveExecutionId(undefined); workflowStateStore.executingNode.executingNode.length = 0; ws.executionWaitingForWebhook = false; + useBuilderStore().resetManualExecutionStats(); } //// diff --git a/packages/frontend/editor-ui/src/app/views/NodeView.vue b/packages/frontend/editor-ui/src/app/views/NodeView.vue index 2c5343a46bb..1a34b8c2e28 100644 --- a/packages/frontend/editor-ui/src/app/views/NodeView.vue +++ b/packages/frontend/editor-ui/src/app/views/NodeView.vue @@ -2149,7 +2149,7 @@ onBeforeUnmount(() => { v-if="builderStore.streaming" :class="$style.thinkingPill" show-stop - @stop="builderStore.stopStreaming" + @stop="builderStore.abortStreaming" /> diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/assistant.api.test.ts b/packages/frontend/editor-ui/src/features/ai/assistant/assistant.api.test.ts index 90092709627..94cc8453d77 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/assistant.api.test.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/assistant.api.test.ts @@ -38,6 +38,7 @@ describe('API: ai', () => { it('should call streamRequest with the correct parameters', () => { const payload: ChatRequest.RequestPayload = { payload: { + id: '12345', role: 'user', type: 'message', text: 'Build me a workflow', @@ -62,6 +63,7 @@ describe('API: ai', () => { it('should pass abort signal when provided', () => { const payload: ChatRequest.RequestPayload = { payload: { + id: '12345', role: 'user', type: 'message', text: 'Build me a workflow', @@ -177,6 +179,7 @@ describe('API: ai', () => { const payload: ChatRequest.RequestPayload = { payload: { + id: '12345', role: 'user', type: 'message', text: 'Build me a workflow', @@ -211,6 +214,7 @@ describe('API: ai', () => { const payload: ChatRequest.RequestPayload = { payload: { + id: '12345', role: 'user', type: 'message', text: 'Build me a workflow', @@ -228,6 +232,7 @@ describe('API: ai', () => { it('should handle complex workflow context in payload', () => { const payload: ChatRequest.RequestPayload = { payload: { + id: '12345', role: 'user', type: 'message', text: 'Improve my workflow', @@ -271,6 +276,7 @@ describe('API: ai', () => { it('should handle undefined parameters correctly', () => { const payload: ChatRequest.RequestPayload = { payload: { + id: '12345', role: 'user', type: 'message', text: 'Build me a workflow', diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/assistant.store.ts b/packages/frontend/editor-ui/src/features/ai/assistant/assistant.store.ts index b249e4c11ef..601ac7bd051 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/assistant.store.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/assistant.store.ts @@ -29,6 +29,7 @@ import { useChatPanelStateStore } from './chatPanelState.store'; import { useCredentialsStore } from '@/features/credentials/credentials.store'; import { useAIAssistantHelpers } from '@/features/ai/assistant/composables/useAIAssistantHelpers'; import type { WorkflowState } from '@/app/composables/useWorkflowState'; +import { v4 as uuid } from 'uuid'; export const ENABLED_VIEWS = ASSISTANT_ENABLED_VIEWS; const READABLE_TYPES = ['code-diff', 'text', 'block']; @@ -555,6 +556,7 @@ export const useAssistantStore = defineStore(STORES.ASSISTANT, () => { rootStore.restApiContext, { payload: { + id: uuid(), role: 'user', type: 'message', text: chatMessage.text, diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/assistant.types.ts b/packages/frontend/editor-ui/src/features/ai/assistant/assistant.types.ts index 54bf17b437b..b734f2ad2a5 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/assistant.types.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/assistant.types.ts @@ -102,6 +102,7 @@ export namespace ChatRequest { role: 'user'; type: 'message'; text: string; + id: string; quickReplyType?: string; context?: UserContext; workflowContext?: WorkflowContext; diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.test.ts b/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.test.ts index e328b275926..5acabfed3a8 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.test.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.test.ts @@ -606,7 +606,7 @@ describe('AI Builder store', () => { expect(builderStore.streamingAbortController).toBeInstanceOf(AbortController); }); - it('should call abort on existing controller when stopStreaming is called', () => { + it('should call abort on existing controller when abortStreaming is called', () => { const builderStore = useBuilderStore(); // First start a request to create an abort controller @@ -620,8 +620,8 @@ describe('AI Builder store', () => { // Spy on the abort method const abortSpy = vi.spyOn(controller!, 'abort'); - // Call stopStreaming - builderStore.stopStreaming(); + // Call abortStreaming + builderStore.abortStreaming(); // Verify abort was called expect(abortSpy).toHaveBeenCalled(); @@ -698,7 +698,7 @@ describe('AI Builder store', () => { expect(builderStore.streamingAbortController).toBe(firstController); // Now properly stop streaming first - builderStore.stopStreaming(); + builderStore.abortStreaming(); // Verify abort was called and controller was cleared expect(abortSpy).toHaveBeenCalled(); @@ -1708,4 +1708,570 @@ describe('AI Builder store', () => { expect(builderStore.creditsRemaining).toBe(80); }); }); + + describe('abortStreaming telemetry', () => { + it('tracks end of response with aborted flag when aborting', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + builderStore.sendChatMessage({ text: 'test' }); + + track.mockClear(); + builderStore.abortStreaming(); + + expect(track).toHaveBeenCalledWith( + 'End of response from builder', + expect.objectContaining({ + aborted: true, + user_message_id: expect.any(String), + workflow_id: 'test-workflow-id', + }), + ); + }); + + it('includes workflow modifications in abort telemetry', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + builderStore.sendChatMessage({ text: 'test' }); + + track.mockClear(); + builderStore.abortStreaming(); + + expect(track).toHaveBeenCalledWith( + 'End of response from builder', + expect.objectContaining({ + tools_called: expect.any(Array), + start_workflow_json: expect.any(String), + end_workflow_json: expect.any(String), + }), + ); + }); + + it('includes todos count in abort telemetry', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + builderStore.sendChatMessage({ text: 'test' }); + + track.mockClear(); + builderStore.abortStreaming(); + + expect(track).toHaveBeenCalledWith( + 'End of response from builder', + expect.objectContaining({ + credentials_todo_count: expect.any(Number), + placeholders_todo_count: expect.any(Number), + todos: expect.any(Array), + }), + ); + }); + + it('does not track telemetry if no streaming message in progress', () => { + const builderStore = useBuilderStore(); + + // Don't start any streaming + track.mockClear(); + + builderStore.abortStreaming(); + + expect(track).not.toHaveBeenCalledWith('End of response from builder', expect.anything()); + }); + }); + + describe('workflowTodos', () => { + it('returns empty array when no validation issues exist', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = []; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toEqual([]); + }); + + it('includes credential validation issues', () => { + workflowsStore.workflowValidationIssues = [ + { node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toContainEqual( + expect.objectContaining({ type: 'credentials' }), + ); + }); + + it('includes placeholder issues from node parameters', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE__Enter URL__>', + }, + }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toContainEqual( + expect.objectContaining({ type: 'parameters', node: 'HTTP Request' }), + ); + }); + + it('combines credential and placeholder issues', () => { + workflowsStore.workflowValidationIssues = [ + { node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' }, + ]; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE__Enter URL__>', + }, + }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos.length).toBeGreaterThanOrEqual(2); + expect(builderStore.workflowTodos).toContainEqual( + expect.objectContaining({ type: 'credentials' }), + ); + expect(builderStore.workflowTodos).toContainEqual( + expect.objectContaining({ type: 'parameters' }), + ); + }); + }); + + describe('placeholderIssues', () => { + it('returns empty array when nodes have no parameters', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'Start', + type: 'n8n-nodes-base.start', + typeVersion: 1, + position: [0, 0], + parameters: {}, + }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toEqual([]); + }); + + it('returns empty array when node has undefined parameters', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'Start', + type: 'n8n-nodes-base.start', + typeVersion: 1, + position: [0, 0], + } as Parameters[0], + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toEqual([]); + }); + + it('detects placeholders in nested object parameters', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + options: { + headers: { + authorization: '<__PLACEHOLDER_VALUE__Enter API Key__>', + }, + }, + }, + }, + ]; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + expect(placeholderIssues).toHaveLength(1); + expect(placeholderIssues[0]).toMatchObject({ + node: 'HTTP Request', + type: 'parameters', + }); + }); + + it('detects placeholders in array parameters', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + headers: [ + { name: 'Content-Type', value: 'application/json' }, + { name: 'Authorization', value: '<__PLACEHOLDER_VALUE__Enter Token__>' }, + ], + }, + }, + ]; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + expect(placeholderIssues).toHaveLength(1); + expect(placeholderIssues[0]).toMatchObject({ + node: 'HTTP Request', + type: 'parameters', + }); + }); + + it('detects multiple placeholders in the same node', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE__Enter URL__>', + body: '<__PLACEHOLDER_VALUE__Enter Body__>', + }, + }, + ]; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + expect(placeholderIssues).toHaveLength(2); + }); + + it('detects placeholders across multiple nodes', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE__Enter URL__>', + }, + }, + { + id: 'node-2', + name: 'Slack', + type: 'n8n-nodes-base.slack', + typeVersion: 1, + position: [200, 0], + parameters: { + channel: '<__PLACEHOLDER_VALUE__Enter Channel__>', + }, + }, + ]; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + expect(placeholderIssues).toHaveLength(2); + expect(placeholderIssues).toContainEqual(expect.objectContaining({ node: 'HTTP Request' })); + expect(placeholderIssues).toContainEqual(expect.objectContaining({ node: 'Slack' })); + }); + + it('deduplicates identical placeholder issues (same node, path, and label)', () => { + workflowsStore.workflowValidationIssues = []; + // Simulate a scenario where the same placeholder appears twice + // (which shouldn't happen in practice but tests the deduplication) + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE__Enter URL__>', + }, + }, + ]; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + // Should only have 1 issue, not duplicates + expect(placeholderIssues).toHaveLength(1); + }); + + it('skips placeholder when existing parameter issue already has the same message', () => { + const placeholderLabel = 'Enter URL'; + // The message format from the store uses i18n which is mocked to return the key + const expectedMessage = 'aiAssistant.builder.executeMessage.fillParameter'; + + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: `<__PLACEHOLDER_VALUE__${placeholderLabel}__>`, + }, + issues: { + parameters: { + url: [expectedMessage], + }, + }, + }, + ]; + workflowsStore.workflowValidationIssues = []; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + // Should be skipped because the message already exists + expect(placeholderIssues).toHaveLength(0); + }); + + it('does not skip placeholder when existing parameter issue has different message', () => { + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE__Enter URL__>', + }, + issues: { + parameters: { + url: ['Some other validation error'], + }, + }, + }, + ]; + workflowsStore.workflowValidationIssues = []; + + const builderStore = useBuilderStore(); + const placeholderIssues = builderStore.workflowTodos.filter((t) => t.type === 'parameters'); + // Should still create the placeholder issue + expect(placeholderIssues).toHaveLength(1); + }); + + it('ignores non-string parameter values', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + timeout: 5000, + enabled: true, + config: null, + }, + }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toEqual([]); + }); + + it('ignores strings that do not match placeholder format', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: 'https://example.com', + body: 'regular string', + partial: '<__PLACEHOLDER_VALUE__missing end', + wrongPrefix: 'PLACEHOLDER__test__>', + }, + }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toEqual([]); + }); + + it('ignores placeholder with empty label', () => { + workflowsStore.workflowValidationIssues = []; + workflowsStore.workflow.nodes = [ + { + id: 'node-1', + name: 'HTTP Request', + type: 'n8n-nodes-base.httpRequest', + typeVersion: 1, + position: [0, 0], + parameters: { + url: '<__PLACEHOLDER_VALUE____>', // empty label + body: '<__PLACEHOLDER_VALUE__ __>', // whitespace-only label + }, + }, + ]; + + const builderStore = useBuilderStore(); + expect(builderStore.workflowTodos).toEqual([]); + }); + + it('filters out non-credential and non-parameter validation issues', () => { + workflowsStore.workflowValidationIssues = [ + { node: 'HTTP Request', type: 'credentials', value: 'Missing credentials' }, + { node: 'HTTP Request', type: 'parameters', value: 'Missing parameter' }, + { node: 'HTTP Request', type: 'execution', value: 'Execution error' }, + { node: 'HTTP Request', type: 'unknown' as 'parameters', value: 'Unknown issue' }, + ]; + workflowsStore.workflow.nodes = []; + + const builderStore = useBuilderStore(); + // Should only include credentials and parameters types + expect(builderStore.workflowTodos).toHaveLength(2); + expect(builderStore.workflowTodos).toContainEqual( + expect.objectContaining({ type: 'credentials' }), + ); + expect(builderStore.workflowTodos).toContainEqual( + expect.objectContaining({ type: 'parameters' }), + ); + }); + }); + + describe('manual execution stats telemetry', () => { + it('should include success count in telemetry when sending message', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + + builderStore.incrementManualExecutionStats('success'); + builderStore.sendChatMessage({ text: 'test' }); + + expect(track).toHaveBeenCalledWith( + 'User submitted builder message', + expect.objectContaining({ + manual_exec_success_count_since_prev_msg: 1, + manual_exec_error_count_since_prev_msg: 0, + }), + ); + }); + + it('should include error count in telemetry when sending message', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + + builderStore.incrementManualExecutionStats('error'); + builderStore.sendChatMessage({ text: 'test' }); + + expect(track).toHaveBeenCalledWith( + 'User submitted builder message', + expect.objectContaining({ + manual_exec_success_count_since_prev_msg: 0, + manual_exec_error_count_since_prev_msg: 1, + }), + ); + }); + + it('should include multiple incremented counts in telemetry', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + + builderStore.incrementManualExecutionStats('success'); + builderStore.incrementManualExecutionStats('success'); + builderStore.incrementManualExecutionStats('error'); + builderStore.sendChatMessage({ text: 'test' }); + + expect(track).toHaveBeenCalledWith( + 'User submitted builder message', + expect.objectContaining({ + manual_exec_success_count_since_prev_msg: 2, + manual_exec_error_count_since_prev_msg: 1, + }), + ); + }); + + it('should reset stats after sending message', async () => { + const builderStore = useBuilderStore(); + + // First message with some stats + apiSpy.mockImplementationOnce((_ctx, _payload, onMessage, onDone) => { + onMessage({ + messages: [{ type: 'message', role: 'assistant', text: 'Hello!' }], + sessionId: 'test-session', + }); + onDone(); + }); + + builderStore.incrementManualExecutionStats('success'); + builderStore.incrementManualExecutionStats('error'); + builderStore.sendChatMessage({ text: 'first message' }); + + await vi.waitFor(() => expect(builderStore.streaming).toBe(false)); + + // Verify first message had the stats + expect(track).toHaveBeenCalledWith( + 'User submitted builder message', + expect.objectContaining({ + manual_exec_success_count_since_prev_msg: 1, + manual_exec_error_count_since_prev_msg: 1, + }), + ); + + track.mockClear(); + + // Second message should have reset stats (zero counts) + apiSpy.mockImplementationOnce((_ctx, _payload, onMessage, onDone) => { + onMessage({ + messages: [{ type: 'message', role: 'assistant', text: 'Hello again!' }], + sessionId: 'test-session', + }); + onDone(); + }); + + builderStore.sendChatMessage({ text: 'second message' }); + + await vi.waitFor(() => expect(builderStore.streaming).toBe(false)); + + expect(track).toHaveBeenCalledWith( + 'User submitted builder message', + expect.objectContaining({ + manual_exec_success_count_since_prev_msg: 0, + manual_exec_error_count_since_prev_msg: 0, + }), + ); + }); + + it('should include zero counts when no manual executions occurred', () => { + const builderStore = useBuilderStore(); + + apiSpy.mockImplementationOnce(() => {}); + + builderStore.sendChatMessage({ text: 'test' }); + + expect(track).toHaveBeenCalledWith( + 'User submitted builder message', + expect.objectContaining({ + manual_exec_success_count_since_prev_msg: 0, + manual_exec_error_count_since_prev_msg: 0, + }), + ); + }); + }); }); diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.ts b/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.ts index 2bbe440e681..6af43da868c 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/builder.store.ts @@ -20,10 +20,11 @@ import { getSessionsMetadata, } from '@/features/ai/assistant/assistant.api'; import { generateMessageId, createBuilderPayload } from './builder.utils'; +import { useBuilderTodos, type TodosTrackingPayload } from './composables/useBuilderTodos'; import { useRootStore } from '@n8n/stores/useRootStore'; import type { WorkflowDataUpdate } from '@n8n/rest-api-client/api/workflows'; import pick from 'lodash/pick'; -import { type INodeExecutionData, jsonParse } from 'n8n-workflow'; +import { type ITelemetryTrackProperties, type INodeExecutionData, jsonParse } from 'n8n-workflow'; import { useToast } from '@/app/composables/useToast'; import { injectWorkflowState } from '@/app/composables/useWorkflowState'; import { useNodeTypesStore } from '@/app/stores/nodeTypes.store'; @@ -31,10 +32,34 @@ import { useCredentialsStore } from '@/features/credentials/credentials.store'; import { getAuthTypeForNodeCredential, getMainAuthField } from '@/app/utils/nodeTypesUtils'; import { stringSizeInBytes } from '@/app/utils/typesUtils'; import { useNDVStore } from '@/features/ndv/shared/ndv.store'; +import { dedupe } from 'n8n-workflow'; const INFINITE_CREDITS = -1; export const ENABLED_VIEWS = BUILDER_ENABLED_VIEWS; +interface EndOfStreamingTrackingPayload { + userMessageId: string; + startWorkflowJson: string; +} + +interface UserSubmittedBuilderMessageTrackingPayload + extends ITelemetryTrackProperties, + TodosTrackingPayload { + source: 'chat' | 'canvas'; + message: string; + session_id: string; + start_workflow_json: string; + workflow_id: string; + type: 'message' | 'execution'; + manual_exec_success_count_since_prev_msg: number; + manual_exec_error_count_since_prev_msg: number; + user_message_id: string; + execution_data?: string; + execution_status?: string; + error_message?: string; + error_node_type?: string; +} + export const useBuilderStore = defineStore(STORES.BUILDER, () => { // Core state const chatMessages = ref([]); @@ -45,6 +70,12 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { const creditsQuota = ref(); const creditsClaimed = ref(); const hasMessages = ref(false); + const manualExecStatsInBetweenMessages = ref<{ success: number; error: number }>({ + success: 0, + error: 0, + }); + + const currentStreamingMessage = ref(); // Store dependencies const settings = useSettingsStore(); @@ -70,6 +101,8 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { getRunningTools, } = useBuilderMessages(); + const { workflowTodos, getTodosToTrack } = useBuilderTodos(); + const trackingSessionId = computed(() => rootStore.pushRef); const workflowPrompt = computed(() => { @@ -124,17 +157,74 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { initialGeneration.value = false; } + function incrementManualExecutionStats(type: 'success' | 'error') { + manualExecStatsInBetweenMessages.value[type]++; + } + + function resetManualExecutionStats() { + manualExecStatsInBetweenMessages.value = { + success: 0, + error: 0, + }; + } + // Message handling functions function addLoadingAssistantMessage(message: string) { builderThinkingMessage.value = message; } - function stopStreaming() { + function getWorkflowModifications({ + userMessageId, + startWorkflowJson, + }: EndOfStreamingTrackingPayload) { + const newToolMessages = toolMessages.value.filter((toolMsg) => + toolMsg.id?.startsWith(userMessageId), + ); + const endWorkflowJson = getWorkflowSnapshot(); + + return { + tools_called: dedupe(newToolMessages.map((toolMsg) => toolMsg.toolName)), + start_workflow_json: startWorkflowJson, + end_workflow_json: endWorkflowJson, + }; + } + + type StopStreamingPayload = + | { + error: string; + } + | { aborted: true }; + function trackEndBuilderResponse(payload?: StopStreamingPayload) { + if (!currentStreamingMessage.value) { + return; + } + + const { userMessageId } = currentStreamingMessage.value; + + telemetry.track('End of response from builder', { + // todo user_id available as trait already? + user_message_id: userMessageId, + workflow_id: workflowsStore.workflowId, + session_id: trackingSessionId.value, + ...getWorkflowModifications(currentStreamingMessage.value), + ...payload, + ...getTodosToTrack(), + }); + } + + function stopStreaming(payload?: StopStreamingPayload) { streaming.value = false; if (streamingAbortController.value) { streamingAbortController.value.abort(); streamingAbortController.value = null; } + + trackEndBuilderResponse(payload); + currentStreamingMessage.value = undefined; + } + + function abortStreaming() { + stopStreaming({ aborted: true }); } // Error handling @@ -144,10 +234,12 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { * The retry function, if provided, will remove the error message before retrying. * Tracks error telemetry */ - function handleServiceError(e: unknown, id: string, retry?: () => Promise) { + function handleServiceError(e: unknown, userMessageId: string, retry?: () => Promise) { assert(e instanceof Error); - stopStreaming(); + stopStreaming({ + error: e.message, + }); builderThinkingMessage.value = undefined; if (e.name === 'AbortError') { @@ -163,17 +255,11 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { const errorMessage = createErrorMessage( locale.baseText('aiAssistant.serviceError.message', { interpolate: { message: e.message } }), - id, + userMessageId, retry, ); chatMessages.value = [...chatMessages.value, errorMessage]; - - telemetry.track('Workflow generation errored', { - error: e.message, - session_id: trackingSessionId.value, - workflow_id: workflowsStore.workflowId, - }); } // Helper functions @@ -203,6 +289,68 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { }; } + // Telemetry functions + /** + * Tracks when a user submits a message to the builder. + * Captures workflow state, execution data, and todo counts for analytics. + */ + function trackUserSubmittedBuilderMessage(options: { + text: string; + source: 'chat' | 'canvas'; + type: 'message' | 'execution'; + userMessageId: string; + currentWorkflowJson: string; + errorMessage?: string; + errorNodeType?: string; + executionStatus?: string; + }) { + const { + text, + source, + type, + userMessageId, + currentWorkflowJson, + errorMessage, + errorNodeType, + executionStatus, + } = options; + + const trackingPayload: UserSubmittedBuilderMessageTrackingPayload = { + source, + message: text, + session_id: trackingSessionId.value, + start_workflow_json: currentWorkflowJson, + workflow_id: workflowsStore.workflowId, + type, + manual_exec_success_count_since_prev_msg: manualExecStatsInBetweenMessages.value.success, + manual_exec_error_count_since_prev_msg: manualExecStatsInBetweenMessages.value.error, + user_message_id: userMessageId, + ...getTodosToTrack(), + }; + + if (type === 'execution') { + let resultData = '{}'; + let resultDataSizeKb = 0; + + try { + resultData = JSON.stringify(workflowsStore.workflowExecutionData ?? {}); + resultDataSizeKb = stringSizeInBytes(resultData) / 1024; + } catch (error) { + // Handle circular structure errors gracefully + console.warn('Failed to stringify execution data for telemetry:', error); + } + + trackingPayload.execution_data = resultDataSizeKb > 512 ? '{}' : resultData; + trackingPayload.execution_status = executionStatus ?? ''; + if (executionStatus === 'error') { + trackingPayload.error_message = errorMessage ?? ''; + trackingPayload.error_node_type = errorNodeType ?? ''; + } + } + + telemetry.track('User submitted builder message', trackingPayload); + } + // Core API functions /** * Sends a message to the AI builder service and handles the streaming response. @@ -242,51 +390,38 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { if (options.initialGeneration !== undefined) { initialGeneration.value = options.initialGeneration; } - const messageId = generateMessageId(); - + const userMessageId = generateMessageId(); const currentWorkflowJson = getWorkflowSnapshot(); - const trackingPayload: Record = { - source, - message: text, - session_id: trackingSessionId.value, - start_workflow_json: currentWorkflowJson, - workflow_id: workflowsStore.workflowId, - type, + + currentStreamingMessage.value = { + userMessageId, + startWorkflowJson: currentWorkflowJson, }; - if (type === 'execution') { - let resultData = '{}'; - let resultDataSizeKb = 0; + trackUserSubmittedBuilderMessage({ + text, + source, + type, + userMessageId, + currentWorkflowJson, + errorMessage, + errorNodeType, + executionStatus, + }); - try { - resultData = JSON.stringify(workflowsStore.workflowExecutionData ?? {}); - resultDataSizeKb = stringSizeInBytes(resultData) / 1024; - } catch (error) { - // Handle circular structure errors gracefully - console.warn('Failed to stringify execution data for telemetry:', error); - } + resetManualExecutionStats(); - trackingPayload.execution_data = resultDataSizeKb > 512 ? '{}' : resultData; - trackingPayload.execution_status = executionStatus ?? ''; - if (executionStatus === 'error') { - trackingPayload.error_message = errorMessage ?? ''; - trackingPayload.error_node_type = errorNodeType ?? ''; - } - } - - telemetry.track('User submitted builder message', trackingPayload); - - prepareForStreaming(text, messageId); + prepareForStreaming(text, userMessageId); const executionResult = workflowsStore.workflowExecutionData?.data?.resultData; - const payload = createBuilderPayload(text, { + const payload = createBuilderPayload(text, userMessageId, { quickReplyType, workflow: workflowsStore.workflow, executionData: executionResult, nodesForSchema: Object.keys(workflowsStore.nodesByName), }); - const retry = createRetryHandler(messageId, async () => sendChatMessage(options)); + const retry = createRetryHandler(userMessageId, async () => sendChatMessage(options)); // Abort previous streaming request if any if (streamingAbortController.value) { @@ -302,7 +437,7 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { const result = processAssistantMessages( chatMessages.value, response.messages, - generateMessageId(), + userMessageId, retry, ); chatMessages.value = result.messages; @@ -315,11 +450,11 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { } }, () => stopStreaming(), - (e) => handleServiceError(e, messageId, retry), + (e) => handleServiceError(e, userMessageId, retry), streamingAbortController.value?.signal, ); } catch (e: unknown) { - handleServiceError(e, messageId, retry); + handleServiceError(e, userMessageId, retry); } } @@ -574,9 +709,10 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { creditsRemaining, hasNoCreditsRemaining, hasMessages: computed(() => hasMessages.value), + workflowTodos, // Methods - stopStreaming, + abortStreaming, resetBuilderChat, sendChatMessage, loadSessions, @@ -586,5 +722,7 @@ export const useBuilderStore = defineStore(STORES.BUILDER, () => { updateBuilderCredits, getRunningTools, fetchSessionsMetadata, + incrementManualExecutionStats, + resetManualExecutionStats, }; }); diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/builder.utils.ts b/packages/frontend/editor-ui/src/features/ai/assistant/builder.utils.ts index 7ecf56f1b6a..3fefce02360 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/builder.utils.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/builder.utils.ts @@ -8,12 +8,17 @@ import { import type { IRunExecutionData } from 'n8n-workflow'; import type { IWorkflowDb } from '@/Interface'; +export function generateShortId() { + return Math.random().toString(36).substr(2, 9); +} + export function generateMessageId(): string { - return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + return `${Date.now()}-${generateShortId()}`; } export function createBuilderPayload( text: string, + id: string, options: { quickReplyType?: string; executionData?: IRunExecutionData['resultData']; @@ -67,6 +72,7 @@ export function createBuilderPayload( return { role: 'user', type: 'message', + id, text, quickReplyType: options.quickReplyType, workflowContext, diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.test.ts b/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.test.ts index 08c610f0b1d..0ef45d86cfb 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.test.ts +++ b/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.test.ts @@ -1354,249 +1354,6 @@ describe('AskAssistantBuild', () => { }); }); - it('should track categorization telemetry when categorize_prompt tool completes', async () => { - renderComponent(); - - // Simulate streaming starts - builderStore.$patch({ streaming: true }); - await flushPromises(); - - // Add categorization tool message - builderStore.toolMessages = [ - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'categorize_prompt', - toolCallId: faker.string.uuid(), - status: 'completed', - updates: [ - { - type: 'output', - data: { - categorization: { - techniques: ['chatbot', 'notification'], - confidence: 0.85, - }, - }, - }, - ], - }, - ]; - - // Simulate streaming stops (this triggers trackWorkflowModifications) - builderStore.$patch({ streaming: false }); - await flushPromises(); - - expect(trackMock).toHaveBeenCalledWith('Classifier labels user prompt', { - user_id: undefined, - workflow_id: 'abc123', - classifier_labels: ['chatbot', 'notification'], - confidence: 0.85, - session_id: 'app_session_id', - timestamp: expect.any(String), - }); - }); - - it('should handle multiple canvas generations correctly', async () => { - const originalWorkflow = { - nodes: [], - connections: {}, - }; - builderStore.getWorkflowSnapshot.mockReturnValue(JSON.stringify(originalWorkflow)); - - const intermediaryWorkflow = { - nodes: [ - { - id: 'node1', - name: 'Start', - type: 'n8n-nodes-base.start', - position: [0, 0], - typeVersion: 1, - parameters: {}, - } as INodeUi, - ], - connections: {}, - }; - - const finalWorkflow = { - nodes: [ - { - id: 'node1', - name: 'Start', - type: 'n8n-nodes-base.start', - position: [0, 0], - typeVersion: 1, - parameters: {}, - } as INodeUi, - - { - id: 'node2', - name: 'HttpReuqest', - type: 'n8n-nodes-base.httpRequest', - position: [0, 0], - typeVersion: 1, - parameters: {}, - } as INodeUi, - ], - connections: {}, - }; - workflowsStore.$patch({ - workflow: originalWorkflow, - }); - - renderComponent(); - - builderStore.$patch({ streaming: true }); - await flushPromises(); - - // Trigger the watcher by updating workflowMessages - builderStore.workflowMessages = [ - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'workflow-updated' as const, - codeSnippet: JSON.stringify(intermediaryWorkflow), - }, - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'workflow-updated' as const, - codeSnippet: JSON.stringify(finalWorkflow), - }, - ]; - - const toolCallId1_1 = '1234'; - const toolCallId1_2 = '3333'; - const toolCallId2 = '4567'; - const toolCallId3 = '8901'; - - builderStore.toolMessages = [ - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'first-tool', - toolCallId: toolCallId1_1, - status: 'completed', - updates: [], - }, - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'first-tool', - toolCallId: toolCallId1_2, - status: 'completed', - updates: [], - }, - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'second-tool', - toolCallId: toolCallId2, - status: 'running', - updates: [], - }, - ]; - - builderStore.$patch({ streaming: false }); - await flushPromises(); - - expect(trackMock).toHaveBeenCalledOnce(); - expect(trackMock).toHaveBeenCalledWith('Workflow modified by builder', { - end_workflow_json: JSON.stringify(finalWorkflow), - session_id: 'app_session_id', - start_workflow_json: JSON.stringify(originalWorkflow), - // first-tool is added once, even though it completed twice - // second-tool is ignored because it's running - tools_called: ['first-tool'], - workflow_id: 'abc123', - }); - - trackMock.mockClear(); - - builderStore.$patch({ streaming: true }); - - await flushPromises(); - // second run after new messages - const updatedWorkflow2 = { - ...finalWorkflow, - nodes: [ - ...finalWorkflow.nodes, - { - id: 'node1', - name: 'Updated', - type: 'n8n-nodes-base.updated', - position: [0, 0], - typeVersion: 1, - parameters: {}, - }, - ], - }; - builderStore.workflowMessages = [ - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'workflow-updated' as const, - codeSnippet: JSON.stringify(updatedWorkflow2), - }, - ]; - - builderStore.toolMessages = [ - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'first-tool', - toolCallId: toolCallId1_1, - status: 'completed', - updates: [], - }, - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'first-tool', - toolCallId: toolCallId1_2, - status: 'completed', - updates: [], - }, - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'second-tool', - toolCallId: toolCallId2, - status: 'completed', - updates: [], - }, - { - id: faker.string.uuid(), - role: 'assistant' as const, - type: 'tool' as const, - toolName: 'third-tool', - toolCallId: toolCallId3, - status: 'completed', - updates: [], - }, - ]; - - builderStore.$patch({ streaming: false }); - await flushPromises(); - - expect(trackMock).toHaveBeenCalledOnce(); - expect(trackMock).toHaveBeenCalledWith('Workflow modified by builder', { - end_workflow_json: JSON.stringify(updatedWorkflow2), - session_id: 'app_session_id', - start_workflow_json: JSON.stringify(originalWorkflow), - // first-tool is ignored, because it was tracked in first run (same tool call id) - tools_called: ['second-tool', 'third-tool'], - workflow_id: 'abc123', - }); - }); - describe('shouldTidyUp logic', () => { it('should set tidyUp to true when new nodes are added', async () => { const originalWorkflow = { nodes: [], connections: {} }; diff --git a/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.vue b/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.vue index 317975bf44a..de6f41bb340 100644 --- a/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.vue +++ b/packages/frontend/editor-ui/src/features/ai/assistant/components/Agent/AskAssistantBuild.vue @@ -33,9 +33,6 @@ const { goToUpgrade } = usePageRedirectionHelper(); // Track processed workflow updates const processedWorkflowUpdates = ref(new Set()); -const trackedTools = ref(new Set()); -const trackedCategorizations = ref(new Set()); -const workflowUpdated = ref<{ start: string; end: string } | undefined>(); const shouldTidyUp = ref(false); const n8nChatRef = ref>(); @@ -107,9 +104,6 @@ async function onUserMessage(content: string) { function onNewWorkflow() { builderStore.resetBuilderChat(); processedWorkflowUpdates.value.clear(); - trackedTools.value.clear(); - trackedCategorizations.value.clear(); - workflowUpdated.value = undefined; shouldTidyUp.value = false; } @@ -130,71 +124,6 @@ function onFeedback(feedback: RatingFeedback) { } } -function dedupeToolNames(toolNames: string[]): string[] { - return [...new Set(toolNames)]; -} - -function isCategorizationData( - data: unknown, -): data is { techniques: string[]; confidence?: number } { - return ( - typeof data === 'object' && - data !== null && - 'techniques' in data && - Array.isArray(data.techniques) && - data.techniques.every((t) => typeof t === 'string') - ); -} - -function trackWorkflowCategorization() { - // Track categorization telemetry - builderStore.toolMessages.forEach((toolMsg) => { - if (toolMsg.toolName !== 'categorize_prompt') return; - if (toolMsg.status !== 'completed') return; - if (!toolMsg.toolCallId) return; - if (trackedCategorizations.value.has(toolMsg.toolCallId)) return; - - const outputUpdate = toolMsg.updates.find((u) => u.type === 'output'); - const categorizationData = outputUpdate?.data?.categorization; - - if (!isCategorizationData(categorizationData)) return; - - trackedCategorizations.value.add(toolMsg.toolCallId); - - telemetry.track('Classifier labels user prompt', { - user_id: usersStore.currentUserId ?? undefined, - workflow_id: workflowsStore.workflowId, - classifier_labels: categorizationData.techniques, - confidence: categorizationData.confidence, - session_id: builderStore.trackingSessionId, - timestamp: new Date().toISOString(), - }); - }); -} - -function trackWorkflowModifications() { - if (workflowUpdated.value) { - // Track tool usage for telemetry - const newToolMessages = builderStore.toolMessages.filter( - (toolMsg) => - toolMsg.status !== 'running' && - toolMsg.toolCallId && - !trackedTools.value.has(toolMsg.toolCallId), - ); - - newToolMessages.forEach((toolMsg) => trackedTools.value.add(toolMsg.toolCallId ?? '')); - telemetry.track('Workflow modified by builder', { - tools_called: dedupeToolNames(newToolMessages.map((toolMsg) => toolMsg.toolName)), - session_id: builderStore.trackingSessionId, - start_workflow_json: workflowUpdated.value.start, - end_workflow_json: workflowUpdated.value.end, - workflow_id: workflowsStore.workflowId, - }); - - workflowUpdated.value = undefined; - } -} - function onWorkflowExecuted() { const executionData = workflowsStore.workflowExecutionData; const executionStatus = executionData?.status ?? 'unknown'; @@ -257,8 +186,6 @@ watch( if (msg.id && isWorkflowUpdatedMessage(msg)) { processedWorkflowUpdates.value.add(msg.id); - const originalWorkflowJson = - workflowUpdated.value?.start ?? builderStore.getWorkflowSnapshot(); const result = builderStore.applyWorkflowUpdate(msg.codeSnippet); if (result.success) { @@ -274,11 +201,6 @@ watch( regenerateIds: false, trackEvents: false, }); - - workflowUpdated.value = { - start: originalWorkflowJson, - end: msg.codeSnippet, - }; } } }); @@ -291,11 +213,6 @@ watch( watch( () => builderStore.streaming, async (isStreaming) => { - if (!isStreaming) { - trackWorkflowModifications(); - trackWorkflowCategorization(); - } - if ( builderStore.initialGeneration && !isStreaming && @@ -348,7 +265,7 @@ defineExpose({ @message="onUserMessage" @upgrade-click="() => goToUpgrade('ai-builder-sidebar', 'upgrade-builder')" @feedback="onFeedback" - @stop="builderStore.stopStreaming" + @stop="builderStore.abortStreaming" >