feat(copilot): attribute tool metrics to invoking agents (#6266)

* feat(copilot): attribute Sim tool calls to agents

* fix(review): bound agent metric labels

* feat(copilot): attribute tool latency to agents

* feat(copilot): attribute tool spans to agents

* refactor(copilot): simplify tool attribute naming

* fix(copilot): bound tool span agent labels

apps/sim/lib/copilot/request/handlers/handlers.test.ts

@@ -408,6 +408,7 @@ describe('sse-handlers tool lifecycle', () => {
 
     const updated = context.toolCalls.get('tool-1')
     expect(updated?.status).toBe(MothershipStreamV1ToolOutcome.success)
+    expect(updated?.agentId).toBe('main')
     // Display titles are derived client-side from the tool name (+args), not the
     // stream; read with no path resolves to the static "Reading file".
     expect(updated?.displayTitle).toBe('Reading file')
@@ -889,9 +890,11 @@ describe('sse-handlers tool lifecycle', () => {
       expect.any(Object)
     )
     expect(context.toolCalls.get('sub-tool-1')?.params).toEqual({ name: 'Example Workflow' })
+    expect(context.toolCalls.get('sub-tool-1')?.agentId).toBe('workflow')
     expect(context.subAgentToolCalls['parent-1']?.[0]?.params).toEqual({
       name: 'Example Workflow',
     })
+    expect(context.subAgentToolCalls['parent-1']?.[0]?.agentId).toBe('workflow')
   })
 
   it('routes subagent text using the event scope parent tool call id', async () => {
@@ -973,6 +976,40 @@ describe('sse-handlers tool lifecycle', () => {
     await sleep(0)
 
     expect(context.subAgentToolCalls['parent-1']?.[0]?.id).toBe('sub-tool-scope-1')
+    expect(context.toolCalls.get('sub-tool-scope-1')?.agentId).toBe('deploy')
+  })
+
+  it('retains the first agent attribution on replayed partial tool calls', async () => {
+    context.toolCalls.set('replayed-read', {
+      id: 'replayed-read',
+      name: 'read',
+      status: 'executing',
+    })
+
+    const replayPartial = (agentId: string) =>
+      subAgentHandlers.tool(
+        {
+          type: MothershipStreamV1EventType.tool,
+          scope: { lane: 'subagent', parentToolCallId: 'parent-1', agentId },
+          payload: {
+            toolCallId: 'replayed-read',
+            toolName: 'read',
+            executor: MothershipStreamV1ToolExecutor.go,
+            mode: MothershipStreamV1ToolMode.sync,
+            phase: MothershipStreamV1ToolPhase.call,
+            status: 'generating',
+            partial: true,
+          },
+        } satisfies StreamEvent,
+        context,
+        execContext,
+        { interactive: false, timeout: 1000 }
+      )
+
+    await replayPartial('workflow')
+    await replayPartial('deploy')
+
+    expect(context.toolCalls.get('replayed-read')?.agentId).toBe('workflow')
   })
 
   it('pairs compaction lifecycle events within each scoped subagent lane', async () => {

apps/sim/lib/copilot/request/handlers/tool.ts

@@ -285,6 +285,7 @@ export async function handleToolEvent(
 ): Promise<void> {
   const isSubagent = scope === 'subagent'
   const parentToolCallId = isSubagent ? getScopedParentToolCallId(event, context) : undefined
+  const agentId = event.scope?.agentId ?? 'main'
 
   if (isSubagent && !parentToolCallId) return
 
@@ -332,6 +333,7 @@ export async function handleToolEvent(
     options,
     parentToolCallId,
     scope,
+    agentId,
     getScopedSpanIdentity(event)
   )
 }
@@ -409,6 +411,7 @@ async function handleCallPhase(
   options: OrchestratorOptions,
   parentToolCallId: string | undefined,
   scope: ToolScope,
+  agentId: string,
   spanIdentity: { spanId?: string; parentSpanId?: string }
 ): Promise<void> {
   const { toolCallId, toolName } = data
@@ -416,6 +419,7 @@ async function handleCallPhase(
   const isGenerating = data.status === TOOL_CALL_STATUS.generating
   const isPartial = data.partial === true || isGenerating
   const existing = context.toolCalls.get(toolCallId)
+  if (existing) existing.agentId ??= agentId
   const isSubagent = scope === 'subagent'
   const ui = getToolCallUI(data)
 
@@ -459,12 +463,13 @@ async function handleCallPhase(
       toolName,
       args,
       parentToolCallId!,
+      agentId,
       ui,
       spanIdentity,
       !isPartial
     )
   } else {
-    registerMainToolCall(context, toolCallId, toolName, args, existing, ui, !isPartial)
+    registerMainToolCall(context, toolCallId, toolName, args, existing, agentId, ui, !isPartial)
   }
 
   if (isPartial) return
@@ -554,6 +559,7 @@ function registerSubagentToolCall(
   toolName: string,
   args: Record<string, unknown> | undefined,
   parentToolCallId: string,
+  agentId: string,
   ui: { title?: string; phaseLabel?: string; hidden?: boolean },
   spanIdentity: { spanId?: string; parentSpanId?: string },
   finalized: boolean
@@ -574,6 +580,7 @@ function registerSubagentToolCall(
       id: toolCallId,
       name: toolName,
       status: 'pending',
+      agentId,
       params: args,
       startTime: Date.now(),
     }
@@ -594,6 +601,7 @@ function registerSubagentToolCall(
   const subagentToolCalls = context.subAgentToolCalls[parentToolCallId]
   const existingSubagentToolCall = subagentToolCalls.find((tc) => tc.id === toolCallId)
   if (existingSubagentToolCall) {
+    existingSubagentToolCall.agentId ??= agentId
     if (!rebindResolvedIntegrationCall(existingSubagentToolCall, toolName, args)) {
       updateToolCallFromFrame(existingSubagentToolCall, toolName, args, finalized)
     }
@@ -609,6 +617,7 @@ function registerMainToolCall(
   toolName: string,
   args: Record<string, unknown> | undefined,
   existing: ToolCallState | undefined,
+  agentId: string,
   ui: { title?: string; phaseLabel?: string; hidden?: boolean },
   finalized: boolean
 ): void {
@@ -633,6 +642,7 @@ function registerMainToolCall(
       id: toolCallId,
       name: toolName,
       status: 'pending',
+      agentId,
       params: args,
       startTime: Date.now(),
     }

apps/sim/lib/copilot/request/metrics.test.ts

@@ -0,0 +1,78 @@
+/**
+ * @vitest-environment node
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { toolCallsAdd, toolDurationRecord } = vi.hoisted(() => ({
+  toolCallsAdd: vi.fn(),
+  toolDurationRecord: vi.fn(),
+}))
+
+vi.mock('@opentelemetry/api', () => ({
+  metrics: {
+    getMeter: vi.fn(() => ({
+      createCounter: vi.fn(() => ({ add: toolCallsAdd })),
+      createHistogram: vi.fn((name: string) => ({
+        record: name === 'copilot.tool.duration' ? toolDurationRecord : vi.fn(),
+      })),
+    })),
+  },
+}))
+
+import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
+import { normalizeToolAgentId, recordSimToolMetric } from '@/lib/copilot/request/metrics'
+
+describe('recordSimToolMetric', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it.each(['main', 'workflow'])(
+    'attributes call counts and duration to the registered %s agent',
+    (agentId) => {
+      recordSimToolMetric('read', agentId, 'success', 125)
+
+      const baseAttributes = {
+        [TraceAttr.ToolName]: 'read',
+        [TraceAttr.ToolExecutor]: 'sim',
+        [TraceAttr.ToolOutcome]: 'success',
+      }
+      expect(toolCallsAdd).toHaveBeenCalledWith(1, {
+        ...baseAttributes,
+        [TraceAttr.GenAiAgentName]: agentId,
+      })
+      expect(toolDurationRecord).toHaveBeenCalledWith(125, {
+        ...baseAttributes,
+        [TraceAttr.GenAiAgentName]: agentId,
+      })
+    }
+  )
+
+  it('collapses unknown agent IDs to the bounded fallback', () => {
+    recordSimToolMetric('read', 'tenant-defined-agent', 'success', 125)
+
+    const baseAttributes = {
+      [TraceAttr.ToolName]: 'read',
+      [TraceAttr.ToolExecutor]: 'sim',
+      [TraceAttr.ToolOutcome]: 'success',
+    }
+    expect(toolCallsAdd).toHaveBeenCalledWith(1, {
+      ...baseAttributes,
+      [TraceAttr.GenAiAgentName]: 'other',
+    })
+    expect(toolDurationRecord).toHaveBeenCalledWith(125, {
+      ...baseAttributes,
+      [TraceAttr.GenAiAgentName]: 'other',
+    })
+  })
+
+  it.each([
+    { agentId: 'main', expected: 'main' },
+    { agentId: 'workflow', expected: 'workflow' },
+    { agentId: 'tenant-defined-agent', expected: 'other' },
+    { agentId: '', expected: 'other' },
+  ])('normalizes $agentId to $expected for every telemetry signal', ({ agentId, expected }) => {
+    expect(normalizeToolAgentId(agentId)).toBe(expected)
+  })
+})

apps/sim/lib/copilot/request/metrics.ts

@@ -5,9 +5,9 @@
 // contracts/metrics_v1.go) so the Go∪Sim union is queryable as one series set
 // — e.g. `copilot.tool.duration` split by `tool.executor` (go|client|sim).
 //
-// Bounded cardinality only: tool.name is capped to the shared tool catalog
-// (else "other"); vfs phase / file-read outcome are bounded sets. NEVER a
-// user/chat/request id (those explode Prometheus series).
+// Bounded cardinality only: tool.name and gen_ai.agent.name are capped to the
+// shared catalogs (else "other"); vfs phase / file-read outcome are bounded
+// sets. NEVER a user/chat/request id (those explode Prometheus series).
 import { type Counter, type Histogram, metrics } from '@opentelemetry/api'
 import { Metric } from '@/lib/copilot/generated/metrics-v1'
 import { TOOL_CATALOG } from '@/lib/copilot/generated/tool-catalog-v1'
@@ -68,15 +68,30 @@ function cappedToolName(name: string): string {
   return TOOL_CATALOG[name] ? name : 'other'
 }
 
+const REGISTERED_AGENT_IDS = new Set([
+  'main',
+  ...Object.values(TOOL_CATALOG).flatMap(({ subagentId }) => (subagentId ? [subagentId] : [])),
+])
+
+export function normalizeToolAgentId(agentId: string): string {
+  return REGISTERED_AGENT_IDS.has(agentId) ? agentId : 'other'
+}
+
 // recordSimToolMetric emits copilot.tool.calls (+1) and copilot.tool.duration
 // for one server-side Sim tool dispatch (executor=sim). outcome is the bounded
 // tool outcome (success/error/…). Pure telemetry.
-export function recordSimToolMetric(name: string, outcome: string, durationMs: number): void {
+export function recordSimToolMetric(
+  name: string,
+  agentId: string,
+  outcome: string,
+  durationMs: number
+): void {
   const { toolDuration, toolCalls } = instruments()
   const attrs = {
     [TraceAttr.ToolName]: cappedToolName(name),
     [TraceAttr.ToolExecutor]: 'sim',
     [TraceAttr.ToolOutcome]: outcome,
+    [TraceAttr.GenAiAgentName]: normalizeToolAgentId(agentId),
   }
   toolCalls.add(1, attrs)
   if (durationMs >= 0) toolDuration.record(durationMs, attrs)

apps/sim/lib/copilot/request/otel.ts

@@ -22,6 +22,7 @@ import {
 import { TraceAttr } from '@/lib/copilot/generated/trace-attributes-v1'
 import { TraceSpan } from '@/lib/copilot/generated/trace-spans-v1'
 import { contextFromRequestHeaders } from '@/lib/copilot/request/go/propagation'
+import { normalizeToolAgentId } from '@/lib/copilot/request/metrics'
 import { isExplicitStopReason } from '@/lib/copilot/request/session/abort-reason'
 
 // OTel GenAI content-capture env var (spec:
@@ -284,6 +285,7 @@ export async function withCopilotToolSpan<T>(
   input: {
     toolName: string
     toolCallId: string
+    agentName: string
     runId?: string
     chatId?: string
     argsBytes?: number
@@ -299,6 +301,7 @@ export async function withCopilotToolSpan<T>(
         [TraceAttr.ToolName]: input.toolName,
         [TraceAttr.ToolCallId]: input.toolCallId,
         [TraceAttr.ToolExecutor]: 'sim',
+        [TraceAttr.GenAiAgentName]: normalizeToolAgentId(input.agentName),
         ...(input.runId ? { [TraceAttr.RunId]: input.runId } : {}),
         ...(input.chatId ? { [TraceAttr.ChatId]: input.chatId } : {}),
         ...(typeof input.argsBytes === 'number'

apps/sim/lib/copilot/request/tools/executor.test.ts

@@ -1,13 +1,37 @@
 import '@sim/testing/mocks/executor'
 
-import { describe, expect, it } from 'vitest'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { recordSimToolMetric, setAttribute, withCopilotToolSpan } = vi.hoisted(() => {
+  const setAttribute = vi.fn()
+  return {
+    recordSimToolMetric: vi.fn(),
+    setAttribute,
+    withCopilotToolSpan: vi.fn(
+      (_input: unknown, fn: (span: { setAttribute: typeof setAttribute }) => Promise<unknown>) =>
+        fn({ setAttribute })
+    ),
+  }
+})
+
+vi.mock('@/lib/copilot/request/metrics', () => ({
+  recordSimToolMetric,
+}))
+
+vi.mock('@/lib/copilot/request/otel', () => ({
+  withCopilotToolSpan,
+}))
+
 import { TOOL_WATCHDOG_DEFAULT_MS, TOOL_WATCHDOG_LONG_RUNNING_MS } from '@/lib/copilot/constants'
+import { MothershipStreamV1ToolOutcome } from '@/lib/copilot/generated/mothership-stream-v1'
+import { createStreamingContext } from '@/lib/copilot/request/context/request-context'
 import {
   buildToolExecutionContext,
+  executeToolAndReport,
   pendingToolWaitBudgetMs,
   toolWatchdogTimeoutMs,
 } from '@/lib/copilot/request/tools/executor'
-import type { ExecutionContext } from '@/lib/copilot/request/types'
+import type { ExecutionContext, ToolCallState } from '@/lib/copilot/request/types'
 
 describe('toolWatchdogTimeoutMs', () => {
   it('gives request-scoped MCP tools the long-running watchdog', () => {
@@ -58,3 +82,74 @@ describe('buildToolExecutionContext', () => {
     })
   })
 })
+
+describe('executeToolAndReport metrics', () => {
+  const executionContext: ExecutionContext = {
+    userId: 'user-1',
+    workflowId: 'workflow-1',
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('forwards the stored agent on normal completion', async () => {
+    const toolCall: ToolCallState = {
+      id: 'call-1',
+      name: 'read',
+      status: MothershipStreamV1ToolOutcome.success,
+      result: { success: true, output: 'done' },
+      agentId: 'workflow',
+      endTime: Date.now(),
+    }
+    const context = createStreamingContext({
+      toolCalls: new Map([[toolCall.id, toolCall]]),
+    })
+
+    await executeToolAndReport(toolCall.id, context, executionContext)
+
+    expect(recordSimToolMetric).toHaveBeenCalledWith(
+      'read',
+      'workflow',
+      MothershipStreamV1ToolOutcome.success,
+      expect.any(Number)
+    )
+    expect(withCopilotToolSpan).toHaveBeenCalledWith(
+      expect.objectContaining({ agentName: 'workflow' }),
+      expect.any(Function)
+    )
+  })
+
+  it.each([
+    { agentId: 'workflow', expectedAgentId: 'workflow' },
+    { agentId: undefined, expectedAgentId: 'main' },
+  ])(
+    'forwards $expectedAgentId when an unexpected error occurs',
+    async ({ agentId, expectedAgentId }) => {
+      const toolCall: ToolCallState = {
+        id: 'call-2',
+        name: 'read',
+        status: MothershipStreamV1ToolOutcome.error,
+        agentId,
+        endTime: Date.now(),
+      }
+      const context = createStreamingContext({
+        toolCalls: new Map([[toolCall.id, toolCall]]),
+      })
+
+      await expect(executeToolAndReport(toolCall.id, context, executionContext)).rejects.toThrow(
+        'missing a canonical error'
+      )
+      expect(recordSimToolMetric).toHaveBeenCalledWith(
+        'read',
+        expectedAgentId,
+        MothershipStreamV1ToolOutcome.error,
+        expect.any(Number)
+      )
+      expect(withCopilotToolSpan).toHaveBeenCalledWith(
+        expect.objectContaining({ agentName: expectedAgentId }),
+        expect.any(Function)
+      )
+    }
+  )
+})

apps/sim/lib/copilot/request/tools/executor.ts

@@ -408,6 +408,7 @@ export async function executeToolAndReport(
     {
       toolName: toolCall.name,
       toolCallId: toolCall.id,
+      agentName: toolCall.agentId ?? 'main',
       runId: context.runId,
       chatId: execContext.chatId,
       argsBytes: argsPayload?.length,
@@ -428,7 +429,12 @@ export async function executeToolAndReport(
         }
         // Durable Grafana signal for "which Sim tool is slowest" (executor=sim);
         // pairs with the Go executor-boundary metric (U15) as one series set.
-        recordSimToolMetric(toolCall.name, completion.status, durationMs)
+        recordSimToolMetric(
+          toolCall.name,
+          toolCall.agentId ?? 'main',
+          completion.status,
+          durationMs
+        )
         return completion
       } catch (err) {
         // executeToolAndReportInner threw (infra/unexpected error, not a normal
@@ -437,7 +443,12 @@ export async function executeToolAndReport(
         const durationMs = Date.now() - startedAt
         otelSpan.setAttribute(TraceAttr.ToolOutcome, 'error')
         otelSpan.setAttribute(TraceAttr.ToolDurationMs, durationMs)
-        recordSimToolMetric(toolCall.name, 'error', durationMs)
+        recordSimToolMetric(
+          toolCall.name,
+          toolCall.agentId ?? 'main',
+          MothershipStreamV1ToolOutcome.error,
+          durationMs
+        )
         throw err
       }
     }

apps/sim/lib/copilot/request/types.ts

@@ -25,6 +25,8 @@ export interface ToolCallState {
   id: string
   name: string
   status: ToolCallStatus
+  /** Bounded registry ID of the agent that invoked this tool. */
+  agentId?: string
   displayTitle?: string
   /** Model-authored activity text for a gateway-resolved integration call. */
   integrationDescription?: string