From 6215a6e8dae3d00fac8b5a34bc9798fbd55d784f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 15 Mar 2025 15:36:13 -0700 Subject: [PATCH] feat(logs): added gantt chart/trace span for workflow executions --- sim/app/executor/handlers.ts | 5 +- sim/app/lib/logs/execution-logger.ts | 4 + sim/app/providers/anthropic/index.ts | 360 +++++++++------ sim/app/providers/cerebras/index.ts | 118 ++++- sim/app/providers/deepseek/index.ts | 419 +++++++++++------- sim/app/providers/google/index.ts | 415 ++++++++++------- sim/app/providers/groq/index.ts | 409 ++++++++++------- sim/app/providers/openai/index.ts | 143 ++++-- sim/app/providers/types.ts | 18 + sim/app/providers/xai/index.ts | 398 ++++++++++------- .../w/[id]/hooks/use-workflow-execution.ts | 251 ++++++++++- sim/app/w/logs/components/sidebar/sidebar.tsx | 28 ++ .../trace-spans/trace-spans-display.tsx | 359 +++++++++++++++ sim/app/w/logs/stores/types.ts | 19 +- 14 files changed, 2144 insertions(+), 802 deletions(-) create mode 100644 sim/app/w/logs/components/trace-spans/trace-spans-display.tsx diff --git a/sim/app/executor/handlers.ts b/sim/app/executor/handlers.ts index 6795ea2fa5..4679f9c289 100644 --- a/sim/app/executor/handlers.ts +++ b/sim/app/executor/handlers.ts @@ -257,7 +257,7 @@ export class AgentBlockHandler implements BlockHandler { toolCallsCount: response.toolCalls?.length || 0, }) - // For structured responses, try to parse the content + // If structured responses, try to parse the content if (responseFormat) { try { const parsedContent = JSON.parse(response.content) @@ -284,6 +284,7 @@ export class AgentBlockHandler implements BlockHandler { count: response.toolCalls.length, } : undefined, + providerTiming: response.timing || undefined, }, } @@ -316,6 +317,7 @@ export class AgentBlockHandler implements BlockHandler { : [], count: response.toolCalls?.length || 0, }, + providerTiming: response.timing || undefined, }, } } @@ -345,6 +347,7 @@ export class AgentBlockHandler implements BlockHandler { : [], count: response.toolCalls?.length || 0, }, + providerTiming: response.timing || undefined, }, } } diff --git a/sim/app/lib/logs/execution-logger.ts b/sim/app/lib/logs/execution-logger.ts index cacb02dfc5..75260a5115 100644 --- a/sim/app/lib/logs/execution-logger.ts +++ b/sim/app/lib/logs/execution-logger.ts @@ -403,6 +403,10 @@ export async function persistExecutionLogs( duration: result.success ? `${totalDuration}ms` : 'NA', trigger: triggerType, createdAt: new Date(), + metadata: { + traceSpans: (result as any).traceSpans || [], + totalDuration: (result as any).totalDuration || totalDuration, + }, }) } catch (error: any) { logger.error(`Error persisting execution logs: ${error.message}`, { error }) diff --git a/sim/app/providers/anthropic/index.ts b/sim/app/providers/anthropic/index.ts index 8c4e559916..6efc7ff024 100644 --- a/sim/app/providers/anthropic/index.ts +++ b/sim/app/providers/anthropic/index.ts @@ -1,7 +1,7 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('Anthropic Provider') @@ -186,153 +186,251 @@ ${fieldDescriptions} payload.tools = tools } - // Make the initial API request - let currentResponse = await anthropic.messages.create(payload) - let content = '' - - // Extract text content from the message - if (Array.isArray(currentResponse.content)) { - content = currentResponse.content - .filter((item) => item.type === 'text') - .map((item) => item.text) - .join('\n') - } - - let tokens = { - prompt: currentResponse.usage?.input_tokens || 0, - completion: currentResponse.usage?.output_tokens || 0, - total: - (currentResponse.usage?.input_tokens || 0) + (currentResponse.usage?.output_tokens || 0), - } - - let toolCalls = [] - let toolResults = [] - let currentMessages = [...messages] - let iterationCount = 0 - const MAX_ITERATIONS = 10 // Prevent infinite loops + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() try { - while (iterationCount < MAX_ITERATIONS) { - // Check for tool calls - const toolUses = currentResponse.content.filter((item) => item.type === 'tool_use') - if (!toolUses || toolUses.length === 0) { - break - } + // Make the initial API request + const initialCallTime = Date.now() + let currentResponse = await anthropic.messages.create(payload) + const firstResponseTime = Date.now() - initialCallTime - // Process each tool call - for (const toolUse of toolUses) { - try { - const toolName = toolUse.name - const toolArgs = toolUse.input as Record + let content = '' - // Get the tool from the tools registry - const tool = request.tools?.find((t) => t.id === toolName) - if (!tool) continue - - // Execute the tool - const mergedArgs = { ...tool.params, ...toolArgs } - const result = await executeTool(toolName, mergedArgs) - - if (!result.success) continue - - toolResults.push(result.output) - toolCalls.push({ - name: toolName, - arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, - result: result.output, - }) - - // Add the tool call and result to messages - const toolUseId = generateToolUseId(toolName) - - currentMessages.push({ - role: 'assistant', - content: [ - { - type: 'tool_use', - id: toolUseId, - name: toolName, - input: toolArgs, - } as any, - ], - }) - - currentMessages.push({ - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: toolUseId, - content: JSON.stringify(result.output), - } as any, - ], - }) - } catch (error) { - logger.error('Error processing tool call:', { error }) - } - } - - // Make the next request with updated messages - const nextPayload = { - ...payload, - messages: currentMessages, - } - - // Make the next request - currentResponse = await anthropic.messages.create(nextPayload) - - // Update content if we have a text response - const textContent = currentResponse.content + // Extract text content from the message + if (Array.isArray(currentResponse.content)) { + content = currentResponse.content .filter((item) => item.type === 'text') .map((item) => item.text) .join('\n') + } - if (textContent) { - content = textContent + let tokens = { + prompt: currentResponse.usage?.input_tokens || 0, + completion: currentResponse.usage?.output_tokens || 0, + total: + (currentResponse.usage?.input_tokens || 0) + (currentResponse.usage?.output_tokens || 0), + } + + let toolCalls = [] + let toolResults = [] + let currentMessages = [...messages] + let iterationCount = 0 + const MAX_ITERATIONS = 10 // Prevent infinite loops + + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 + + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + + try { + while (iterationCount < MAX_ITERATIONS) { + // Check for tool calls + const toolUses = currentResponse.content.filter((item) => item.type === 'tool_use') + if (!toolUses || toolUses.length === 0) { + break + } + + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + + // Process each tool call + for (const toolUse of toolUses) { + try { + const toolName = toolUse.name + const toolArgs = toolUse.input as Record + + // Get the tool from the tools registry + const tool = request.tools?.find((t) => t.id === toolName) + if (!tool) continue + + // Execute the tool + const toolCallStartTime = Date.now() + const mergedArgs = { ...tool.params, ...toolArgs } + const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime + + if (!result.success) continue + + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + + toolResults.push(result.output) + toolCalls.push({ + name: toolName, + arguments: toolArgs, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, + result: result.output, + }) + + // Add the tool call and result to messages + const toolUseId = generateToolUseId(toolName) + + currentMessages.push({ + role: 'assistant', + content: [ + { + type: 'tool_use', + id: toolUseId, + name: toolName, + input: toolArgs, + } as any, + ], + }) + + currentMessages.push({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: toolUseId, + content: JSON.stringify(result.output), + } as any, + ], + }) + } catch (error) { + logger.error('Error processing tool call:', { error }) + } + } + + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + // Make the next request with updated messages + const nextPayload = { + ...payload, + messages: currentMessages, + } + + // Time the next model call + const nextModelStartTime = Date.now() + + // Make the next request + currentResponse = await anthropic.messages.create(nextPayload) + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + + // Update content if we have a text response + const textContent = currentResponse.content + .filter((item) => item.type === 'text') + .map((item) => item.text) + .join('\n') + + if (textContent) { + content = textContent + } + + // Update token counts + if (currentResponse.usage) { + tokens.prompt += currentResponse.usage.input_tokens || 0 + tokens.completion += currentResponse.usage.output_tokens || 0 + tokens.total += + (currentResponse.usage.input_tokens || 0) + (currentResponse.usage.output_tokens || 0) + } + + iterationCount++ } + } catch (error) { + logger.error('Error in Anthropic request:', { error }) + throw error + } - // Update token counts - if (currentResponse.usage) { - tokens.prompt += currentResponse.usage.input_tokens || 0 - tokens.completion += currentResponse.usage.output_tokens || 0 - tokens.total += - (currentResponse.usage.input_tokens || 0) + (currentResponse.usage.output_tokens || 0) + // If the content looks like it contains JSON, extract just the JSON part + if (content.includes('{') && content.includes('}')) { + try { + const jsonMatch = content.match(/\{[\s\S]*\}/m) + if (jsonMatch) { + content = jsonMatch[0] + } + } catch (e) { + logger.error('Error extracting JSON from response:', { error: e }) } + } - iterationCount++ + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + return { + content, + model: request.model || 'claude-3-7-sonnet-20250219', + tokens, + toolCalls: + toolCalls.length > 0 + ? toolCalls.map((tc) => ({ + name: tc.name, + arguments: tc.arguments as Record, + startTime: tc.startTime, + endTime: tc.endTime, + duration: tc.duration, + result: tc.result, + })) + : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, } } catch (error) { - logger.error('Error in Anthropic request:', { error }) - throw error - } + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - // If the content looks like it contains JSON, extract just the JSON part - if (content.includes('{') && content.includes('}')) { - try { - const jsonMatch = content.match(/\{[\s\S]*\}/m) - if (jsonMatch) { - content = jsonMatch[0] - } - } catch (e) { - logger.error('Error extracting JSON from response:', { error: e }) + logger.error('Error in Anthropic request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, } - } - return { - content, - model: request.model || 'claude-3-7-sonnet-20250219', - tokens, - toolCalls: - toolCalls.length > 0 - ? toolCalls.map((tc) => ({ - name: tc.name, - arguments: tc.arguments as Record, - })) - : undefined, - toolResults: toolResults.length > 0 ? toolResults : undefined, + throw enhancedError } }, } diff --git a/sim/app/providers/cerebras/index.ts b/sim/app/providers/cerebras/index.ts index 1c016e77f2..4b61ef6781 100644 --- a/sim/app/providers/cerebras/index.ts +++ b/sim/app/providers/cerebras/index.ts @@ -1,7 +1,7 @@ import { Cerebras } from '@cerebras/cerebras_cloud_sdk' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('Cerebras Provider') @@ -17,6 +17,10 @@ export const cerebrasProvider: ProviderConfig = { throw new Error('API key is required for Cerebras') } + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() + try { const client = new Cerebras({ apiKey: request.apiKey, @@ -83,7 +87,9 @@ export const cerebrasProvider: ProviderConfig = { } // Make the initial API request + const initialCallTime = Date.now() let currentResponse = (await client.chat.completions.create(payload)) as CerebrasResponse + const firstResponseTime = Date.now() - initialCallTime let content = currentResponse.choices[0]?.message?.content || '' let tokens = { @@ -97,6 +103,21 @@ export const cerebrasProvider: ProviderConfig = { let iterationCount = 0 const MAX_ITERATIONS = 10 // Prevent infinite loops + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 + + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + // Keep track of processed tool calls to avoid duplicates const processedToolCallIds = new Set() // Keep track of tool call signatures to detect repeats @@ -115,6 +136,9 @@ export const cerebrasProvider: ProviderConfig = { break } + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + // Process each tool call let processedAnyToolCall = false let hasRepeatedToolCalls = false @@ -145,18 +169,30 @@ export const cerebrasProvider: ProviderConfig = { if (!tool) continue // Execute the tool + const toolCallStartTime = Date.now() const mergedArgs = { ...tool.params, ...toolArgs } const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime if (!result.success) continue + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + toolResults.push(result.output) toolCalls.push({ name: toolName, arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, result: result.output, }) @@ -186,8 +222,15 @@ export const cerebrasProvider: ProviderConfig = { } } + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + // After processing tool calls, get a final response if (processedAnyToolCall || hasRepeatedToolCalls) { + // Time the next model call + const nextModelStartTime = Date.now() + // Make the final request const finalPayload = { ...payload, @@ -199,6 +242,21 @@ export const cerebrasProvider: ProviderConfig = { finalPayload )) as CerebrasResponse + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Final response`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + if (finalResponse.choices[0]?.message?.content) { content = finalResponse.choices[0].message.content } @@ -221,11 +279,29 @@ export const cerebrasProvider: ProviderConfig = { messages: currentMessages, } + // Time the next model call + const nextModelStartTime = Date.now() + // Make the next request currentResponse = (await client.chat.completions.create( nextPayload )) as CerebrasResponse + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + // Update token counts if (currentResponse.usage) { tokens.prompt += currentResponse.usage.prompt_tokens || 0 @@ -240,16 +316,46 @@ export const cerebrasProvider: ProviderConfig = { logger.error('Error in Cerebras tool processing:', { error }) } + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + return { content, model: request.model, tokens, toolCalls: toolCalls.length > 0 ? toolCalls : undefined, toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, } } catch (error) { - logger.error('Error in Cerebras request:', { error }) - throw error + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + logger.error('Error in Cerebras request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/sim/app/providers/deepseek/index.ts b/sim/app/providers/deepseek/index.ts index fe9e01531c..1ad4f19762 100644 --- a/sim/app/providers/deepseek/index.ts +++ b/sim/app/providers/deepseek/index.ts @@ -1,7 +1,7 @@ import OpenAI from 'openai' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('Deepseek Provider') @@ -18,184 +18,277 @@ export const deepseekProvider: ProviderConfig = { throw new Error('API key is required for Deepseek') } - // Deepseek uses the OpenAI SDK with a custom baseURL - const deepseek = new OpenAI({ - apiKey: request.apiKey, - baseURL: 'https://api.deepseek.com/v1', - dangerouslyAllowBrowser: true, - }) - - // Start with an empty array for all messages - const allMessages = [] - - // Add system prompt if present - if (request.systemPrompt) { - allMessages.push({ - role: 'system', - content: request.systemPrompt, - }) - } - - // Add context if present - if (request.context) { - allMessages.push({ - role: 'user', - content: request.context, - }) - } - - // Add remaining messages - if (request.messages) { - allMessages.push(...request.messages) - } - - // Transform tools to OpenAI format if provided - const tools = request.tools?.length - ? request.tools.map((tool) => ({ - type: 'function', - function: { - name: tool.id, - description: tool.description, - parameters: tool.parameters, - }, - })) - : undefined - - const payload: any = { - model: 'deepseek-chat', // Hardcode to deepseek-chat regardless of what's selected in the UI - messages: allMessages, - } - - // Add optional parameters - if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens - - // Add tools if provided - if (tools?.length) { - payload.tools = tools - payload.tool_choice = 'auto' - } - - // Make the initial API request - let currentResponse = await deepseek.chat.completions.create(payload) - let content = currentResponse.choices[0]?.message?.content || '' - - // Clean up the response content if it exists - if (content) { - // Remove any markdown code block markers - content = content.replace(/```json\n?|\n?```/g, '') - // Trim any whitespace - content = content.trim() - } - - let tokens = { - prompt: currentResponse.usage?.prompt_tokens || 0, - completion: currentResponse.usage?.completion_tokens || 0, - total: currentResponse.usage?.total_tokens || 0, - } - let toolCalls = [] - let toolResults = [] - let currentMessages = [...allMessages] - let iterationCount = 0 - const MAX_ITERATIONS = 10 // Prevent infinite loops + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() try { - while (iterationCount < MAX_ITERATIONS) { - // Check for tool calls - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls - if (!toolCallsInResponse || toolCallsInResponse.length === 0) { - break - } + // Deepseek uses the OpenAI SDK with a custom baseURL + const deepseek = new OpenAI({ + apiKey: request.apiKey, + baseURL: 'https://api.deepseek.com/v1', + dangerouslyAllowBrowser: true, + }) - // Process each tool call - for (const toolCall of toolCallsInResponse) { - try { - const toolName = toolCall.function.name - const toolArgs = JSON.parse(toolCall.function.arguments) + // Start with an empty array for all messages + const allMessages = [] - // Get the tool from the tools registry - const tool = request.tools?.find((t) => t.id === toolName) - if (!tool) continue + // Add system prompt if present + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } - // Execute the tool - const mergedArgs = { ...tool.params, ...toolArgs } - const result = await executeTool(toolName, mergedArgs) + // Add context if present + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } - if (!result.success) continue + // Add remaining messages + if (request.messages) { + allMessages.push(...request.messages) + } - toolResults.push(result.output) - toolCalls.push({ - name: toolName, - arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, - result: result.output, - }) + // Transform tools to OpenAI format if provided + const tools = request.tools?.length + ? request.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.id, + description: tool.description, + parameters: tool.parameters, + }, + })) + : undefined - // Add the tool call and result to messages - currentMessages.push({ - role: 'assistant', - content: null, - tool_calls: [ - { - id: toolCall.id, - type: 'function', - function: { - name: toolName, - arguments: toolCall.function.arguments, - }, - }, - ], - }) + const payload: any = { + model: 'deepseek-chat', // Hardcode to deepseek-chat regardless of what's selected in the UI + messages: allMessages, + } - currentMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: JSON.stringify(result.output), - }) - } catch (error) { - logger.error('Error processing tool call:', { error }) + // Add optional parameters + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + // Add tools if provided + if (tools?.length) { + payload.tools = tools + payload.tool_choice = 'auto' + } + + // Make the initial API request + const initialCallTime = Date.now() + let currentResponse = await deepseek.chat.completions.create(payload) + const firstResponseTime = Date.now() - initialCallTime + + let content = currentResponse.choices[0]?.message?.content || '' + + // Clean up the response content if it exists + if (content) { + // Remove any markdown code block markers + content = content.replace(/```json\n?|\n?```/g, '') + // Trim any whitespace + content = content.trim() + } + + let tokens = { + prompt: currentResponse.usage?.prompt_tokens || 0, + completion: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + let toolCalls = [] + let toolResults = [] + let currentMessages = [...allMessages] + let iterationCount = 0 + const MAX_ITERATIONS = 10 // Prevent infinite loops + + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 + + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + + try { + while (iterationCount < MAX_ITERATIONS) { + // Check for tool calls + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break } + + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + + // Process each tool call + for (const toolCall of toolCallsInResponse) { + try { + const toolName = toolCall.function.name + const toolArgs = JSON.parse(toolCall.function.arguments) + + // Get the tool from the tools registry + const tool = request.tools?.find((t) => t.id === toolName) + if (!tool) continue + + // Execute the tool + const toolCallStartTime = Date.now() + const mergedArgs = { ...tool.params, ...toolArgs } + const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime + + if (!result.success) continue + + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + + toolResults.push(result.output) + toolCalls.push({ + name: toolName, + arguments: toolArgs, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, + result: result.output, + }) + + // Add the tool call and result to messages + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: [ + { + id: toolCall.id, + type: 'function', + function: { + name: toolName, + arguments: toolCall.function.arguments, + }, + }, + ], + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(result.output), + }) + } catch (error) { + logger.error('Error processing tool call:', { error }) + } + } + + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + // Make the next request with updated messages + const nextPayload = { + ...payload, + messages: currentMessages, + } + + // Time the next model call + const nextModelStartTime = Date.now() + + // Make the next request + currentResponse = await deepseek.chat.completions.create(nextPayload) + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + + // Update content if we have a text response + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + // Clean up the response content + content = content.replace(/```json\n?|\n?```/g, '') + content = content.trim() + } + + // Update token counts + if (currentResponse.usage) { + tokens.prompt += currentResponse.usage.prompt_tokens || 0 + tokens.completion += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ } + } catch (error) { + logger.error('Error in Deepseek request:', { error }) + } - // Make the next request with updated messages - const nextPayload = { - ...payload, - messages: currentMessages, - } + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - // Make the next request - currentResponse = await deepseek.chat.completions.create(nextPayload) - - // Update content if we have a text response - if (currentResponse.choices[0]?.message?.content) { - content = currentResponse.choices[0].message.content - // Clean up the response content - content = content.replace(/```json\n?|\n?```/g, '') - content = content.trim() - } - - // Update token counts - if (currentResponse.usage) { - tokens.prompt += currentResponse.usage.prompt_tokens || 0 - tokens.completion += currentResponse.usage.completion_tokens || 0 - tokens.total += currentResponse.usage.total_tokens || 0 - } - - iterationCount++ + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, } } catch (error) { - logger.error('Error in Deepseek request:', { error }) - throw error - } + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - return { - content, - model: request.model, - tokens, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - toolResults: toolResults.length > 0 ? toolResults : undefined, + logger.error('Error in Deepseek request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/sim/app/providers/google/index.ts b/sim/app/providers/google/index.ts index 864e92ef05..b89b5f4d4b 100644 --- a/sim/app/providers/google/index.ts +++ b/sim/app/providers/google/index.ts @@ -1,7 +1,7 @@ import OpenAI from 'openai' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('Google Provider') @@ -18,182 +18,275 @@ export const googleProvider: ProviderConfig = { throw new Error('API key is required for Google Gemini') } - const openai = new OpenAI({ - apiKey: request.apiKey, - baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai/', - dangerouslyAllowBrowser: true, - }) - - // Start with an empty array for all messages - const allMessages = [] - - // Add system prompt if present - if (request.systemPrompt) { - allMessages.push({ - role: 'system', - content: request.systemPrompt, - }) - } - - // Add context if present - if (request.context) { - allMessages.push({ - role: 'user', - content: request.context, - }) - } - - // Add remaining messages - if (request.messages) { - allMessages.push(...request.messages) - } - - // Transform tools to OpenAI format if provided - const tools = request.tools?.length - ? request.tools.map((tool) => ({ - type: 'function', - function: { - name: tool.id, - description: tool.description, - parameters: tool.parameters, - }, - })) - : undefined - - // Build the request payload - const payload: any = { - model: request.model || 'gemini-2.0-flash', - messages: allMessages, - } - - // Add optional parameters - if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens - - // Add response format for structured output if specified - if (request.responseFormat) { - payload.response_format = { - type: 'json_schema', - schema: request.responseFormat.schema || request.responseFormat, - } - } - - // Add tools if provided - if (tools?.length) { - payload.tools = tools - payload.tool_choice = 'auto' - } - - // Make the initial API request - let currentResponse = await openai.chat.completions.create(payload) - let content = currentResponse.choices[0]?.message?.content || '' - let tokens = { - prompt: currentResponse.usage?.prompt_tokens || 0, - completion: currentResponse.usage?.completion_tokens || 0, - total: currentResponse.usage?.total_tokens || 0, - } - let toolCalls = [] - let toolResults = [] - let currentMessages = [...allMessages] - let iterationCount = 0 - const MAX_ITERATIONS = 10 // Prevent infinite loops + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() try { - while (iterationCount < MAX_ITERATIONS) { - // Check for tool calls - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls - if (!toolCallsInResponse || toolCallsInResponse.length === 0) { - break + const openai = new OpenAI({ + apiKey: request.apiKey, + baseURL: 'https://generativelanguage.googleapis.com/v1beta/openai/', + dangerouslyAllowBrowser: true, + }) + + // Start with an empty array for all messages + const allMessages = [] + + // Add system prompt if present + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } + + // Add context if present + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } + + // Add remaining messages + if (request.messages) { + allMessages.push(...request.messages) + } + + // Transform tools to OpenAI format if provided + const tools = request.tools?.length + ? request.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.id, + description: tool.description, + parameters: tool.parameters, + }, + })) + : undefined + + // Build the request payload + const payload: any = { + model: request.model || 'gemini-2.0-flash', + messages: allMessages, + } + + // Add optional parameters + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + // Add response format for structured output if specified + if (request.responseFormat) { + payload.response_format = { + type: 'json_schema', + schema: request.responseFormat.schema || request.responseFormat, } + } - // Process each tool call - for (const toolCall of toolCallsInResponse) { - try { - const toolName = toolCall.function.name - const toolArgs = JSON.parse(toolCall.function.arguments) + // Add tools if provided + if (tools?.length) { + payload.tools = tools + payload.tool_choice = 'auto' + } - // Get the tool from the tools registry - const tool = request.tools?.find((t) => t.id === toolName) - if (!tool) continue + // Make the initial API request + const initialCallTime = Date.now() + let currentResponse = await openai.chat.completions.create(payload) + const firstResponseTime = Date.now() - initialCallTime - // Execute the tool - const mergedArgs = { ...tool.params, ...toolArgs } - const result = await executeTool(toolName, mergedArgs) + let content = currentResponse.choices[0]?.message?.content || '' + let tokens = { + prompt: currentResponse.usage?.prompt_tokens || 0, + completion: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + let toolCalls = [] + let toolResults = [] + let currentMessages = [...allMessages] + let iterationCount = 0 + const MAX_ITERATIONS = 10 // Prevent infinite loops - if (!result.success) continue + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 - toolResults.push(result.output) - toolCalls.push({ - name: toolName, - arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, - result: result.output, - }) + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] - // Add the tool call and result to messages - currentMessages.push({ - role: 'assistant', - content: null, - tool_calls: [ - { - id: toolCall.id, - type: 'function', - function: { - name: toolName, - arguments: toolCall.function.arguments, - }, - }, - ], - }) - - const toolResultContent = JSON.stringify(result.output) - - currentMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: toolResultContent, - }) - } catch (error) { - logger.error('Error processing tool call:', { error }) + try { + while (iterationCount < MAX_ITERATIONS) { + // Check for tool calls + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break } + + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + + // Process each tool call + for (const toolCall of toolCallsInResponse) { + try { + const toolName = toolCall.function.name + const toolArgs = JSON.parse(toolCall.function.arguments) + + // Get the tool from the tools registry + const tool = request.tools?.find((t) => t.id === toolName) + if (!tool) continue + + // Execute the tool + const toolCallStartTime = Date.now() + const mergedArgs = { ...tool.params, ...toolArgs } + const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime + + if (!result.success) continue + + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + + toolResults.push(result.output) + toolCalls.push({ + name: toolName, + arguments: toolArgs, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, + result: result.output, + }) + + // Add the tool call and result to messages + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: [ + { + id: toolCall.id, + type: 'function', + function: { + name: toolName, + arguments: toolCall.function.arguments, + }, + }, + ], + }) + + const toolResultContent = JSON.stringify(result.output) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: toolResultContent, + }) + } catch (error) { + logger.error('Error processing tool call:', { error }) + } + } + + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + // Make the next request with updated messages + const nextPayload = { + ...payload, + messages: currentMessages, + } + + // Time the next model call + const nextModelStartTime = Date.now() + + // Make the next request + currentResponse = await openai.chat.completions.create(nextPayload) + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + + // Update content if we have a text response + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + // Update token counts + if (currentResponse.usage) { + tokens.prompt += currentResponse.usage.prompt_tokens || 0 + tokens.completion += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ } + } catch (error) { + logger.error('Error in Google Gemini request:', { error }) + } - // Make the next request with updated messages - const nextPayload = { - ...payload, - messages: currentMessages, - } + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - // Make the next request - currentResponse = await openai.chat.completions.create(nextPayload) - - // Update content if we have a text response - if (currentResponse.choices[0]?.message?.content) { - content = currentResponse.choices[0].message.content - } - - // Update token counts - if (currentResponse.usage) { - tokens.prompt += currentResponse.usage.prompt_tokens || 0 - tokens.completion += currentResponse.usage.completion_tokens || 0 - tokens.total += currentResponse.usage.total_tokens || 0 - } - - iterationCount++ + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, } } catch (error) { - logger.error('Error in Google Gemini request:', { error }) - throw error - } + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - return { - content, - model: request.model, - tokens, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - toolResults: toolResults.length > 0 ? toolResults : undefined, + logger.error('Error in Google Gemini request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/sim/app/providers/groq/index.ts b/sim/app/providers/groq/index.ts index 3a31d2e3a2..a2395ea12a 100644 --- a/sim/app/providers/groq/index.ts +++ b/sim/app/providers/groq/index.ts @@ -1,7 +1,7 @@ import { Groq } from 'groq-sdk' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('Groq Provider') @@ -18,179 +18,272 @@ export const groqProvider: ProviderConfig = { throw new Error('API key is required for Groq') } - const groq = new Groq({ - apiKey: request.apiKey, - dangerouslyAllowBrowser: true, - }) - - // Start with an empty array for all messages - const allMessages = [] - - // Add system prompt if present - if (request.systemPrompt) { - allMessages.push({ - role: 'system', - content: request.systemPrompt, - }) - } - - // Add context if present - if (request.context) { - allMessages.push({ - role: 'user', - content: request.context, - }) - } - - // Add remaining messages - if (request.messages) { - allMessages.push(...request.messages) - } - - // Transform tools to function format if provided - const tools = request.tools?.length - ? request.tools.map((tool) => ({ - type: 'function', - function: { - name: tool.id, - description: tool.description, - parameters: tool.parameters, - }, - })) - : undefined - - // Build the request payload - const payload: any = { - model: (request.model || 'groq/llama-3.3-70b-specdec').replace('groq/', ''), - messages: allMessages, - } - - // Add optional parameters - if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens - - // Add response format for structured output if specified - if (request.responseFormat) { - payload.response_format = { - type: 'json_schema', - schema: request.responseFormat.schema || request.responseFormat, - } - } - - // Add tools if provided - if (tools?.length) { - payload.tools = tools - payload.tool_choice = 'auto' - } - - // Make the initial API request - let currentResponse = await groq.chat.completions.create(payload) - let content = currentResponse.choices[0]?.message?.content || '' - let tokens = { - prompt: currentResponse.usage?.prompt_tokens || 0, - completion: currentResponse.usage?.completion_tokens || 0, - total: currentResponse.usage?.total_tokens || 0, - } - let toolCalls = [] - let toolResults = [] - let currentMessages = [...allMessages] - let iterationCount = 0 - const MAX_ITERATIONS = 10 // Prevent infinite loops + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() try { - while (iterationCount < MAX_ITERATIONS) { - // Check for tool calls - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls - if (!toolCallsInResponse || toolCallsInResponse.length === 0) { - break + const groq = new Groq({ + apiKey: request.apiKey, + dangerouslyAllowBrowser: true, + }) + + // Start with an empty array for all messages + const allMessages = [] + + // Add system prompt if present + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } + + // Add context if present + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } + + // Add remaining messages + if (request.messages) { + allMessages.push(...request.messages) + } + + // Transform tools to function format if provided + const tools = request.tools?.length + ? request.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.id, + description: tool.description, + parameters: tool.parameters, + }, + })) + : undefined + + // Build the request payload + const payload: any = { + model: (request.model || 'groq/llama-3.3-70b-specdec').replace('groq/', ''), + messages: allMessages, + } + + // Add optional parameters + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + // Add response format for structured output if specified + if (request.responseFormat) { + payload.response_format = { + type: 'json_schema', + schema: request.responseFormat.schema || request.responseFormat, } + } - // Process each tool call - for (const toolCall of toolCallsInResponse) { - try { - const toolName = toolCall.function.name - const toolArgs = JSON.parse(toolCall.function.arguments) + // Add tools if provided + if (tools?.length) { + payload.tools = tools + payload.tool_choice = 'auto' + } - // Get the tool from the tools registry - const tool = request.tools?.find((t) => t.id === toolName) - if (!tool) continue + // Make the initial API request + const initialCallTime = Date.now() + let currentResponse = await groq.chat.completions.create(payload) + const firstResponseTime = Date.now() - initialCallTime - // Execute the tool - const mergedArgs = { ...tool.params, ...toolArgs } - const result = await executeTool(toolName, mergedArgs) + let content = currentResponse.choices[0]?.message?.content || '' + let tokens = { + prompt: currentResponse.usage?.prompt_tokens || 0, + completion: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + let toolCalls = [] + let toolResults = [] + let currentMessages = [...allMessages] + let iterationCount = 0 + const MAX_ITERATIONS = 10 // Prevent infinite loops - if (!result.success) continue + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 - toolResults.push(result.output) - toolCalls.push({ - name: toolName, - arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, - result: result.output, - }) + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] - // Add the tool call and result to messages - currentMessages.push({ - role: 'assistant', - content: null, - tool_calls: [ - { - id: toolCall.id, - type: 'function', - function: { - name: toolName, - arguments: toolCall.function.arguments, - }, - }, - ], - }) - - currentMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: JSON.stringify(result.output), - }) - } catch (error) { - logger.error('Error processing tool call:', { error }) + try { + while (iterationCount < MAX_ITERATIONS) { + // Check for tool calls + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break } + + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + + // Process each tool call + for (const toolCall of toolCallsInResponse) { + try { + const toolName = toolCall.function.name + const toolArgs = JSON.parse(toolCall.function.arguments) + + // Get the tool from the tools registry + const tool = request.tools?.find((t) => t.id === toolName) + if (!tool) continue + + // Execute the tool + const toolCallStartTime = Date.now() + const mergedArgs = { ...tool.params, ...toolArgs } + const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime + + if (!result.success) continue + + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + + toolResults.push(result.output) + toolCalls.push({ + name: toolName, + arguments: toolArgs, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, + result: result.output, + }) + + // Add the tool call and result to messages + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: [ + { + id: toolCall.id, + type: 'function', + function: { + name: toolName, + arguments: toolCall.function.arguments, + }, + }, + ], + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(result.output), + }) + } catch (error) { + logger.error('Error processing tool call:', { error }) + } + } + + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + // Make the next request with updated messages + const nextPayload = { + ...payload, + messages: currentMessages, + } + + // Time the next model call + const nextModelStartTime = Date.now() + + // Make the next request + currentResponse = await groq.chat.completions.create(nextPayload) + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + + // Update content if we have a text response + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + // Update token counts + if (currentResponse.usage) { + tokens.prompt += currentResponse.usage.prompt_tokens || 0 + tokens.completion += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ } + } catch (error) { + logger.error('Error in Groq request:', { error }) + } - // Make the next request with updated messages - const nextPayload = { - ...payload, - messages: currentMessages, - } + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - // Make the next request - currentResponse = await groq.chat.completions.create(nextPayload) - - // Update content if we have a text response - if (currentResponse.choices[0]?.message?.content) { - content = currentResponse.choices[0].message.content - } - - // Update token counts - if (currentResponse.usage) { - tokens.prompt += currentResponse.usage.prompt_tokens || 0 - tokens.completion += currentResponse.usage.completion_tokens || 0 - tokens.total += currentResponse.usage.total_tokens || 0 - } - - iterationCount++ + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, } } catch (error) { - logger.error('Error in Groq request:', { error }) - throw error - } + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - return { - content, - model: request.model, - tokens, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - toolResults: toolResults.length > 0 ? toolResults : undefined, + logger.error('Error in Groq request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/sim/app/providers/openai/index.ts b/sim/app/providers/openai/index.ts index 7cb62b301b..ab3fe132fa 100644 --- a/sim/app/providers/openai/index.ts +++ b/sim/app/providers/openai/index.ts @@ -1,7 +1,7 @@ import OpenAI from 'openai' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('OpenAI Provider') @@ -106,21 +106,43 @@ export const openaiProvider: ProviderConfig = { logger.info(`Configured ${tools.length} tools for OpenAI request`) } - // Make the initial API request - let currentResponse = await openai.chat.completions.create(payload) - let content = currentResponse.choices[0]?.message?.content || '' - let tokens = { - prompt: currentResponse.usage?.prompt_tokens || 0, - completion: currentResponse.usage?.completion_tokens || 0, - total: currentResponse.usage?.total_tokens || 0, - } - let toolCalls = [] - let toolResults = [] - let currentMessages = [...allMessages] - let iterationCount = 0 - const MAX_ITERATIONS = 10 // Prevent infinite loops + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() try { + // Make the initial API request + const initialCallTime = Date.now() + let currentResponse = await openai.chat.completions.create(payload) + const firstResponseTime = Date.now() - initialCallTime + + let content = currentResponse.choices[0]?.message?.content || '' + let tokens = { + prompt: currentResponse.usage?.prompt_tokens || 0, + completion: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + let toolCalls = [] + let toolResults = [] + let currentMessages = [...allMessages] + let iterationCount = 0 + const MAX_ITERATIONS = 10 // Prevent infinite loops + + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 + + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + while (iterationCount < MAX_ITERATIONS) { // Check for tool calls const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls @@ -132,6 +154,9 @@ export const openaiProvider: ProviderConfig = { `Processing ${toolCallsInResponse.length} tool calls (iteration ${iterationCount + 1}/${MAX_ITERATIONS})` ) + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + // Process each tool call for (const toolCall of toolCallsInResponse) { try { @@ -143,18 +168,30 @@ export const openaiProvider: ProviderConfig = { if (!tool) continue // Execute the tool + const toolCallStartTime = Date.now() const mergedArgs = { ...tool.params, ...toolArgs } const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime if (!result.success) continue + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + toolResults.push(result.output) toolCalls.push({ name: toolName, arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, result: result.output, }) @@ -187,15 +224,37 @@ export const openaiProvider: ProviderConfig = { } } + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + // Make the next request with updated messages const nextPayload = { ...payload, messages: currentMessages, } + // Time the next model call + const nextModelStartTime = Date.now() + // Make the next request currentResponse = await openai.chat.completions.create(nextPayload) + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + // Update content if we have a text response if (currentResponse.choices[0]?.message?.content) { content = currentResponse.choices[0].message.content @@ -210,17 +269,47 @@ export const openaiProvider: ProviderConfig = { iterationCount++ } - } catch (error) { - logger.error('Error in OpenAI request:', { error }) - throw error - } - return { - content, - model: request.model, - tokens, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - toolResults: toolResults.length > 0 ? toolResults : undefined, + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, + } + } catch (error) { + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + logger.error('Error in OpenAI request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/sim/app/providers/types.ts b/sim/app/providers/types.ts index fd05c8553f..787997ecbe 100644 --- a/sim/app/providers/types.ts +++ b/sim/app/providers/types.ts @@ -39,6 +39,14 @@ export interface FunctionCallResponse { input?: Record } +export interface TimeSegment { + type: 'model' | 'tool' + name: string + startTime: number + endTime: number + duration: number +} + export interface ProviderResponse { content: string model: string @@ -49,6 +57,16 @@ export interface ProviderResponse { } toolCalls?: FunctionCallResponse[] toolResults?: any[] + timing?: { + startTime: string // ISO timestamp when provider execution started + endTime: string // ISO timestamp when provider execution completed + duration: number // Total duration in milliseconds + modelTime?: number // Time spent in model generation (excluding tool calls) + toolsTime?: number // Time spent in tool calls + firstResponseTime?: number // Time to first token/response + iterations?: number // Number of model calls for tool use + timeSegments?: TimeSegment[] // Detailed timeline of all operations + } } export interface ProviderToolConfig { diff --git a/sim/app/providers/xai/index.ts b/sim/app/providers/xai/index.ts index c65198bc6a..860b4e6a57 100644 --- a/sim/app/providers/xai/index.ts +++ b/sim/app/providers/xai/index.ts @@ -1,7 +1,7 @@ import OpenAI from 'openai' import { createLogger } from '@/lib/logs/console-logger' import { executeTool } from '@/tools' -import { ProviderConfig, ProviderRequest, ProviderResponse } from '../types' +import { ProviderConfig, ProviderRequest, ProviderResponse, TimeSegment } from '../types' const logger = createLogger('XAI Provider') @@ -18,174 +18,268 @@ export const xAIProvider: ProviderConfig = { throw new Error('API key is required for xAI') } - const xai = new OpenAI({ - apiKey: request.apiKey, - baseURL: 'https://api.x.ai/v1', - dangerouslyAllowBrowser: true, - }) - - const allMessages = [] - - if (request.systemPrompt) { - allMessages.push({ - role: 'system', - content: request.systemPrompt, - }) - } - - if (request.context) { - allMessages.push({ - role: 'user', - content: request.context, - }) - } - - if (request.messages) { - allMessages.push(...request.messages) - } - - const tools = request.tools?.length - ? request.tools.map((tool) => ({ - type: 'function', - function: { - name: tool.id, - description: tool.description, - parameters: tool.parameters, - }, - })) - : undefined - - const payload: any = { - model: request.model || 'grok-2-latest', - messages: allMessages, - } - - if (request.temperature !== undefined) payload.temperature = request.temperature - if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens - - if (request.responseFormat) { - payload.response_format = { - type: 'json_schema', - json_schema: { - name: request.responseFormat.name || 'structured_response', - schema: request.responseFormat.schema || request.responseFormat, - strict: request.responseFormat.strict !== false, - }, - } - - if (allMessages.length > 0 && allMessages[0].role === 'system') { - allMessages[0].content = `${allMessages[0].content}\n\nYou MUST respond with a valid JSON object. DO NOT include any other text, explanations, or markdown formatting in your response - ONLY the JSON object.` - } else { - allMessages.unshift({ - role: 'system', - content: `You MUST respond with a valid JSON object. DO NOT include any other text, explanations, or markdown formatting in your response - ONLY the JSON object.`, - }) - } - } - - if (tools?.length) { - payload.tools = tools - payload.tool_choice = 'auto' - } - - let currentResponse = await xai.chat.completions.create(payload) - let content = currentResponse.choices[0]?.message?.content || '' - let tokens = { - prompt: currentResponse.usage?.prompt_tokens || 0, - completion: currentResponse.usage?.completion_tokens || 0, - total: currentResponse.usage?.total_tokens || 0, - } - let toolCalls = [] - let toolResults = [] - let currentMessages = [...allMessages] - let iterationCount = 0 - const MAX_ITERATIONS = 10 + // Start execution timer for the entire provider execution + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() try { - while (iterationCount < MAX_ITERATIONS) { - const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls - if (!toolCallsInResponse || toolCallsInResponse.length === 0) { - break + const xai = new OpenAI({ + apiKey: request.apiKey, + baseURL: 'https://api.x.ai/v1', + dangerouslyAllowBrowser: true, + }) + + const allMessages = [] + + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } + + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } + + if (request.messages) { + allMessages.push(...request.messages) + } + + const tools = request.tools?.length + ? request.tools.map((tool) => ({ + type: 'function', + function: { + name: tool.id, + description: tool.description, + parameters: tool.parameters, + }, + })) + : undefined + + const payload: any = { + model: request.model || 'grok-2-latest', + messages: allMessages, + } + + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + if (request.responseFormat) { + payload.response_format = { + type: 'json_schema', + json_schema: { + name: request.responseFormat.name || 'structured_response', + schema: request.responseFormat.schema || request.responseFormat, + strict: request.responseFormat.strict !== false, + }, } - for (const toolCall of toolCallsInResponse) { - try { - const toolName = toolCall.function.name - const toolArgs = JSON.parse(toolCall.function.arguments) + if (allMessages.length > 0 && allMessages[0].role === 'system') { + allMessages[0].content = `${allMessages[0].content}\n\nYou MUST respond with a valid JSON object. DO NOT include any other text, explanations, or markdown formatting in your response - ONLY the JSON object.` + } else { + allMessages.unshift({ + role: 'system', + content: `You MUST respond with a valid JSON object. DO NOT include any other text, explanations, or markdown formatting in your response - ONLY the JSON object.`, + }) + } + } - const tool = request.tools?.find((t) => t.id === toolName) - if (!tool) continue + if (tools?.length) { + payload.tools = tools + payload.tool_choice = 'auto' + } - const mergedArgs = { ...tool.params, ...toolArgs } - const result = await executeTool(toolName, mergedArgs) + // Make the initial API request + const initialCallTime = Date.now() + let currentResponse = await xai.chat.completions.create(payload) + const firstResponseTime = Date.now() - initialCallTime - if (!result.success) continue + let content = currentResponse.choices[0]?.message?.content || '' + let tokens = { + prompt: currentResponse.usage?.prompt_tokens || 0, + completion: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + let toolCalls = [] + let toolResults = [] + let currentMessages = [...allMessages] + let iterationCount = 0 + const MAX_ITERATIONS = 10 - toolResults.push(result.output) - toolCalls.push({ - name: toolName, - arguments: toolArgs, - startTime: result.timing?.startTime, - endTime: result.timing?.endTime, - duration: result.timing?.duration, - result: result.output, - }) + // Track time spent in model vs tools + let modelTime = firstResponseTime + let toolsTime = 0 - currentMessages.push({ - role: 'assistant', - content: null, - tool_calls: [ - { - id: toolCall.id, - type: 'function', - function: { - name: toolName, - arguments: toolCall.function.arguments, - }, - }, - ], - }) + // Track each model and tool call segment with timestamps + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: 'Initial response', + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] - currentMessages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: JSON.stringify(result.output), - }) - } catch (error) { - logger.error('Error processing tool call:', { error }) + try { + while (iterationCount < MAX_ITERATIONS) { + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break } + + // Track time for tool calls in this batch + const toolsStartTime = Date.now() + + for (const toolCall of toolCallsInResponse) { + try { + const toolName = toolCall.function.name + const toolArgs = JSON.parse(toolCall.function.arguments) + + const tool = request.tools?.find((t) => t.id === toolName) + if (!tool) continue + + const toolCallStartTime = Date.now() + const mergedArgs = { ...tool.params, ...toolArgs } + const result = await executeTool(toolName, mergedArgs) + const toolCallEndTime = Date.now() + const toolCallDuration = toolCallEndTime - toolCallStartTime + + if (!result.success) continue + + // Add to time segments + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallDuration, + }) + + toolResults.push(result.output) + toolCalls.push({ + name: toolName, + arguments: toolArgs, + startTime: new Date(toolCallStartTime).toISOString(), + endTime: new Date(toolCallEndTime).toISOString(), + duration: toolCallDuration, + result: result.output, + }) + + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: [ + { + id: toolCall.id, + type: 'function', + function: { + name: toolName, + arguments: toolCall.function.arguments, + }, + }, + ], + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(result.output), + }) + } catch (error) { + logger.error('Error processing tool call:', { error }) + } + } + + // Calculate tool call time for this iteration + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + const nextPayload = { + ...payload, + messages: currentMessages, + } + + // Time the next model call + const nextModelStartTime = Date.now() + + currentResponse = await xai.chat.completions.create(nextPayload) + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + // Add to time segments + timeSegments.push({ + type: 'model', + name: `Model response (iteration ${iterationCount + 1})`, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + // Add to model time + modelTime += thisModelTime + + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + if (currentResponse.usage) { + tokens.prompt += currentResponse.usage.prompt_tokens || 0 + tokens.completion += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ } + } catch (error) { + logger.error('Error in xAI request:', { error }) + } - const nextPayload = { - ...payload, - messages: currentMessages, - } + // Calculate overall timing + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - currentResponse = await xai.chat.completions.create(nextPayload) - - if (currentResponse.choices[0]?.message?.content) { - content = currentResponse.choices[0].message.content - } - - if (currentResponse.usage) { - tokens.prompt += currentResponse.usage.prompt_tokens || 0 - tokens.completion += currentResponse.usage.completion_tokens || 0 - tokens.total += currentResponse.usage.total_tokens || 0 - } - - iterationCount++ + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, } } catch (error) { - logger.error('Error in xAI request:', { error }) - throw error - } + // Include timing information even for errors + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime - return { - content, - model: request.model, - tokens, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - toolResults: toolResults.length > 0 ? toolResults : undefined, + logger.error('Error in xAI request:', { error, duration: totalDuration }) + + // Create a new error with timing information + const enhancedError = new Error(error instanceof Error ? error.message : String(error)) + // @ts-ignore - Adding timing property to the error + enhancedError.timing = { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + } + + throw enhancedError } }, } diff --git a/sim/app/w/[id]/hooks/use-workflow-execution.ts b/sim/app/w/[id]/hooks/use-workflow-execution.ts index 46d27b3dc4..c0ed0a7b22 100644 --- a/sim/app/w/[id]/hooks/use-workflow-execution.ts +++ b/sim/app/w/[id]/hooks/use-workflow-execution.ts @@ -6,15 +6,252 @@ import { useExecutionStore } from '@/stores/execution/store' import { useNotificationStore } from '@/stores/notifications/store' import { useEnvironmentStore } from '@/stores/settings/environment/store' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' -import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { mergeSubblockState } from '@/stores/workflows/utils' import { useWorkflowStore } from '@/stores/workflows/workflow/store' +import { TraceSpan } from '@/app/w/logs/stores/types' import { Executor } from '@/executor' import { ExecutionResult } from '@/executor/types' import { Serializer } from '@/serializer' const logger = createLogger('useWorkflowExecution') +// Helper function to build a tree of trace spans from execution logs +function buildTraceSpans(result: ExecutionResult): { + traceSpans: TraceSpan[] + totalDuration: number +} { + // If no logs, return empty spans + if (!result.logs || result.logs.length === 0) { + return { traceSpans: [], totalDuration: 0 } + } + + // Store all spans as a map for faster lookup + const spanMap = new Map() + + // First pass: Create spans for each block + result.logs.forEach((log) => { + // Skip logs that don't have block execution information + if (!log.blockId || !log.blockType) return + + // Create a unique ID for this span using blockId and timestamp + const spanId = `${log.blockId}-${new Date(log.startedAt).getTime()}` + + // Extract duration if available + const duration = log.durationMs || 0 + + // Create the span + const span: TraceSpan = { + id: spanId, + name: log.blockName || log.blockId, + type: log.blockType, + duration: duration, + startTime: log.startedAt, + endTime: log.endedAt, + status: log.error ? 'error' : 'success', + children: [], + } + + // Add provider timing data if it exists + if (log.output?.response?.providerTiming) { + const providerTiming = log.output.response.providerTiming + + // If we have time segments, use them to create a more detailed timeline + if (providerTiming.timeSegments && providerTiming.timeSegments.length > 0) { + const segmentStartTime = new Date(log.startedAt).getTime() + const children: TraceSpan[] = [] + + // Process segments in order + providerTiming.timeSegments.forEach( + ( + segment: { + type: string + name: string + startTime: number + endTime: number + duration: number + }, + index: number + ) => { + const relativeStart = segment.startTime - segmentStartTime + + // Enhance the segment name to include model information for model segments + let enhancedName = segment.name + if (segment.type === 'model') { + const modelName = log.output.response.model || '' + + if (segment.name === 'Initial response') { + enhancedName = `Initial response${modelName ? ` (${modelName})` : ''}` + } else if (segment.name.includes('iteration')) { + // Extract the iteration number + const iterationMatch = segment.name.match(/\(iteration (\d+)\)/) + const iterationNum = iterationMatch ? iterationMatch[1] : '' + + enhancedName = `Model response${iterationNum ? ` (iteration ${iterationNum})` : ''}${modelName ? ` (${modelName})` : ''}` + } + } + + const segmentSpan: TraceSpan = { + id: `${spanId}-segment-${index}`, + name: enhancedName, + // Make sure we handle model and tool types, and fallback to generic 'span' for anything else + type: segment.type === 'model' || segment.type === 'tool' ? segment.type : 'span', + duration: segment.duration, + startTime: new Date(segment.startTime).toISOString(), + endTime: new Date(segment.endTime).toISOString(), + status: 'success', + // Add relative timing display for segments after the first one + relativeStartMs: index === 0 ? undefined : relativeStart, + // For model segments, add token info if available + ...(segment.type === 'model' && { + tokens: index === 0 ? log.output.response.tokens?.completion : undefined, + }), + } + + children.push(segmentSpan) + } + ) + + // Add all segments as children + if (!span.children) span.children = [] + span.children.push(...children) + } + // If no segments but we have provider timing, create a provider span + else { + // Create a child span for the provider execution + const providerSpan: TraceSpan = { + id: `${spanId}-provider`, + name: log.output.response.model || 'AI Provider', + type: 'provider', + duration: providerTiming.duration || 0, + startTime: providerTiming.startTime || log.startedAt, + endTime: providerTiming.endTime || log.endedAt, + status: 'success', + tokens: log.output.response.tokens?.total, + } + + // If we have model time, create a child span for just the model processing + if (providerTiming.modelTime) { + const modelName = log.output.response.model || '' + const modelSpan: TraceSpan = { + id: `${spanId}-model`, + name: `Model Generation${modelName ? ` (${modelName})` : ''}`, + type: 'model', + duration: providerTiming.modelTime, + startTime: providerTiming.startTime, // Approximate + endTime: providerTiming.endTime, // Approximate + status: 'success', + tokens: log.output.response.tokens?.completion, + } + + if (!providerSpan.children) providerSpan.children = [] + providerSpan.children.push(modelSpan) + } + + if (!span.children) span.children = [] + span.children.push(providerSpan) + + // When using provider timing without segments, still add tool calls if they exist + if (log.output?.response?.toolCalls?.list) { + span.toolCalls = log.output.response.toolCalls.list.map((tc: any) => ({ + name: tc.name, + duration: tc.duration || 0, + startTime: tc.startTime || log.startedAt, + endTime: tc.endTime || log.endedAt, + status: tc.error ? 'error' : 'success', + input: tc.arguments || tc.input, + output: tc.result || tc.output, + error: tc.error, + })) + } + } + } else { + // When not using provider timing at all, add tool calls if they exist + if (log.output?.response?.toolCalls?.list) { + span.toolCalls = log.output.response.toolCalls.list.map((tc: any) => ({ + name: tc.name, + duration: tc.duration || 0, + startTime: tc.startTime || log.startedAt, + endTime: tc.endTime || log.endedAt, + status: tc.error ? 'error' : 'success', + input: tc.arguments || tc.input, + output: tc.result || tc.output, + error: tc.error, + })) + } + } + + // Store in map + spanMap.set(spanId, span) + }) + + // Second pass: Build the hierarchy + // We'll first need to sort logs chronologically + const sortedLogs = [...result.logs].sort((a, b) => { + const aTime = new Date(a.startedAt).getTime() + const bTime = new Date(b.startedAt).getTime() + return aTime - bTime + }) + + // Track parent spans using a stack + const spanStack: TraceSpan[] = [] + const rootSpans: TraceSpan[] = [] + + // Process logs to build the hierarchy + sortedLogs.forEach((log) => { + if (!log.blockId || !log.blockType) return + + const spanId = `${log.blockId}-${new Date(log.startedAt).getTime()}` + const span = spanMap.get(spanId) + if (!span) return + + // If we have a non-empty stack, check if this span should be a child + if (spanStack.length > 0) { + const potentialParent = spanStack[spanStack.length - 1] + const parentStartTime = new Date(potentialParent.startTime).getTime() + const parentEndTime = new Date(potentialParent.endTime).getTime() + const spanStartTime = new Date(span.startTime).getTime() + + // If this span starts after the parent starts and the parent is still on the stack, + // we'll assume it's a child span + if (spanStartTime >= parentStartTime && spanStartTime <= parentEndTime) { + if (!potentialParent.children) potentialParent.children = [] + potentialParent.children.push(span) + } else { + // This span doesn't belong to the current parent, pop from stack + while ( + spanStack.length > 0 && + new Date(spanStack[spanStack.length - 1].endTime).getTime() < spanStartTime + ) { + spanStack.pop() + } + + // Check if we still have a parent + if (spanStack.length > 0) { + const newParent = spanStack[spanStack.length - 1] + if (!newParent.children) newParent.children = [] + newParent.children.push(span) + } else { + // No parent, this is a root span + rootSpans.push(span) + } + } + } else { + // Empty stack, this is a root span + rootSpans.push(span) + } + + // Check if this span could be a parent to future spans + if (log.blockType === 'agent' || log.blockType === 'workflow') { + spanStack.push(span) + } + }) + + // Calculate total duration as the sum of root spans + const totalDuration = rootSpans.reduce((sum, span) => sum + span.duration, 0) + + return { traceSpans: rootSpans, totalDuration } +} + export function useWorkflowExecution() { const { blocks, edges, loops } = useWorkflowStore() const { activeWorkflowId } = useWorkflowRegistry() @@ -26,6 +263,16 @@ export function useWorkflowExecution() { const persistLogs = async (executionId: string, result: ExecutionResult) => { try { + // Build trace spans from execution logs + const { traceSpans, totalDuration } = buildTraceSpans(result) + + // Add trace spans to the execution result + const enrichedResult = { + ...result, + traceSpans, + totalDuration, + } + const response = await fetch(`/api/workflow/${activeWorkflowId}/log`, { method: 'POST', headers: { @@ -33,7 +280,7 @@ export function useWorkflowExecution() { }, body: JSON.stringify({ executionId, - result, + result: enrichedResult, }), }) diff --git a/sim/app/w/logs/components/sidebar/sidebar.tsx b/sim/app/w/logs/components/sidebar/sidebar.tsx index 41582f7b06..f2befcefaf 100644 --- a/sim/app/w/logs/components/sidebar/sidebar.tsx +++ b/sim/app/w/logs/components/sidebar/sidebar.tsx @@ -8,6 +8,7 @@ import { ScrollArea } from '@/components/ui/scroll-area' import { WorkflowLog } from '@/app/w/logs/stores/types' import { formatDate } from '@/app/w/logs/utils/format-date' import { ToolCallsDisplay } from '../tool-calls/tool-calls-display' +import { TraceSpansDisplay } from '../trace-spans/trace-spans-display' interface LogSidebarProps { log: WorkflowLog | null @@ -121,6 +122,22 @@ export function Sidebar({ log, isOpen, onClose }: LogSidebarProps) { return formatJsonContent(log.message) }, [log]) + // Determine if this is a workflow execution log + const isWorkflowExecutionLog = useMemo(() => { + if (!log) return false + // Check if message contains "workflow executed" or similar phrases + return ( + log.message.toLowerCase().includes('workflow executed') || + log.message.toLowerCase().includes('execution completed') || + (log.trigger === 'manual' && log.duration) + ) + }, [log]) + + // Helper to determine if we have trace spans to display + const hasTraceSpans = useMemo(() => { + return !!(log?.metadata?.traceSpans && log.metadata.traceSpans.length > 0) + }, [log]) + const handleMouseDown = (e: React.MouseEvent) => { setIsDragging(true) e.preventDefault() @@ -262,6 +279,17 @@ export function Sidebar({ log, isOpen, onClose }: LogSidebarProps) { )} + {/* Trace Spans (if available and this is a workflow execution log) */} + {isWorkflowExecutionLog && log.metadata?.traceSpans && ( +
+

Trace Spans

+ +
+ )} + {/* Tool Calls (if available) */} {log.metadata?.toolCalls && log.metadata.toolCalls.length > 0 && (
diff --git a/sim/app/w/logs/components/trace-spans/trace-spans-display.tsx b/sim/app/w/logs/components/trace-spans/trace-spans-display.tsx new file mode 100644 index 0000000000..e041c3d670 --- /dev/null +++ b/sim/app/w/logs/components/trace-spans/trace-spans-display.tsx @@ -0,0 +1,359 @@ +'use client' + +import { useState } from 'react' +import { + BarChart, + ChevronDown, + ChevronRight, + Code, + Cpu, + Database, + ExternalLink, +} from 'lucide-react' +import { + AgentIcon, + ApiIcon, + ChartBarIcon, + CodeIcon, + ConditionalIcon, + ConnectIcon, +} from '@/components/icons' +import { cn } from '@/lib/utils' +import { TraceSpan } from '../../stores/types' + +interface TraceSpansDisplayProps { + traceSpans?: TraceSpan[] + totalDuration?: number +} + +export function TraceSpansDisplay({ traceSpans, totalDuration = 0 }: TraceSpansDisplayProps) { + if (!traceSpans || traceSpans.length === 0) { + return
No trace data available
+ } + + // Format total duration for better readability + const formatTotalDuration = (ms: number) => { + if (ms < 1000) return `${ms}ms` + return `${(ms / 1000).toFixed(2)}s (${ms}ms)` + } + + // Find the earliest start time among all spans to be the workflow start time + const workflowStartTime = traceSpans.reduce((earliest, span) => { + const startTime = new Date(span.startTime).getTime() + return startTime < earliest ? startTime : earliest + }, Infinity) + + return ( +
+ {/* Show only the total duration in the header */} +
+ {formatTotalDuration(totalDuration)} +
+ +
+ {traceSpans.map((span, index) => ( + + ))} +
+
+ ) +} + +interface TraceSpanItemProps { + span: TraceSpan + depth: number + totalDuration: number + isLast: boolean + parentStartTime: number // Start time of the parent span for offset calculation + workflowStartTime: number // Start time of the entire workflow +} + +function TraceSpanItem({ + span, + depth, + totalDuration, + isLast, + parentStartTime, + workflowStartTime, +}: TraceSpanItemProps): JSX.Element { + const [expanded, setExpanded] = useState(false) // Collapsed by default + const hasChildren = span.children && span.children.length > 0 + const hasToolCalls = span.toolCalls && span.toolCalls.length > 0 + + // Calculate timing information + const spanStartTime = new Date(span.startTime).getTime() + const spanEndTime = new Date(span.endTime).getTime() + const duration = span.duration || spanEndTime - spanStartTime + const startOffset = spanStartTime - parentStartTime // Time from parent start to this span's start + + // Calculate the position relative to the workflow start time (for Gantt chart style) + const relativeStartPercent = + totalDuration > 0 ? ((spanStartTime - workflowStartTime) / totalDuration) * 100 : 0 + const durationPercent = totalDuration > 0 ? (duration / totalDuration) * 100 : 0 + + // Ensure values are within valid range + const safeStartPercent = Math.min(100, Math.max(0, relativeStartPercent)) + const safeWidthPercent = Math.max(2, Math.min(100 - safeStartPercent, durationPercent)) + + // For parent-relative timing display + const startOffsetPercentage = totalDuration > 0 ? (startOffset / totalDuration) * 100 : 0 + + // Get appropriate icon based on span type + const getSpanIcon = () => { + const type = span.type.toLowerCase() + + // Expand/collapse for spans with children + if (hasChildren || hasToolCalls) { + return expanded ? : + } + + // Block type specific icons + if (type === 'agent') { + return + } + + if (type === 'evaluator') { + return + } + + if (type === 'condition') { + return + } + + if (type === 'router') { + return + } + + if (type === 'model') { + return + } + + if (type === 'function') { + return + } + + if (type === 'tool') { + return + } + + if (type === 'api') { + return + } + + return + } + + // Format milliseconds as +XXms for relative timing + const formatRelativeTime = (ms: number) => { + if (ms === 0) return 'start' + return `+${ms}ms` + } + + // Get color based on span type + const getSpanColor = (type: string) => { + switch (type.toLowerCase()) { + case 'agent': + return '#7F2FFF' // Purple from AgentBlock + case 'provider': + return '#818cf8' // Indigo for provider + case 'model': + return '#10a37f' // Green from OpenAIBlock + case 'function': + return '#FF402F' // Orange-red from FunctionBlock + case 'tool': + return '#f97316' // Orange for tools + case 'router': + return '#2FA1FF' // Blue from RouterBlock + case 'condition': + return '#FF972F' // Orange from ConditionBlock + case 'evaluator': + return '#2FA1FF' // Blue from EvaluatorBlock + case 'api': + return '#2F55FF' // Blue from ApiBlock + default: + return '#6b7280' // Gray for others + } + } + + const spanColor = getSpanColor(span.type) + + // Format duration to be more readable + const formatDuration = (ms: number) => { + if (ms < 1000) return `${ms}ms` + return `${(ms / 1000).toFixed(2)}s` + } + + // Extract model name from span name using a more general pattern + const extractModelName = (spanName: string) => { + // Try to match model name in parentheses + const modelMatch = spanName.match(/\(([\w\.-]+)\)/i) + return modelMatch ? modelMatch[1] : '' + } + + // Format span name for display + const formatSpanName = (span: TraceSpan) => { + if (span.type === 'model') { + const modelName = extractModelName(span.name) + + if (span.name.includes('Initial response')) { + return ( + <> + Initial response{' '} + {modelName && ({modelName})} + + ) + } else if (span.name.includes('(iteration')) { + const iterMatch = span.name.match(/\(iteration (\d+)\)/) + const iterNum = iterMatch ? iterMatch[1] : '' + return ( + <> + Model response{' '} + {iterNum && (iteration {iterNum})}{' '} + {modelName && ({modelName})} + + ) + } else if (span.name.includes('Model Generation')) { + return ( + <> + Model Generation{' '} + {modelName && ({modelName})} + + ) + } + } + + return span.name + } + + return ( +
+ {/* Span header */} +
setExpanded(!expanded)} + style={{ paddingLeft: `${depth * 16 + 8}px` }} + > +
{getSpanIcon()}
+ +
+
+
+ + {formatSpanName(span)} + + {depth > 0 && ( + + {span.relativeStartMs !== undefined + ? `+${span.relativeStartMs}ms` + : formatRelativeTime(startOffset)} + + )} + {span.tokens && ( + + {span.tokens} tokens + + )} +
+ {formatDuration(duration)} +
+ +
+ {/* Timeline visualization - all spans are positioned relative to workflow start */} +
+
+
+ + {/* Duration text - always show in ms */} + + {`${duration}ms`} + +
+
+
+ + {/* Children and tool calls */} + {expanded && ( +
+ {/* Render child spans */} + {hasChildren && ( +
+ {span.children!.map((childSpan, index) => ( + + ))} +
+ )} + + {/* Render tool calls as spans */} + {hasToolCalls && ( +
+ {span.toolCalls!.map((toolCall, index) => { + // Create a pseudo-span for each tool call + const toolStartTime = toolCall.startTime + ? new Date(toolCall.startTime).getTime() + : spanStartTime + const toolEndTime = toolCall.endTime + ? new Date(toolCall.endTime).getTime() + : toolStartTime + (toolCall.duration || 0) + + const toolSpan: TraceSpan = { + id: `${span.id}-tool-${index}`, + name: toolCall.name, + type: 'tool', + duration: toolCall.duration || toolEndTime - toolStartTime, + startTime: new Date(toolStartTime).toISOString(), + endTime: new Date(toolEndTime).toISOString(), + status: toolCall.error ? 'error' : 'success', + } + + return ( + + ) + })} +
+ )} +
+ )} +
+ ) +} diff --git a/sim/app/w/logs/stores/types.ts b/sim/app/w/logs/stores/types.ts index 3e2d3caddb..5657eb3c3b 100644 --- a/sim/app/w/logs/stores/types.ts +++ b/sim/app/w/logs/stores/types.ts @@ -22,6 +22,20 @@ export interface ToolCallMetadata { toolCalls?: ToolCall[] } +export interface TraceSpan { + id: string + name: string + type: string + duration: number // in milliseconds + startTime: string + endTime: string + children?: TraceSpan[] + toolCalls?: ToolCall[] + status?: 'success' | 'error' + tokens?: number + relativeStartMs?: number // Time in ms from the start of the parent span +} + export interface WorkflowLog { id: string workflowId: string @@ -32,7 +46,10 @@ export interface WorkflowLog { trigger: string | null createdAt: string workflow?: WorkflowData | null - metadata?: ToolCallMetadata | Record // Add metadata for tool calls + metadata?: ToolCallMetadata & { + traceSpans?: TraceSpan[] + totalDuration?: number + } } export interface LogsResponse {