diff --git a/blocks/blocks/agent.ts b/blocks/blocks/agent.ts index cb7959fef2..e2f624de7b 100644 --- a/blocks/blocks/agent.ts +++ b/blocks/blocks/agent.ts @@ -60,7 +60,41 @@ export const AgentBlock: BlockConfig = { context: { type: 'string', required: false }, model: { type: 'string', required: true }, apiKey: { type: 'string', required: true }, - responseFormat: { type: 'json', required: false }, + responseFormat: { + type: 'json', + required: false, + description: + 'Define the expected response format. If not provided, returns plain text content.', + schema: { + type: 'object', + properties: { + fields: { + type: 'array', + items: { + type: 'object', + properties: { + name: { + type: 'string', + minLength: 1, + }, + type: { + type: 'string', + enum: ['string', 'number', 'boolean', 'array', 'object'], + }, + description: { + type: 'string', + }, + }, + required: ['name', 'type'], + additionalProperties: false, + }, + minItems: 1, + }, + }, + required: ['fields'], + additionalProperties: false, + }, + }, temperature: { type: 'number', required: false }, tools: { type: 'json', required: false }, }, @@ -124,6 +158,20 @@ export const AgentBlock: BlockConfig = { title: 'Response Format', type: 'code', layout: 'full', + placeholder: `{ + "fields": [ + { + "name": "sentiment", + "type": "string", + "description": "The sentiment of the text (positive, negative, neutral)" + }, + { + "name": "score", + "type": "number", + "description": "Confidence score between 0 and 1" + } + ] +}`, }, ], }, diff --git a/blocks/types.ts b/blocks/types.ts index e3fb932d9f..7473067487 100644 --- a/blocks/types.ts +++ b/blocks/types.ts @@ -48,6 +48,19 @@ export type BlockOutput = export interface ParamConfig { type: ParamType required: boolean + description?: string + schema?: { + type: string + properties: Record + required?: string[] + additionalProperties?: boolean + items?: { + type: string + properties?: Record + required?: string[] + additionalProperties?: boolean + } + } } export interface SubBlockConfig { diff --git a/executor/index.ts b/executor/index.ts index 072c67ebba..40c4b82519 100644 --- a/executor/index.ts +++ b/executor/index.ts @@ -203,11 +203,6 @@ export class Executor { inputs: Record, context: ExecutionContext ): Promise { - // console.log(`Executing block ${block.metadata?.title} (${block.id})`, { - // type: block.metadata?.type, - // inputs - // }); - // Start timing const startTime = new Date() const blockLog: BlockLog = { @@ -256,11 +251,25 @@ export class Executor { // Special handling for agent blocks that use providers if (block.metadata?.type === 'agent') { - // console.log('Agent inputs:', { - // systemPrompt: inputs.systemPrompt, - // context: inputs.context, - // tools: inputs.tools - // }); + console.log('Executing agent block with inputs:', inputs) + + // Get response format from inputs if provided + let responseFormat = undefined + if (inputs.responseFormat) { + try { + // If it's already a string, parse it once + if (typeof inputs.responseFormat === 'string') { + responseFormat = JSON.parse(inputs.responseFormat) + } else { + // If it's somehow already an object, use it directly + responseFormat = inputs.responseFormat + } + console.log('Parsed responseFormat:', responseFormat) + } catch (error: any) { + console.error('Error parsing responseFormat:', error) + throw new Error('Invalid response format: ' + error.message) + } + } const model = inputs.model || 'gpt-4o' const providerId = getProviderFromModel(model) @@ -269,24 +278,18 @@ export class Executor { const tools = Array.isArray(inputs.tools) ? inputs.tools .map((tool: any) => { - // console.log('Processing tool:', tool); - // Get the tool ID from the block type const block = getAllBlocks().find((b: BlockConfig) => b.type === tool.type) const toolId = block?.tools.access[0] if (!toolId) { - // console.log('No tool ID found for type:', tool.type); return null } - // Get the tool configuration const toolConfig = getTool(toolId) if (!toolConfig) { - // console.log('No tool config found for ID:', toolId); return null } - // Return the tool configuration with resolved parameters - const toolSetup = { + return { id: toolConfig.id, name: toolConfig.name, description: toolConfig.description, @@ -309,14 +312,10 @@ export class Executor { .map(([key]) => key), }, } - // console.log('Tool setup:', toolSetup); - return toolSetup }) .filter((t): t is NonNullable => t !== null) : [] - // console.log('Formatted tools:', tools); - const response = await executeProviderRequest(providerId, { model, systemPrompt: inputs.systemPrompt, @@ -325,28 +324,42 @@ export class Executor { temperature: inputs.temperature, maxTokens: inputs.maxTokens, apiKey: inputs.apiKey, + responseFormat, }) - // console.log('Provider response:', { - // content: response.content, - // toolCalls: response.toolCalls - // }); - - const output = { - response: { - content: response.content, - model: response.model, - tokens: response.tokens || { - prompt: 0, - completion: 0, - total: 0, - }, - toolCalls: { - list: response.toolCalls || [], - count: response.toolCalls?.length || 0, - }, - }, - } + // If responseFormat was specified, return the content directly as the response + // with metadata as additional fields + const output = responseFormat + ? { + ...JSON.parse(response.content), // The formatted response as the root + tokens: response.tokens || { + prompt: 0, + completion: 0, + total: 0, + }, + toolCalls: response.toolCalls + ? { + list: response.toolCalls, + count: response.toolCalls.length, + } + : undefined, + } + : { + // Default format when no responseFormat specified + response: { + content: response.content, + model: response.model, + tokens: response.tokens || { + prompt: 0, + completion: 0, + total: 0, + }, + toolCalls: { + list: response.toolCalls || [], + count: response.toolCalls?.length || 0, + }, + }, + } blockLog.success = true blockLog.output = output @@ -368,11 +381,6 @@ export class Executor { throw new Error(`Tool ${block.config.tool} not found`) } - // console.log('Executing tool:', { - // tool: block.config.tool, - // inputs - // }); - const result = await executeTool(block.config.tool, inputs) if (!result.success) { diff --git a/providers/anthropic/index.ts b/providers/anthropic/index.ts index 1f2b3009dd..86edc5bcea 100644 --- a/providers/anthropic/index.ts +++ b/providers/anthropic/index.ts @@ -120,7 +120,15 @@ export const anthropicProvider: ProviderConfig = { }) } - // Build the request payload exactly as Anthropic expects + // Ensure there's at least one message by adding the system prompt as a user message if no messages exist + if (messages.length === 0) { + messages.push({ + role: 'user', + content: [{ type: 'text', text: request.systemPrompt || '' }], + }) + } + + // Build the request payload const payload = { model: request.model || 'claude-3-5-sonnet-20241022', messages, @@ -130,6 +138,23 @@ export const anthropicProvider: ProviderConfig = { ...(functions && { tools: functions }), } + // If response format is specified, add strict formatting instructions + if (request.responseFormat) { + payload.system = `${payload.system}\n\nIMPORTANT RESPONSE FORMAT INSTRUCTIONS: +1. Your response must be EXACTLY in this format, with no additional fields: +{ +${request.responseFormat.fields.map((field) => ` "${field.name}": ${field.type === 'string' ? '"value"' : field.type === 'array' ? '[]' : field.type === 'object' ? '{}' : field.type === 'number' ? '0' : 'true/false'}`).join(',\n')} +} + +Field descriptions: +${request.responseFormat.fields.map((field) => `${field.name} (${field.type})${field.description ? `: ${field.description}` : ''}`).join('\n')} + +2. DO NOT include any explanatory text before or after the JSON +3. DO NOT wrap the response in an array +4. DO NOT add any fields not specified in the schema +5. Your response MUST be valid JSON and include all the specified fields with their correct types` + } + return payload }, @@ -156,19 +181,27 @@ export const anthropicProvider: ProviderConfig = { content = messageContent } - const result = { - content, - model: rawResponse?.model || response?.model || 'claude-3-5-sonnet-20241022', - ...(rawResponse?.usage && { - tokens: { - prompt: rawResponse.usage.input_tokens, - completion: rawResponse.usage.output_tokens, - total: rawResponse.usage.input_tokens + rawResponse.usage.output_tokens, - }, - }), + // If the content looks like it contains JSON, extract just the JSON part + if (content.includes('{') && content.includes('}')) { + try { + const jsonMatch = content.match(/\{[\s\S]*\}/m) + if (jsonMatch) { + content = jsonMatch[0] + } + } catch (e) { + console.error('Error extracting JSON from response:', e) + } } - return result + return { + content, + model: rawResponse?.model || response?.model || 'claude-3-5-sonnet-20241022', + tokens: rawResponse?.usage && { + prompt: rawResponse.usage.input_tokens, + completion: rawResponse.usage.output_tokens, + total: rawResponse.usage.input_tokens + rawResponse.usage.output_tokens, + }, + } } catch (error) { console.error('Error in transformResponse:', error) return { content: '' } diff --git a/providers/deepseek/index.ts b/providers/deepseek/index.ts index 9d60535702..d702115c9e 100644 --- a/providers/deepseek/index.ts +++ b/providers/deepseek/index.ts @@ -116,8 +116,17 @@ export const deepseekProvider: ProviderConfig = { const output = response.choices?.[0]?.message + // Try to clean up the response content if it exists + let content = output?.content || '' + if (content) { + // Remove any markdown code block markers + content = content.replace(/```json\n?|\n?```/g, '') + // Trim any whitespace + content = content.trim() + } + return { - content: output?.content || '', + content, tokens: response.usage && { prompt: response.usage.prompt_tokens, completion: response.usage.completion_tokens, diff --git a/providers/google/index.ts b/providers/google/index.ts index a23abb3213..381eee8b36 100644 --- a/providers/google/index.ts +++ b/providers/google/index.ts @@ -114,7 +114,68 @@ export const googleProvider: ProviderConfig = { transformRequest: (request: ProviderRequest, functions?: any) => { // Combine system prompt and context into a single message if both exist - const initialMessage = request.systemPrompt + (request.context ? `\n\n${request.context}` : '') + let initialMessage = request.systemPrompt + + // Add response format for structured output if specified + let generationConfig: any = { + temperature: request.temperature || 0.7, + maxOutputTokens: request.maxTokens || 1024, + } + + if (request.responseFormat) { + // Convert our standard JSON Schema format to Google's expected format + const googleSchema = { + type: 'OBJECT', + properties: request.responseFormat.fields.reduce( + (acc, field) => ({ + ...acc, + [field.name]: { + type: + field.type === 'array' + ? 'ARRAY' + : field.type === 'object' + ? 'OBJECT' + : field.type === 'number' + ? 'NUMBER' + : field.type === 'boolean' + ? 'BOOLEAN' + : 'STRING', + description: field.description || '', + ...(field.type === 'array' && { + items: { + type: 'STRING', + }, + }), + ...(field.type === 'object' && { + properties: { + // Add a flexible string field to satisfy Google's validation + _any: { + type: 'STRING', + description: 'Any additional metadata', + }, + }, + }), + }, + }), + {} + ), + } + + // Add schema to generation config + generationConfig = { + ...generationConfig, + response_mime_type: 'application/json', + response_schema: googleSchema, + } + + // Add clear instructions in the initial message + initialMessage = `${initialMessage}\n\nPlease provide your response as a valid JSON object following the specified schema. The response should only contain the JSON data without any additional text or explanations.` + } + + // Add context if provided + if (request.context) { + initialMessage += `\n\n${request.context}` + } const messages = [ { role: 'user', parts: [{ text: initialMessage }] }, @@ -173,10 +234,7 @@ export const googleProvider: ProviderConfig = { return { contents: messages, tools: functions ? [{ functionDeclarations: functions.functionDeclarations }] : undefined, - generationConfig: { - temperature: request.temperature || 0.7, - maxOutputTokens: request.maxTokens || 1024, - }, + generationConfig, } }, diff --git a/providers/openai/index.ts b/providers/openai/index.ts index 65361957e6..04a603c065 100644 --- a/providers/openai/index.ts +++ b/providers/openai/index.ts @@ -99,6 +99,34 @@ export const openaiProvider: ProviderConfig = { // gpt-4o supports standard parameters if (request.temperature !== undefined) payload.temperature = request.temperature if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens + + // Add response format for structured output if specified + if (request.responseFormat) { + // Use OpenAI's simpler response format + payload.response_format = { type: 'json_object' } + + // If we have both function calls and response format, we need to guide the model + if (functions) { + payload.messages[0].content = `${payload.messages[0].content}\n\nProcess:\n1. First, use the provided functions to gather the necessary data\n2. Then, format your final response as a SINGLE JSON object with these exact fields and types:\n${request.responseFormat.fields + .map( + (field) => + `- "${field.name}" (${field.type})${field.description ? `: ${field.description}` : ''}` + ) + .join( + '\n' + )}\n\nYour final response after function calls must be a SINGLE valid JSON object with all required fields and correct types. Do not return multiple objects or include any text outside the JSON.` + } else { + // If no functions, just format as JSON directly + payload.messages[0].content = `${payload.messages[0].content}\n\nYou MUST return a SINGLE JSON object with exactly these fields and types:\n${request.responseFormat.fields + .map( + (field) => + `- "${field.name}" (${field.type})${field.description ? `: ${field.description}` : ''}` + ) + .join( + '\n' + )}\n\nThe response must:\n1. Be a single valid JSON object\n2. Include all the specified fields\n3. Use the correct type for each field\n4. Not include any additional fields\n5. Not include any explanatory text outside the JSON\n6. Not return multiple objects` + } + } } else { // o1 models use max_completion_tokens if (request.maxTokens !== undefined) { diff --git a/providers/service.ts b/providers/service.ts index e75497e2dd..c01d9a9266 100644 --- a/providers/service.ts +++ b/providers/service.ts @@ -2,6 +2,28 @@ import { executeTool, getTool } from '@/tools' import { getProvider } from './registry' import { ProviderRequest, ProviderResponse, TokenInfo } from './types' +// Helper function to generate provider-specific structured output instructions +function generateStructuredOutputInstructions(responseFormat: any): string { + if (!responseFormat?.fields) return '' + + const fields = responseFormat.fields + .map((field: any) => { + return `${field.name} (${field.type})${field.description ? `: ${field.description}` : ''}` + }) + .join('\n') + + return ` +Please provide your response in the following JSON format: +{ + ${responseFormat.fields.map((field: any) => `"${field.name}": "${field.type === 'string' ? 'value' : field.type === 'number' ? '0' : field.type === 'boolean' ? 'true/false' : '[]'}"`).join(',\n ')} +} + +Field descriptions: +${fields} + +Your response MUST be valid JSON and include all the specified fields with their correct types.` +} + export async function executeProviderRequest( providerId: string, request: ProviderRequest @@ -11,6 +33,14 @@ export async function executeProviderRequest( throw new Error(`Provider not found: ${providerId}`) } + // If responseFormat is provided, modify the system prompt to enforce structured output + if (request.responseFormat) { + const structuredOutputInstructions = generateStructuredOutputInstructions( + request.responseFormat + ) + request.systemPrompt = `${request.systemPrompt}\n\n${structuredOutputInstructions}` + } + // Transform tools to provider-specific function format const functions = request.tools && request.tools.length > 0 @@ -38,6 +68,51 @@ export async function executeProviderRequest( const transformedResponse = provider.transformResponse(currentResponse) content = transformedResponse.content + // If responseFormat is specified and we have content (not a function call), validate and parse the response + if (request.responseFormat && content && !provider.hasFunctionCall(currentResponse)) { + try { + // Try to parse the content as JSON + const parsedContent = JSON.parse(content) + + // Validate that all required fields are present and have correct types + const validationErrors = request.responseFormat.fields + .map((field: any) => { + if (!(field.name in parsedContent)) { + return `Missing field: ${field.name}` + } + const value = parsedContent[field.name] + const type = typeof value + if (field.type === 'string' && type !== 'string') { + return `Invalid type for ${field.name}: expected string, got ${type}` + } + if (field.type === 'number' && type !== 'number') { + return `Invalid type for ${field.name}: expected number, got ${type}` + } + if (field.type === 'boolean' && type !== 'boolean') { + return `Invalid type for ${field.name}: expected boolean, got ${type}` + } + if (field.type === 'array' && !Array.isArray(value)) { + return `Invalid type for ${field.name}: expected array, got ${type}` + } + if (field.type === 'object' && (type !== 'object' || Array.isArray(value))) { + return `Invalid type for ${field.name}: expected object, got ${type}` + } + return null + }) + .filter(Boolean) + + if (validationErrors.length > 0) { + throw new Error(`Response format validation failed:\n${validationErrors.join('\n')}`) + } + + // Store the validated JSON response + content = JSON.stringify(parsedContent) + } catch (error: any) { + console.error('Error parsing structured response:', error) + throw new Error(`Failed to parse response as structured output: ${error.message}`) + } + } + // Update tokens if (transformedResponse.tokens) { const newTokens: TokenInfo = { @@ -138,7 +213,7 @@ export async function executeProviderRequest( if (iterationCount >= MAX_ITERATIONS) { console.log('Max iterations reached, breaking loop') } - } catch (error: any) { + } catch (error) { console.error('Error executing tool:', error) throw error } diff --git a/providers/types.ts b/providers/types.ts index 180d1e53d5..bfb9063768 100644 --- a/providers/types.ts +++ b/providers/types.ts @@ -89,6 +89,13 @@ export interface ProviderRequest { maxTokens?: number apiKey: string messages?: Message[] + responseFormat?: { + fields: Array<{ + name: string + type: 'string' | 'number' | 'boolean' | 'array' | 'object' + description?: string + }> + } } // Map of provider IDs to their configurations diff --git a/providers/xai/index.ts b/providers/xai/index.ts index 0d5c38bdb0..4a88ca19c1 100644 --- a/providers/xai/index.ts +++ b/providers/xai/index.ts @@ -94,10 +94,53 @@ export const xAIProvider: ProviderConfig = { return msg }) + // Add response format for structured output if specified + let systemPrompt = request.systemPrompt + if (request.responseFormat) { + systemPrompt += `\n\nYou MUST respond with a valid JSON object. DO NOT include any other text, explanations, or markdown formatting in your response - ONLY the JSON object.\n\nThe response MUST match this schema:\n${JSON.stringify( + { + type: 'object', + properties: request.responseFormat.fields.reduce( + (acc, field) => ({ + ...acc, + [field.name]: { + type: + field.type === 'array' + ? 'array' + : field.type === 'object' + ? 'object' + : field.type, + description: field.description, + }, + }), + {} + ), + required: request.responseFormat.fields.map((f) => f.name), + }, + null, + 2 + )}\n\nExample response format:\n{\n${request.responseFormat.fields + .map( + (f) => + ` "${f.name}": ${ + f.type === 'string' + ? '"value"' + : f.type === 'number' + ? '0' + : f.type === 'boolean' + ? 'true' + : f.type === 'array' + ? '[]' + : '{}' + }` + ) + .join(',\n')}\n}` + } + const payload = { model: request.model || 'grok-2-latest', messages: [ - { role: 'system', content: request.systemPrompt }, + { role: 'system', content: systemPrompt }, ...(request.context ? [{ role: 'user', content: request.context }] : []), ...messages, ], @@ -107,6 +150,45 @@ export const xAIProvider: ProviderConfig = { tools: functions, tool_choice: 'auto', // xAI specific parameter }), + ...(request.responseFormat && { + response_format: { + type: 'json_schema', + json_schema: { + name: 'structured_response', + schema: { + type: 'object', + properties: request.responseFormat.fields.reduce( + (acc, field) => ({ + ...acc, + [field.name]: { + type: + field.type === 'array' + ? 'array' + : field.type === 'object' + ? 'object' + : field.type === 'number' + ? 'number' + : field.type === 'boolean' + ? 'boolean' + : 'string', + description: field.description || '', + ...(field.type === 'array' && { + items: { type: 'string' }, + }), + ...(field.type === 'object' && { + additionalProperties: true, + }), + }, + }), + {} + ), + required: request.responseFormat.fields.map((f) => f.name), + additionalProperties: false, + }, + strict: true, + }, + }, + }), } return payload diff --git a/stores/workflow/store.ts b/stores/workflow/store.ts index e092cccd1e..89eb8288d5 100644 --- a/stores/workflow/store.ts +++ b/stores/workflow/store.ts @@ -39,12 +39,52 @@ export const useWorkflowStore = create()( const blockConfig = getBlock(block.type) if (!blockConfig) return state - // Create new subBlocks state + // Validate responseFormat if it's the agent block's responseFormat input + if (blockConfig.type === 'agent' && subBlockId === 'responseFormat' && value) { + console.log('Validating responseFormat input:', { + type: typeof value, + rawValue: value, + }) + + try { + // Parse the input string to validate JSON but keep original string value + const parsed = JSON.parse(value) + console.log('Parsed responseFormat:', parsed) + + // Simple validation of required schema structure + if (!parsed.fields || !Array.isArray(parsed.fields)) { + console.error('Validation failed: missing fields array') + throw new Error('Response format must have a fields array') + } + + for (const field of parsed.fields) { + console.log('Validating field:', field) + if (!field.name || !field.type) { + console.error('Validation failed: field missing name or type', field) + throw new Error('Each field must have a name and type') + } + if (!['string', 'number', 'boolean', 'array', 'object'].includes(field.type)) { + console.error('Validation failed: invalid field type', field) + throw new Error( + `Invalid type "${field.type}" - must be one of: string, number, boolean, array, object` + ) + } + } + + console.log('responseFormat validation successful') + // Don't modify the value, keep it as the original string + } catch (error: any) { + console.error('responseFormat validation error:', error) + throw new Error(`Invalid JSON schema: ${error.message}`) + } + } + + // Create new subBlocks state with the original value const newSubBlocks = { ...block.subBlocks, [subBlockId]: { ...block.subBlocks[subBlockId], - value, + value: typeof value === 'string' ? value : JSON.stringify(value, null, 2), }, }