Added support for response format in agent block. Works in all but deepseek (will fix) and gemini (does not support for this model). If responseformat is not specified, we get the response back wrapped in 'content' like before

2026-09-24 15:45:35 +08:00 · 2025-02-06 19:59:44 -08:00
parent 2ca3d29c49
commit a10a724a51
11 changed files with 470 additions and 69 deletions
@@ -60,7 +60,41 @@ export const AgentBlock: BlockConfig<AgentResponse> = {
      context: { type: 'string', required: false },
      model: { type: 'string', required: true },
      apiKey: { type: 'string', required: true },
-      responseFormat: { type: 'json', required: false },
+      responseFormat: {
+        type: 'json',
+        required: false,
+        description:
+          'Define the expected response format. If not provided, returns plain text content.',
+        schema: {
+          type: 'object',
+          properties: {
+            fields: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  name: {
+                    type: 'string',
+                    minLength: 1,
+                  },
+                  type: {
+                    type: 'string',
+                    enum: ['string', 'number', 'boolean', 'array', 'object'],
+                  },
+                  description: {
+                    type: 'string',
+                  },
+                },
+                required: ['name', 'type'],
+                additionalProperties: false,
+              },
+              minItems: 1,
+            },
+          },
+          required: ['fields'],
+          additionalProperties: false,
+        },
+      },
      temperature: { type: 'number', required: false },
      tools: { type: 'json', required: false },
    },
@@ -124,6 +158,20 @@ export const AgentBlock: BlockConfig<AgentResponse> = {
        title: 'Response Format',
        type: 'code',
        layout: 'full',
+        placeholder: `{
+  "fields": [
+    {
+      "name": "sentiment",
+      "type": "string",
+      "description": "The sentiment of the text (positive, negative, neutral)"
+    },
+    {
+      "name": "score",
+      "type": "number",
+      "description": "Confidence score between 0 and 1"
+    }
+  ]
+}`,
      },
    ],
  },
@@ -48,6 +48,19 @@ export type BlockOutput =
 export interface ParamConfig {
  type: ParamType
  required: boolean
+  description?: string
+  schema?: {
+    type: string
+    properties: Record<string, any>
+    required?: string[]
+    additionalProperties?: boolean
+    items?: {
+      type: string
+      properties?: Record<string, any>
+      required?: string[]
+      additionalProperties?: boolean
+    }
+  }
 }

 export interface SubBlockConfig {
@@ -203,11 +203,6 @@ export class Executor {
    inputs: Record<string, any>,
    context: ExecutionContext
  ): Promise<BlockOutput> {
-    // console.log(`Executing block ${block.metadata?.title} (${block.id})`, {
-    //   type: block.metadata?.type,
-    //   inputs
-    // });
-
    // Start timing
    const startTime = new Date()
    const blockLog: BlockLog = {
@@ -256,11 +251,25 @@ export class Executor {

      // Special handling for agent blocks that use providers
      if (block.metadata?.type === 'agent') {
-        // console.log('Agent inputs:', {
-        //   systemPrompt: inputs.systemPrompt,
-        //   context: inputs.context,
-        //   tools: inputs.tools
-        // });
+        console.log('Executing agent block with inputs:', inputs)
+
+        // Get response format from inputs if provided
+        let responseFormat = undefined
+        if (inputs.responseFormat) {
+          try {
+            // If it's already a string, parse it once
+            if (typeof inputs.responseFormat === 'string') {
+              responseFormat = JSON.parse(inputs.responseFormat)
+            } else {
+              // If it's somehow already an object, use it directly
+              responseFormat = inputs.responseFormat
+            }
+            console.log('Parsed responseFormat:', responseFormat)
+          } catch (error: any) {
+            console.error('Error parsing responseFormat:', error)
+            throw new Error('Invalid response format: ' + error.message)
+          }
+        }

        const model = inputs.model || 'gpt-4o'
        const providerId = getProviderFromModel(model)
@@ -269,24 +278,18 @@ export class Executor {
        const tools = Array.isArray(inputs.tools)
          ? inputs.tools
              .map((tool: any) => {
-                // console.log('Processing tool:', tool);
-                // Get the tool ID from the block type
                const block = getAllBlocks().find((b: BlockConfig) => b.type === tool.type)
                const toolId = block?.tools.access[0]
                if (!toolId) {
-                  // console.log('No tool ID found for type:', tool.type);
                  return null
                }

-                // Get the tool configuration
                const toolConfig = getTool(toolId)
                if (!toolConfig) {
-                  // console.log('No tool config found for ID:', toolId);
                  return null
                }

-                // Return the tool configuration with resolved parameters
-                const toolSetup = {
+                return {
                  id: toolConfig.id,
                  name: toolConfig.name,
                  description: toolConfig.description,
@@ -309,14 +312,10 @@ export class Executor {
                      .map(([key]) => key),
                  },
                }
-                // console.log('Tool setup:', toolSetup);
-                return toolSetup
              })
              .filter((t): t is NonNullable<typeof t> => t !== null)
          : []

-        // console.log('Formatted tools:', tools);
-
        const response = await executeProviderRequest(providerId, {
          model,
          systemPrompt: inputs.systemPrompt,
@@ -325,28 +324,42 @@ export class Executor {
          temperature: inputs.temperature,
          maxTokens: inputs.maxTokens,
          apiKey: inputs.apiKey,
+          responseFormat,
        })

-        // console.log('Provider response:', {
-        //   content: response.content,
-        //   toolCalls: response.toolCalls
-        // });
-
-        const output = {
-          response: {
-            content: response.content,
-            model: response.model,
-            tokens: response.tokens || {
-              prompt: 0,
-              completion: 0,
-              total: 0,
-            },
-            toolCalls: {
-              list: response.toolCalls || [],
-              count: response.toolCalls?.length || 0,
-            },
-          },
-        }
+        // If responseFormat was specified, return the content directly as the response
+        // with metadata as additional fields
+        const output = responseFormat
+          ? {
+              ...JSON.parse(response.content), // The formatted response as the root
+              tokens: response.tokens || {
+                prompt: 0,
+                completion: 0,
+                total: 0,
+              },
+              toolCalls: response.toolCalls
+                ? {
+                    list: response.toolCalls,
+                    count: response.toolCalls.length,
+                  }
+                : undefined,
+            }
+          : {
+              // Default format when no responseFormat specified
+              response: {
+                content: response.content,
+                model: response.model,
+                tokens: response.tokens || {
+                  prompt: 0,
+                  completion: 0,
+                  total: 0,
+                },
+                toolCalls: {
+                  list: response.toolCalls || [],
+                  count: response.toolCalls?.length || 0,
+                },
+              },
+            }

        blockLog.success = true
        blockLog.output = output
@@ -368,11 +381,6 @@ export class Executor {
        throw new Error(`Tool ${block.config.tool} not found`)
      }

-      // console.log('Executing tool:', {
-      //   tool: block.config.tool,
-      //   inputs
-      // });
-
      const result = await executeTool(block.config.tool, inputs)

      if (!result.success) {
@@ -120,7 +120,15 @@ export const anthropicProvider: ProviderConfig = {
      })
    }

-    // Build the request payload exactly as Anthropic expects
+    // Ensure there's at least one message by adding the system prompt as a user message if no messages exist
+    if (messages.length === 0) {
+      messages.push({
+        role: 'user',
+        content: [{ type: 'text', text: request.systemPrompt || '' }],
+      })
+    }
+
+    // Build the request payload
    const payload = {
      model: request.model || 'claude-3-5-sonnet-20241022',
      messages,
@@ -130,6 +138,23 @@ export const anthropicProvider: ProviderConfig = {
      ...(functions && { tools: functions }),
    }

+    // If response format is specified, add strict formatting instructions
+    if (request.responseFormat) {
+      payload.system = `${payload.system}\n\nIMPORTANT RESPONSE FORMAT INSTRUCTIONS:
+1. Your response must be EXACTLY in this format, with no additional fields:
+{
+${request.responseFormat.fields.map((field) => `  "${field.name}": ${field.type === 'string' ? '"value"' : field.type === 'array' ? '[]' : field.type === 'object' ? '{}' : field.type === 'number' ? '0' : 'true/false'}`).join(',\n')}
+}
+
+Field descriptions:
+${request.responseFormat.fields.map((field) => `${field.name} (${field.type})${field.description ? `: ${field.description}` : ''}`).join('\n')}
+
+2. DO NOT include any explanatory text before or after the JSON
+3. DO NOT wrap the response in an array
+4. DO NOT add any fields not specified in the schema
+5. Your response MUST be valid JSON and include all the specified fields with their correct types`
+    }
+
    return payload
  },

@@ -156,19 +181,27 @@ export const anthropicProvider: ProviderConfig = {
        content = messageContent
      }

-      const result = {
-        content,
-        model: rawResponse?.model || response?.model || 'claude-3-5-sonnet-20241022',
-        ...(rawResponse?.usage && {
-          tokens: {
-            prompt: rawResponse.usage.input_tokens,
-            completion: rawResponse.usage.output_tokens,
-            total: rawResponse.usage.input_tokens + rawResponse.usage.output_tokens,
-          },
-        }),
+      // If the content looks like it contains JSON, extract just the JSON part
+      if (content.includes('{') && content.includes('}')) {
+        try {
+          const jsonMatch = content.match(/\{[\s\S]*\}/m)
+          if (jsonMatch) {
+            content = jsonMatch[0]
+          }
+        } catch (e) {
+          console.error('Error extracting JSON from response:', e)
+        }
      }

-      return result
+      return {
+        content,
+        model: rawResponse?.model || response?.model || 'claude-3-5-sonnet-20241022',
+        tokens: rawResponse?.usage && {
+          prompt: rawResponse.usage.input_tokens,
+          completion: rawResponse.usage.output_tokens,
+          total: rawResponse.usage.input_tokens + rawResponse.usage.output_tokens,
+        },
+      }
    } catch (error) {
      console.error('Error in transformResponse:', error)
      return { content: '' }
@@ -116,8 +116,17 @@ export const deepseekProvider: ProviderConfig = {

    const output = response.choices?.[0]?.message

+    // Try to clean up the response content if it exists
+    let content = output?.content || ''
+    if (content) {
+      // Remove any markdown code block markers
+      content = content.replace(/```json\n?|\n?```/g, '')
+      // Trim any whitespace
+      content = content.trim()
+    }
+
    return {
-      content: output?.content || '',
+      content,
      tokens: response.usage && {
        prompt: response.usage.prompt_tokens,
        completion: response.usage.completion_tokens,
@@ -114,7 +114,68 @@ export const googleProvider: ProviderConfig = {

  transformRequest: (request: ProviderRequest, functions?: any) => {
    // Combine system prompt and context into a single message if both exist
-    const initialMessage = request.systemPrompt + (request.context ? `\n\n${request.context}` : '')
+    let initialMessage = request.systemPrompt
+
+    // Add response format for structured output if specified
+    let generationConfig: any = {
+      temperature: request.temperature || 0.7,
+      maxOutputTokens: request.maxTokens || 1024,
+    }
+
+    if (request.responseFormat) {
+      // Convert our standard JSON Schema format to Google's expected format
+      const googleSchema = {
+        type: 'OBJECT',
+        properties: request.responseFormat.fields.reduce(
+          (acc, field) => ({
+            ...acc,
+            [field.name]: {
+              type:
+                field.type === 'array'
+                  ? 'ARRAY'
+                  : field.type === 'object'
+                    ? 'OBJECT'
+                    : field.type === 'number'
+                      ? 'NUMBER'
+                      : field.type === 'boolean'
+                        ? 'BOOLEAN'
+                        : 'STRING',
+              description: field.description || '',
+              ...(field.type === 'array' && {
+                items: {
+                  type: 'STRING',
+                },
+              }),
+              ...(field.type === 'object' && {
+                properties: {
+                  // Add a flexible string field to satisfy Google's validation
+                  _any: {
+                    type: 'STRING',
+                    description: 'Any additional metadata',
+                  },
+                },
+              }),
+            },
+          }),
+          {}
+        ),
+      }
+
+      // Add schema to generation config
+      generationConfig = {
+        ...generationConfig,
+        response_mime_type: 'application/json',
+        response_schema: googleSchema,
+      }
+
+      // Add clear instructions in the initial message
+      initialMessage = `${initialMessage}\n\nPlease provide your response as a valid JSON object following the specified schema. The response should only contain the JSON data without any additional text or explanations.`
+    }
+
+    // Add context if provided
+    if (request.context) {
+      initialMessage += `\n\n${request.context}`
+    }

    const messages = [
      { role: 'user', parts: [{ text: initialMessage }] },
@@ -173,10 +234,7 @@ export const googleProvider: ProviderConfig = {
    return {
      contents: messages,
      tools: functions ? [{ functionDeclarations: functions.functionDeclarations }] : undefined,
-      generationConfig: {
-        temperature: request.temperature || 0.7,
-        maxOutputTokens: request.maxTokens || 1024,
-      },
+      generationConfig,
    }
  },

@@ -99,6 +99,34 @@ export const openaiProvider: ProviderConfig = {
      // gpt-4o supports standard parameters
      if (request.temperature !== undefined) payload.temperature = request.temperature
      if (request.maxTokens !== undefined) payload.max_tokens = request.maxTokens
+
+      // Add response format for structured output if specified
+      if (request.responseFormat) {
+        // Use OpenAI's simpler response format
+        payload.response_format = { type: 'json_object' }
+
+        // If we have both function calls and response format, we need to guide the model
+        if (functions) {
+          payload.messages[0].content = `${payload.messages[0].content}\n\nProcess:\n1. First, use the provided functions to gather the necessary data\n2. Then, format your final response as a SINGLE JSON object with these exact fields and types:\n${request.responseFormat.fields
+            .map(
+              (field) =>
+                `- "${field.name}" (${field.type})${field.description ? `: ${field.description}` : ''}`
+            )
+            .join(
+              '\n'
+            )}\n\nYour final response after function calls must be a SINGLE valid JSON object with all required fields and correct types. Do not return multiple objects or include any text outside the JSON.`
+        } else {
+          // If no functions, just format as JSON directly
+          payload.messages[0].content = `${payload.messages[0].content}\n\nYou MUST return a SINGLE JSON object with exactly these fields and types:\n${request.responseFormat.fields
+            .map(
+              (field) =>
+                `- "${field.name}" (${field.type})${field.description ? `: ${field.description}` : ''}`
+            )
+            .join(
+              '\n'
+            )}\n\nThe response must:\n1. Be a single valid JSON object\n2. Include all the specified fields\n3. Use the correct type for each field\n4. Not include any additional fields\n5. Not include any explanatory text outside the JSON\n6. Not return multiple objects`
+        }
+      }
    } else {
      // o1 models use max_completion_tokens
      if (request.maxTokens !== undefined) {
@@ -2,6 +2,28 @@ import { executeTool, getTool } from '@/tools'
 import { getProvider } from './registry'
 import { ProviderRequest, ProviderResponse, TokenInfo } from './types'

+// Helper function to generate provider-specific structured output instructions
+function generateStructuredOutputInstructions(responseFormat: any): string {
+  if (!responseFormat?.fields) return ''
+
+  const fields = responseFormat.fields
+    .map((field: any) => {
+      return `${field.name} (${field.type})${field.description ? `: ${field.description}` : ''}`
+    })
+    .join('\n')
+
+  return `
+Please provide your response in the following JSON format:
+{
+  ${responseFormat.fields.map((field: any) => `"${field.name}": "${field.type === 'string' ? 'value' : field.type === 'number' ? '0' : field.type === 'boolean' ? 'true/false' : '[]'}"`).join(',\n  ')}
+}
+
+Field descriptions:
+${fields}
+
+Your response MUST be valid JSON and include all the specified fields with their correct types.`
+}
+
 export async function executeProviderRequest(
  providerId: string,
  request: ProviderRequest
@@ -11,6 +33,14 @@ export async function executeProviderRequest(
    throw new Error(`Provider not found: ${providerId}`)
  }

+  // If responseFormat is provided, modify the system prompt to enforce structured output
+  if (request.responseFormat) {
+    const structuredOutputInstructions = generateStructuredOutputInstructions(
+      request.responseFormat
+    )
+    request.systemPrompt = `${request.systemPrompt}\n\n${structuredOutputInstructions}`
+  }
+
  // Transform tools to provider-specific function format
  const functions =
    request.tools && request.tools.length > 0
@@ -38,6 +68,51 @@ export async function executeProviderRequest(
      const transformedResponse = provider.transformResponse(currentResponse)
      content = transformedResponse.content

+      // If responseFormat is specified and we have content (not a function call), validate and parse the response
+      if (request.responseFormat && content && !provider.hasFunctionCall(currentResponse)) {
+        try {
+          // Try to parse the content as JSON
+          const parsedContent = JSON.parse(content)
+
+          // Validate that all required fields are present and have correct types
+          const validationErrors = request.responseFormat.fields
+            .map((field: any) => {
+              if (!(field.name in parsedContent)) {
+                return `Missing field: ${field.name}`
+              }
+              const value = parsedContent[field.name]
+              const type = typeof value
+              if (field.type === 'string' && type !== 'string') {
+                return `Invalid type for ${field.name}: expected string, got ${type}`
+              }
+              if (field.type === 'number' && type !== 'number') {
+                return `Invalid type for ${field.name}: expected number, got ${type}`
+              }
+              if (field.type === 'boolean' && type !== 'boolean') {
+                return `Invalid type for ${field.name}: expected boolean, got ${type}`
+              }
+              if (field.type === 'array' && !Array.isArray(value)) {
+                return `Invalid type for ${field.name}: expected array, got ${type}`
+              }
+              if (field.type === 'object' && (type !== 'object' || Array.isArray(value))) {
+                return `Invalid type for ${field.name}: expected object, got ${type}`
+              }
+              return null
+            })
+            .filter(Boolean)
+
+          if (validationErrors.length > 0) {
+            throw new Error(`Response format validation failed:\n${validationErrors.join('\n')}`)
+          }
+
+          // Store the validated JSON response
+          content = JSON.stringify(parsedContent)
+        } catch (error: any) {
+          console.error('Error parsing structured response:', error)
+          throw new Error(`Failed to parse response as structured output: ${error.message}`)
+        }
+      }
+
      // Update tokens
      if (transformedResponse.tokens) {
        const newTokens: TokenInfo = {
@@ -138,7 +213,7 @@ export async function executeProviderRequest(
    if (iterationCount >= MAX_ITERATIONS) {
      console.log('Max iterations reached, breaking loop')
    }
-  } catch (error: any) {
+  } catch (error) {
    console.error('Error executing tool:', error)
    throw error
  }
@@ -89,6 +89,13 @@ export interface ProviderRequest {
  maxTokens?: number
  apiKey: string
  messages?: Message[]
+  responseFormat?: {
+    fields: Array<{
+      name: string
+      type: 'string' | 'number' | 'boolean' | 'array' | 'object'
+      description?: string
+    }>
+  }
 }

 // Map of provider IDs to their configurations
@@ -94,10 +94,53 @@ export const xAIProvider: ProviderConfig = {
      return msg
    })

+    // Add response format for structured output if specified
+    let systemPrompt = request.systemPrompt
+    if (request.responseFormat) {
+      systemPrompt += `\n\nYou MUST respond with a valid JSON object. DO NOT include any other text, explanations, or markdown formatting in your response - ONLY the JSON object.\n\nThe response MUST match this schema:\n${JSON.stringify(
+        {
+          type: 'object',
+          properties: request.responseFormat.fields.reduce(
+            (acc, field) => ({
+              ...acc,
+              [field.name]: {
+                type:
+                  field.type === 'array'
+                    ? 'array'
+                    : field.type === 'object'
+                      ? 'object'
+                      : field.type,
+                description: field.description,
+              },
+            }),
+            {}
+          ),
+          required: request.responseFormat.fields.map((f) => f.name),
+        },
+        null,
+        2
+      )}\n\nExample response format:\n{\n${request.responseFormat.fields
+        .map(
+          (f) =>
+            `  "${f.name}": ${
+              f.type === 'string'
+                ? '"value"'
+                : f.type === 'number'
+                  ? '0'
+                  : f.type === 'boolean'
+                    ? 'true'
+                    : f.type === 'array'
+                      ? '[]'
+                      : '{}'
+            }`
+        )
+        .join(',\n')}\n}`
+    }
+
    const payload = {
      model: request.model || 'grok-2-latest',
      messages: [
-        { role: 'system', content: request.systemPrompt },
+        { role: 'system', content: systemPrompt },
        ...(request.context ? [{ role: 'user', content: request.context }] : []),
        ...messages,
      ],
@@ -107,6 +150,45 @@ export const xAIProvider: ProviderConfig = {
        tools: functions,
        tool_choice: 'auto', // xAI specific parameter
      }),
+      ...(request.responseFormat && {
+        response_format: {
+          type: 'json_schema',
+          json_schema: {
+            name: 'structured_response',
+            schema: {
+              type: 'object',
+              properties: request.responseFormat.fields.reduce(
+                (acc, field) => ({
+                  ...acc,
+                  [field.name]: {
+                    type:
+                      field.type === 'array'
+                        ? 'array'
+                        : field.type === 'object'
+                          ? 'object'
+                          : field.type === 'number'
+                            ? 'number'
+                            : field.type === 'boolean'
+                              ? 'boolean'
+                              : 'string',
+                    description: field.description || '',
+                    ...(field.type === 'array' && {
+                      items: { type: 'string' },
+                    }),
+                    ...(field.type === 'object' && {
+                      additionalProperties: true,
+                    }),
+                  },
+                }),
+                {}
+              ),
+              required: request.responseFormat.fields.map((f) => f.name),
+              additionalProperties: false,
+            },
+            strict: true,
+          },
+        },
+      }),
    }

    return payload
@@ -39,12 +39,52 @@ export const useWorkflowStore = create<WorkflowStoreWithHistory>()(
          const blockConfig = getBlock(block.type)
          if (!blockConfig) return state

-          // Create new subBlocks state
+          // Validate responseFormat if it's the agent block's responseFormat input
+          if (blockConfig.type === 'agent' && subBlockId === 'responseFormat' && value) {
+            console.log('Validating responseFormat input:', {
+              type: typeof value,
+              rawValue: value,
+            })
+
+            try {
+              // Parse the input string to validate JSON but keep original string value
+              const parsed = JSON.parse(value)
+              console.log('Parsed responseFormat:', parsed)
+
+              // Simple validation of required schema structure
+              if (!parsed.fields || !Array.isArray(parsed.fields)) {
+                console.error('Validation failed: missing fields array')
+                throw new Error('Response format must have a fields array')
+              }
+
+              for (const field of parsed.fields) {
+                console.log('Validating field:', field)
+                if (!field.name || !field.type) {
+                  console.error('Validation failed: field missing name or type', field)
+                  throw new Error('Each field must have a name and type')
+                }
+                if (!['string', 'number', 'boolean', 'array', 'object'].includes(field.type)) {
+                  console.error('Validation failed: invalid field type', field)
+                  throw new Error(
+                    `Invalid type "${field.type}" - must be one of: string, number, boolean, array, object`
+                  )
+                }
+              }
+
+              console.log('responseFormat validation successful')
+              // Don't modify the value, keep it as the original string
+            } catch (error: any) {
+              console.error('responseFormat validation error:', error)
+              throw new Error(`Invalid JSON schema: ${error.message}`)
+            }
+          }
+
+          // Create new subBlocks state with the original value
          const newSubBlocks = {
            ...block.subBlocks,
            [subBlockId]: {
              ...block.subBlocks[subBlockId],
-              value,
+              value: typeof value === 'string' ? value : JSON.stringify(value, null, 2),
            },
          }