improvement(api): add default headers for API requests, user can override (#275)

sim/blocks/blocks/api.ts

@@ -7,7 +7,7 @@ export const ApiBlock: BlockConfig<RequestResponse> = {
   name: 'API',
   description: 'Use any API',
   longDescription:
-    'Connect to any external API with support for all standard HTTP methods and customizable request parameters. Configure headers, query parameters, and request bodies.',
+    'Connect to any external API with support for all standard HTTP methods and customizable request parameters. Configure headers, query parameters, and request bodies. Standard headers (User-Agent, Accept, Cache-Control, etc.) are automatically included.',
   category: 'blocks',
   bgColor: '#2F55FF',
   icon: ApiIcon,
@@ -39,6 +39,7 @@ export const ApiBlock: BlockConfig<RequestResponse> = {
       type: 'table',
       layout: 'full',
       columns: ['Key', 'Value'],
+      description: 'Custom headers (standard headers like User-Agent, Accept, etc. are added automatically)',
     },
     {
       id: 'body',

sim/tools/http/request.test.ts

@@ -106,9 +106,102 @@ describe('HTTP Request Tool', () => {
         'Content-Type': 'application/json',
       })
     })
+
+    test('should set dynamic Referer header correctly', async () => {
+      const originalWindow = global.window
+      Object.defineProperty(global, 'window', {
+        value: {
+          location: {
+            origin: 'https://app.simstudio.dev'
+          }
+        },
+        writable: true
+      })
+
+      // Setup mock response
+      tester.setup(mockHttpResponses.simple)
+
+      // Execute with real request to check Referer header
+      await tester.execute({
+        url: 'https://api.example.com',
+        method: 'GET'
+      })
+
+      // Verify the Referer header was set
+      const fetchCall = (global.fetch as any).mock.calls[0]
+      expect(fetchCall[1].headers.Referer).toBe('https://app.simstudio.dev')
+
+      // Reset window
+      global.window = originalWindow
+    })
+
+    test('should set dynamic Host header correctly', async () => {
+      // Setup mock response
+      tester.setup(mockHttpResponses.simple)
+
+      // Execute with real request to check Host header
+      await tester.execute({
+        url: 'https://api.example.com/endpoint',
+        method: 'GET'
+      })
+
+      // Verify the Host header was set
+      const fetchCall = (global.fetch as any).mock.calls[0]
+      expect(fetchCall[1].headers.Host).toBe('api.example.com')
+      
+      // Test user-provided Host takes precedence
+      await tester.execute({
+        url: 'https://api.example.com/endpoint',
+        method: 'GET',
+        headers: [{ cells: { Key: 'Host', Value: 'custom-host.com' } }]
+      })
+      
+      // Verify the user's Host was used
+      const userHeaderCall = (global.fetch as any).mock.calls[1]
+      expect(userHeaderCall[1].headers.Host).toBe('custom-host.com')
+    })
   })
 
   describe('Request Execution', () => {
+    test('should apply default and dynamic headers to requests', async () => {
+      // Setup mock response
+      tester.setup(mockHttpResponses.simple)
+
+      // Set up browser-like environment
+      const originalWindow = global.window
+      Object.defineProperty(global, 'window', {
+        value: {
+          location: {
+            origin: 'https://app.simstudio.dev'
+          }
+        },
+        writable: true
+      })
+
+      // Execute the tool with method explicitly set to GET
+      await tester.execute({
+        url: 'https://api.example.com/data',
+        method: 'GET',
+      })
+
+      // Verify fetch was called with expected headers
+      const fetchCall = (global.fetch as any).mock.calls[0]
+      const headers = fetchCall[1].headers
+
+      // Check specific header values
+      expect(headers['Host']).toBe('api.example.com')
+      expect(headers['Referer']).toBe('https://app.simstudio.dev')
+      expect(headers['User-Agent']).toContain('Mozilla')
+      expect(headers['Accept']).toBe('*/*')
+      expect(headers['Accept-Encoding']).toContain('gzip')
+      expect(headers['Cache-Control']).toBe('no-cache')
+      expect(headers['Connection']).toBe('keep-alive')
+      expect(headers['Sec-Ch-Ua']).toContain('Chromium')
+
+      // Reset window
+      global.window = originalWindow
+    })
+
     test('should handle successful GET requests', async () => {
       // Setup mock response
       tester.setup(mockHttpResponses.simple)
@@ -263,4 +356,74 @@ describe('HTTP Request Tool', () => {
       expect(result.output).toEqual({})
     })
   })
+
+  describe('Default Headers', () => {
+    test('should apply all default headers correctly', async () => {
+      // Setup mock response
+      tester.setup(mockHttpResponses.simple)
+
+      // Set up browser-like environment
+      const originalWindow = global.window
+      Object.defineProperty(global, 'window', {
+        value: {
+          location: {
+            origin: 'https://app.simstudio.dev'
+          }
+        },
+        writable: true
+      })
+
+      // Execute the tool
+      await tester.execute({
+        url: 'https://api.example.com/data',
+        method: 'GET',
+      })
+
+      // Get the headers from the fetch call
+      const fetchCall = (global.fetch as any).mock.calls[0]
+      const headers = fetchCall[1].headers
+
+      // Check all default headers exist with expected values
+      expect(headers['User-Agent']).toMatch(/Mozilla\/5\.0.*Chrome.*Safari/)
+      expect(headers['Accept']).toBe('*/*')
+      expect(headers['Accept-Encoding']).toBe('gzip, deflate, br')
+      expect(headers['Cache-Control']).toBe('no-cache')
+      expect(headers['Connection']).toBe('keep-alive')
+      expect(headers['Sec-Ch-Ua']).toMatch(/Chromium.*Not-A\.Brand/)
+      expect(headers['Sec-Ch-Ua-Mobile']).toBe('?0')
+      expect(headers['Sec-Ch-Ua-Platform']).toBe('"macOS"')
+      expect(headers['Referer']).toBe('https://app.simstudio.dev')
+      expect(headers['Host']).toBe('api.example.com')
+
+      // Reset window
+      global.window = originalWindow
+    })
+
+    test('should allow overriding default headers', async () => {
+      // Setup mock response
+      tester.setup(mockHttpResponses.simple)
+
+      // Execute with custom headers that override defaults
+      await tester.execute({
+        url: 'https://api.example.com/data',
+        method: 'GET',
+        headers: [
+          { cells: { Key: 'User-Agent', Value: 'Custom Agent' } },
+          { cells: { Key: 'Accept', Value: 'application/json' } }
+        ]
+      })
+
+      // Get the headers from the fetch call
+      const fetchCall = (global.fetch as any).mock.calls[0]
+      const headers = fetchCall[1].headers
+
+      // Verify overridden headers
+      expect(headers['User-Agent']).toBe('Custom Agent')
+      expect(headers['Accept']).toBe('application/json')
+      
+      // Verify other default headers still exist
+      expect(headers['Accept-Encoding']).toBe('gzip, deflate, br')
+      expect(headers['Cache-Control']).toBe('no-cache')
+    })
+  })
 })

sim/tools/http/request.ts

@@ -1,5 +1,29 @@
 import { HttpMethod, TableRow, ToolConfig, ToolResponse } from '../types'
 
+const getReferer = (): string => {
+  if (typeof window !== 'undefined') {
+    return window.location.origin
+  }
+  
+  if (process.env.NEXT_PUBLIC_APP_URL) {
+    return process.env.NEXT_PUBLIC_APP_URL
+  }
+  
+  return 'http://localhost:3000/'
+}
+
+// Default headers that will be applied if not explicitly overridden by user
+const DEFAULT_HEADERS: Record<string, string> = {
+  'User-Agent': 'Mozilla/5.0 (Macintosh Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
+  'Accept': '*/*',
+  'Accept-Encoding': 'gzip, deflate, br',
+  'Cache-Control': 'no-cache',
+  'Connection': 'keep-alive',
+  'Sec-Ch-Ua': '"Chromium"v="135", "Not-A.Brand"v="8"',
+  'Sec-Ch-Ua-Mobile': '?0',
+  'Sec-Ch-Ua-Platform': '"macOS"'
+}
+
 /**
  * Transforms a table from the store format to a key-value object
  * Local copy of the function to break circular dependencies
@@ -97,10 +121,44 @@ export const requestTool: ToolConfig<RequestParams, RequestResponse> = {
   // Direct execution to bypass server for HTTP requests
   directExecution: async (params: RequestParams): Promise<RequestResponse | undefined> => {
     try {
+      // Process URL first to extract hostname for Host header
+      let url = params.url
+      // Strip any surrounding quotes
+      if (typeof url === 'string') {
+        if (
+          (url.startsWith('"') && url.endsWith('"')) ||
+          (url.startsWith("'") && url.endsWith("'"))
+        ) {
+          url = url.slice(1, -1)
+          params.url = url
+        }
+      }
+
+      // Extract hostname for Host header
+      let hostname = ''
+      try {
+        hostname = new URL(url).host
+      } catch (e) {
+        // Invalid URL, will be caught later
+      }
+      
       // Prepare fetch options
+      const userHeaders = transformTable(params.headers || null)
+      const headers = { ...DEFAULT_HEADERS, ...userHeaders }
+      
+      // Add Host header if not explicitly set by user
+      if (hostname && !userHeaders['Host'] && !userHeaders['host']) {
+        headers['Host'] = hostname
+      }
+      
+      // Set dynamic Referer if not explicitly provided by user
+      if (!userHeaders['Referer'] && !userHeaders['referer']) {
+        headers['Referer'] = getReferer()
+      }
+      
       const fetchOptions: RequestInit = {
         method: params.method || 'GET',
-        headers: transformTable(params.headers || null),
+        headers
       }
 
       // Add body for non-GET requests
@@ -109,7 +167,7 @@ export const requestTool: ToolConfig<RequestParams, RequestResponse> = {
           fetchOptions.body = JSON.stringify(params.body)
           // Ensure Content-Type is set
           if (fetchOptions.headers) {
-            ;(fetchOptions.headers as Record<string, string>)['Content-Type'] = 'application/json'
+            (fetchOptions.headers as Record<string, string>)['Content-Type'] = 'application/json'
           } else {
             fetchOptions.headers = { 'Content-Type': 'application/json' }
           }
@@ -135,19 +193,7 @@ export const requestTool: ToolConfig<RequestParams, RequestResponse> = {
 
       try {
         // Process URL with path parameters and query params
-        let url = params.url
 
-        // Strip any surrounding quotes that might have been added during resolution
-        if (typeof url === 'string') {
-          if (
-            (url.startsWith('"') && url.endsWith('"')) ||
-            (url.startsWith("'") && url.endsWith("'"))
-          ) {
-            url = url.slice(1, -1)
-            // Update the params with unquoted URL
-            params.url = url
-          }
-        }
 
         // Replace path parameters
         if (params.pathParams) {
@@ -276,16 +322,34 @@ export const requestTool: ToolConfig<RequestParams, RequestResponse> = {
     method: 'POST' as HttpMethod,
     headers: (params: RequestParams) => {
       const headers = transformTable(params.headers || null)
+      
+      // Merge with default headers
+      const allHeaders = { ...DEFAULT_HEADERS, ...headers }
+      
+      // Add dynamic Host header if not explicitly set
+      try {
+        const hostname = new URL(params.url).host
+        if (hostname && !headers['Host'] && !headers['host']) {
+          allHeaders['Host'] = hostname
+        }
+      } catch (e) {
+        // Invalid URL, will be handled elsewhere
+      }
+
+      // Add dynamic Referer if not explicitly set
+      if (!headers['Referer'] && !headers['referer']) {
+        allHeaders['Referer'] = getReferer()
+      }
 
       // Set appropriate Content-Type
       if (params.formData) {
         // Don't set Content-Type for FormData, browser will set it with boundary
-        return headers
+        return allHeaders
       } else if (params.body) {
-        headers['Content-Type'] = 'application/json'
+        allHeaders['Content-Type'] = 'application/json'
       }
 
-      return headers
+      return allHeaders
     },
     body: (params: RequestParams) => {
       if (params.formData) {