fix(knowledge): harden ingestion pipeline (#7077)

* fix(knowledge): harden ingestion pipeline

* fix(knowledge): correct bounded ingestion edge cases

* fix(knowledge): tighten redaction and chunk validation

* fix(knowledge): close bounded ingestion gaps

* fix(files): preserve parser complexity limits

apps/docs/openapi-v2-knowledge.json

@@ -5278,6 +5278,13 @@
             "maximum": 9007199254740991,
             "description": "Documents unchanged."
           },
+          "docsSkipped": {
+            "default": 0,
+            "description": "Documents intentionally skipped because they could not be indexed safely.",
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 9007199254740991
+          },
           "docsFailed": {
             "type": "integer",
             "minimum": 0,
@@ -5306,6 +5313,7 @@
           "docsUpdated",
           "docsDeleted",
           "docsUnchanged",
+          "docsSkipped",
           "docsFailed",
           "errorMessage"
         ],

apps/sim/app/api/files/parse/route.test.ts

@@ -16,6 +16,7 @@ import {
 } from '@sim/testing'
 import { NextRequest } from 'next/server'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { FileParserError } from '@/lib/file-parsers/errors'
 
 const {
   mockVerifyFileAccess,
@@ -310,6 +311,30 @@ describe('File Parse API Route', () => {
     expect(data.output.content).toBe('plain text content')
   })
 
+  it('should reject parser complexity limits instead of returning raw text', async () => {
+    setupFileApiMocks({
+      cloudEnabled: true,
+      storageProvider: 's3',
+      authenticated: true,
+    })
+    storageServiceMockFns.mockDownloadFile.mockResolvedValue(Buffer.from('{"value":true}'))
+    mockParseBuffer.mockRejectedValueOnce(
+      new FileParserError('complexity_limit', 'JSON document exceeds the complexity limit')
+    )
+
+    const req = createMockRequest('POST', {
+      filePath: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/data.json',
+    })
+
+    const response = await POST(req)
+    const data = await response.json()
+
+    expect(response.status).toBe(200)
+    expect(data.success).toBe(false)
+    expect(data.error).toContain('complexity limit')
+    expect(data).not.toHaveProperty('output')
+  })
+
   it('should handle multiple files', async () => {
     setupFileApiMocks({
       cloudEnabled: false,

apps/sim/app/api/files/parse/route.ts

@@ -13,8 +13,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
 import { sanitizeUrlForLog } from '@/lib/core/utils/logging'
 import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { isSupportedFileType, parseFile } from '@/lib/file-parsers'
-import { isHtmlComplexityError } from '@/lib/file-parsers/html-parser'
-import { isYamlComplexityError } from '@/lib/file-parsers/yaml-parser'
+import { isFileParserError } from '@/lib/file-parsers/errors'
 import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
 import { uploadExecutionFile } from '@/lib/uploads/contexts/execution'
 import {
@@ -1050,10 +1049,9 @@ async function handleGenericTextBuffer(
       }
     } catch (parserError) {
       if (isPayloadSizeLimitError(parserError)) throw parserError
-      // Fail closed on a resource-exhaustion rejection instead of silently
-      // storing the crafted document as raw text.
-      if (isYamlComplexityError(parserError)) throw parserError
-      if (isHtmlComplexityError(parserError)) throw parserError
+      if (isFileParserError(parserError) && parserError.code === 'complexity_limit') {
+        throw parserError
+      }
 
       logger.warn('Specialized parser failed, falling back to generic parsing:', parserError)
     }

apps/sim/app/api/knowledge/search/utils.test.ts

@@ -89,6 +89,20 @@ function makeResult(id: string, distance = 0.1): SearchResult {
   }
 }
 
+const TEST_EMBEDDING = [0.1, 0.2, 0.3, ...Array.from({ length: 1533 }, () => 0)]
+
+function mockNextEmbeddingResponse(): void {
+  vi.mocked(fetch).mockResolvedValueOnce(
+    new Response(
+      JSON.stringify({
+        data: [{ embedding: TEST_EMBEDDING, index: 0 }],
+        usage: { prompt_tokens: 1, total_tokens: 1 },
+      }),
+      { status: 200, headers: { 'Content-Type': 'application/json' } }
+    )
+  )
+}
+
 describe('Knowledge Search Utils', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -553,12 +567,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       const result = await generateSearchEmbedding('test query')
 
@@ -570,7 +579,7 @@ describe('Knowledge Search Utils', () => {
           }),
         })
       )
-      expect(result.embedding).toEqual([0.1, 0.2, 0.3])
+      expect(result.embedding).toEqual(TEST_EMBEDDING)
 
       // Clean up
       Object.keys(env).forEach((key) => delete (env as any)[key])
@@ -583,12 +592,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       const result = await generateSearchEmbedding('test query')
 
@@ -600,7 +604,7 @@ describe('Knowledge Search Utils', () => {
           }),
         })
       )
-      expect(result.embedding).toEqual([0.1, 0.2, 0.3])
+      expect(result.embedding).toEqual(TEST_EMBEDDING)
 
       // Clean up
       Object.keys(env).forEach((key) => delete (env as any)[key])
@@ -616,12 +620,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query')
 
@@ -645,12 +644,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
@@ -733,12 +727,7 @@ describe('Knowledge Search Utils', () => {
         KB_OPENAI_MODEL_NAME: 'text-embedding-ada-002',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query')
 
@@ -764,12 +753,7 @@ describe('Knowledge Search Utils', () => {
         OPENAI_API_KEY: 'test-openai-key',
       })
 
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       await generateSearchEmbedding('test query', 'text-embedding-3-small')
 
@@ -792,12 +776,7 @@ describe('Knowledge Search Utils', () => {
     it('projects verified provenance only in the model-bound embedding payload', async () => {
       Object.keys(env).forEach((key) => delete (env as any)[key])
       Object.assign(env, { OPENAI_API_KEY: 'test-openai-key' })
-      mockNextFetchResponse({
-        json: {
-          data: [{ embedding: [0.1, 0.2, 0.3] }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        },
-      })
+      mockNextEmbeddingResponse()
 
       const registry = new ResolvedSecretTraceRegistry([
         { name: 'TOKEN', plaintext: 'secret-value', encryptedValue: 'encrypted-token' },

apps/sim/app/api/knowledge/utils.test.ts

@@ -110,17 +110,24 @@ vi.mock('@/lib/knowledge/documents/document-processor', () => ({
   }),
 }))
 
-function createEmbeddingFetchMock() {
-  return vi.fn().mockResolvedValue({
-    ok: true,
-    json: async () => ({
-      data: [
-        { embedding: [0.1, 0.2], index: 0 },
-        { embedding: [0.3, 0.4], index: 1 },
-      ],
-      usage: { prompt_tokens: 2, total_tokens: 2 },
+const TEST_EMBEDDING_DIMENSION = 1536
+
+function createTestEmbedding(value: number): number[] {
+  return Array.from({ length: TEST_EMBEDDING_DIMENSION }, () => value)
+}
+
+function createEmbeddingResponse(values: number[]): Response {
+  return new Response(
+    JSON.stringify({
+      data: values.map((value, index) => ({ embedding: createTestEmbedding(value), index })),
+      usage: { prompt_tokens: values.length, total_tokens: values.length },
     }),
-  })
+    { status: 200, headers: { 'Content-Type': 'application/json' } }
+  )
+}
+
+function createEmbeddingFetchMock() {
+  return vi.fn().mockResolvedValue(createEmbeddingResponse([0.1, 0.3]))
 }
 
 vi.stubGlobal('fetch', createEmbeddingFetchMock())
@@ -166,6 +173,10 @@ describe('Knowledge Utils', () => {
           embeddingModel: 'text-embedding-3-small',
           billedAccountUserId: 'billing-user-1',
           uploadedBy: null,
+          filename: 'file.txt',
+          fileUrl: 'https://example.com/file.txt',
+          fileSize: 10,
+          mimeType: 'text/plain',
         },
       ])
       /** Legacy untracked documents have exact-empty provenance. */
@@ -301,13 +312,7 @@ describe('Knowledge Utils', () => {
       })
 
       const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
-          data: [{ embedding: [0.1, 0.2], index: 0 }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        }),
-      } as any)
+      fetchSpy.mockResolvedValueOnce(createEmbeddingResponse([0.1]))
 
       await generateEmbeddings(['test text'])
 
@@ -331,13 +336,7 @@ describe('Knowledge Utils', () => {
       })
 
       const fetchSpy = vi.mocked(fetch)
-      fetchSpy.mockResolvedValueOnce({
-        ok: true,
-        json: async () => ({
-          data: [{ embedding: [0.1, 0.2], index: 0 }],
-          usage: { prompt_tokens: 1, total_tokens: 1 },
-        }),
-      } as any)
+      fetchSpy.mockResolvedValueOnce(createEmbeddingResponse([0.1]))
 
       await generateEmbeddings(['test text'])
 

apps/sim/app/api/tools/embeddings/route.test.ts

@@ -4,13 +4,26 @@
 import { createMockRequest, hybridAuthMockFns } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockEmbed, mockEmbedOpenRouter, mockGetOpenRouterEmbeddingModelMetadata } = vi.hoisted(
-  () => ({
+const {
+  MockEmbeddingOutputLimitError,
+  mockEmbed,
+  mockEmbedOpenRouter,
+  mockGetOpenRouterEmbeddingModelMetadata,
+} = vi.hoisted(() => {
+  class MockEmbeddingOutputLimitError extends Error {
+    constructor(message: string) {
+      super(message)
+      this.name = 'EmbeddingOutputLimitError'
+    }
+  }
+
+  return {
+    MockEmbeddingOutputLimitError,
     mockEmbed: vi.fn(),
     mockEmbedOpenRouter: vi.fn(),
     mockGetOpenRouterEmbeddingModelMetadata: vi.fn(),
-  })
-)
+  }
+})
 
 vi.mock('@/lib/embeddings/openrouter-model-catalog.server', () => ({
   getOpenRouterEmbeddingModelMetadata: mockGetOpenRouterEmbeddingModelMetadata,
@@ -27,6 +40,7 @@ vi.mock('@/lib/embeddings', async () => {
   return {
     embed: mockEmbed,
     embedOpenRouter: mockEmbedOpenRouter,
+    EmbeddingOutputLimitError: MockEmbeddingOutputLimitError,
     DEFAULT_OPENROUTER_EMBEDDING_MODEL: 'openrouter/openai/text-embedding-3-small',
     findEmbeddingModelInfo: catalog.findEmbeddingModelInfo,
     getModelsForProvider: catalog.getModelsForProvider,
@@ -230,6 +244,15 @@ describe('POST /api/tools/embeddings', () => {
     expect((await response.json()).error).toContain('429')
   })
 
+  it('returns 413 when the requested embedding output exceeds the safe aggregate limit', async () => {
+    mockEmbed.mockRejectedValue(
+      new MockEmbeddingOutputLimitError('Embedding output exceeds the safe aggregate limit')
+    )
+    const response = await post(baseBody)
+    expect(response.status).toBe(413)
+    expect((await response.json()).error).toContain('safe aggregate limit')
+  })
+
   it('splits a JSON-array input into separate texts', async () => {
     await post({ ...baseBody, input: '["alpha","beta"]' })
     expect(mockEmbed).toHaveBeenCalledWith(['alpha', 'beta'], expect.anything())

apps/sim/app/api/tools/embeddings/route.ts

@@ -12,6 +12,7 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
   DEFAULT_MODEL_BY_PROVIDER,
   DEFAULT_OPENROUTER_EMBEDDING_MODEL,
+  EmbeddingOutputLimitError,
   embed,
   embedOpenRouter,
   findEmbeddingModelInfo,
@@ -227,6 +228,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     })
   } catch (error) {
     const message = getErrorMessage(error, 'Embedding generation failed')
+    if (error instanceof EmbeddingOutputLimitError) {
+      logger.warn('Embedding output exceeds safe limit', { error: message })
+      return NextResponse.json({ success: false, error: message }, { status: 413 })
+    }
     logger.error('Embedding generation failed', { error: message })
     return NextResponse.json({ success: false, error: message }, { status: 502 })
   }

apps/sim/app/api/v2/knowledge/[knowledgeBaseId]/connectors/route.test.ts

@@ -260,6 +260,45 @@ describe('v2 knowledge connector routes', () => {
     expect(mocks.connectorRemoved).toHaveBeenCalledOnce()
   })
 
+  it('serializes skipped-document counts in non-empty sync history', async () => {
+    mocks.read.mockResolvedValueOnce({
+      connector: {
+        ...connector,
+        syncLogs: [
+          {
+            id: 'sync-log-1',
+            connectorId: CONNECTOR_ID,
+            status: 'completed',
+            startedAt: new Date('2026-01-03T00:00:00Z'),
+            completedAt: new Date('2026-01-03T00:01:00Z'),
+            docsAdded: 1,
+            docsUpdated: 2,
+            docsDeleted: 3,
+            docsUnchanged: 4,
+            docsSkipped: 5,
+            docsFailed: 6,
+            errorMessage: null,
+          },
+        ],
+      },
+    })
+
+    const response = await getConnector(
+      request(
+        `/api/v2/knowledge/${KNOWLEDGE_BASE_ID}/connectors/${CONNECTOR_ID}?workspaceId=${WORKSPACE_ID}`
+      ),
+      connectorContext
+    )
+
+    expect(response.status).toBe(200)
+    expect((await response.json()).data.syncLogs[0]).toMatchObject({
+      id: 'sync-log-1',
+      docsSkipped: 5,
+      startedAt: '2026-01-03T00:00:00.000Z',
+      completedAt: '2026-01-03T00:01:00.000Z',
+    })
+  })
+
   it('passes source changes to application billing without an adapter resolver', async () => {
     const response = await updateConnector(
       request(`/api/v2/knowledge/${KNOWLEDGE_BASE_ID}/connectors/${CONNECTOR_ID}`, 'PATCH', {

apps/sim/app/api/v2/knowledge/connector-utils.ts

@@ -44,6 +44,7 @@ interface KnowledgeConnectorSyncLogProjection {
   docsUpdated: number
   docsDeleted: number
   docsUnchanged: number
+  docsSkipped: number
   docsFailed: number
   errorMessage: string | null
 }

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.test.tsx

@@ -136,6 +136,7 @@ function makeLog(overrides: Partial<SyncLogData> & Pick<SyncLogData, 'status'>):
     docsUpdated: 0,
     docsDeleted: 0,
     docsUnchanged: 0,
+    docsSkipped: 0,
     docsFailed: 0,
     errorMessage: null,
     ...overrides,
@@ -356,6 +357,30 @@ describe('SyncHistory', () => {
     expect(container.textContent).toContain('No changes')
   })
 
+  it('renders a skipped-only completed row as a change', () => {
+    const container = render(makeLog({ status: 'completed', docsSkipped: 4 }))
+
+    expect(icons(container)).toEqual(['icon-circle-check'])
+    expect(container.textContent).toContain('⊘4')
+    expect(container.textContent).not.toContain('No changes')
+  })
+
+  it('renders mixed sync counts as separate ordered markers', () => {
+    const container = render(
+      makeLog({
+        status: 'completed',
+        docsAdded: 2,
+        docsUpdated: 3,
+        docsDeleted: 4,
+        docsFailed: 5,
+        docsSkipped: 6,
+      })
+    )
+
+    expect(container.textContent).toContain('+2 ~3 -4 !5 ⊘6')
+    expect(container.textContent).not.toContain('No changes')
+  })
+
   it('renders a "failed" row as an error with its message', () => {
     const container = render(makeLog({ status: 'failed', errorMessage: 'token expired' }))
 

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx

@@ -716,7 +716,11 @@ export function SyncHistory({ logs, isLoading }: SyncHistoryProps) {
       {logs.map((log) => {
         const state = getSyncLogState(log, now)
         const totalChanges =
-          log.docsAdded + log.docsUpdated + log.docsDeleted + (log.docsFailed ?? 0)
+          log.docsAdded +
+          log.docsUpdated +
+          log.docsDeleted +
+          log.docsSkipped +
+          (log.docsFailed ?? 0)
 
         return (
           <div key={log.id} className='flex items-start gap-2 rounded-md px-2 py-1.5 text-xs'>
@@ -763,6 +767,16 @@ export function SyncHistory({ logs, isLoading }: SyncHistoryProps) {
                             <span className='text-[var(--text-error)]'>!{log.docsFailed}</span>
                           </>
                         )}
+                        {log.docsSkipped > 0 && (
+                          <>
+                            {(log.docsAdded > 0 ||
+                              log.docsUpdated > 0 ||
+                              log.docsDeleted > 0 ||
+                              log.docsFailed > 0) &&
+                              ' '}
+                            <span className='text-[var(--caution)]'>⊘{log.docsSkipped}</span>
+                          </>
+                        )}
                       </>
                     ) : (
                       'No changes'

apps/sim/background/knowledge-connector-sync.test.ts

@@ -1,6 +1,8 @@
 /**
  * @vitest-environment node
  */
+
+import { AbortTaskRunError } from '@trigger.dev/sdk'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const { mockAssertConnectorSyncPayload, mockExecuteSync, mockTask } = vi.hoisted(() => ({
@@ -9,7 +11,10 @@ const { mockAssertConnectorSyncPayload, mockExecuteSync, mockTask } = vi.hoisted
   mockTask: vi.fn((config) => config),
 }))
 
-vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
+vi.mock('@trigger.dev/sdk', () => ({
+  task: mockTask,
+  AbortTaskRunError: class AbortTaskRunError extends Error {},
+}))
 vi.mock('@/lib/knowledge/connectors/queue', () => ({
   assertConnectorSyncPayload: mockAssertConnectorSyncPayload,
 }))
@@ -17,7 +22,10 @@ vi.mock('@/lib/knowledge/connectors/sync-engine', () => ({
   executeSync: mockExecuteSync,
 }))
 
-import { executeConnectorSyncJob } from '@/background/knowledge-connector-sync'
+import {
+  classifyConnectorSyncResult,
+  executeConnectorSyncJob,
+} from '@/background/knowledge-connector-sync'
 
 const BILLING_ATTRIBUTION = {
   actorUserId: 'external-admin',
@@ -40,7 +48,9 @@ describe('knowledge connector sync worker', () => {
       docsUpdated: 0,
       docsDeleted: 0,
       docsUnchanged: 0,
+      docsSkipped: 0,
       docsFailed: 0,
+      processingDispatch: { requested: 0, accepted: 0, failed: 0 },
     })
   })
 
@@ -103,4 +113,164 @@ describe('knowledge connector sync worker', () => {
       dispatchToken: undefined,
     })
   })
+
+  it('fails visibly without retrying an already-persisted partial sync', async () => {
+    mockAssertConnectorSyncPayload.mockReturnValue({
+      connectorId: 'connector-1',
+      requestId: 'request-1',
+      billingAttribution: BILLING_ATTRIBUTION,
+    })
+    mockExecuteSync.mockResolvedValue({
+      docsAdded: 2,
+      docsUpdated: 0,
+      docsDeleted: 0,
+      docsUnchanged: 0,
+      docsSkipped: 3,
+      docsFailed: 1,
+      processingDispatch: { requested: 2, accepted: 1, failed: 1 },
+    })
+
+    const run = executeConnectorSyncJob({
+      connectorId: 'connector-1',
+      requestId: 'request-1',
+      billingAttribution: BILLING_ATTRIBUTION,
+    })
+
+    await expect(run).rejects.toBeInstanceOf(AbortTaskRunError)
+    await expect(run).rejects.toThrow('Connector sync partially failed')
+  })
+
+  it('does not turn intentionally skipped source files into a task failure', () => {
+    expect(
+      classifyConnectorSyncResult({
+        docsAdded: 0,
+        docsUpdated: 0,
+        docsDeleted: 0,
+        docsUnchanged: 0,
+        docsSkipped: 4,
+        docsFailed: 0,
+        processingDispatch: { requested: 0, accepted: 0, failed: 0 },
+      })
+    ).toBe('completed')
+  })
+
+  it('classifies an isolated processing dispatch failure as partial', () => {
+    expect(
+      classifyConnectorSyncResult({
+        docsAdded: 1,
+        docsUpdated: 0,
+        docsDeleted: 0,
+        docsUnchanged: 0,
+        docsSkipped: 0,
+        docsFailed: 0,
+        processingDispatch: { requested: 1, accepted: 0, failed: 1 },
+      })
+    ).toBe('partial')
+  })
+
+  it('reports superseded and non-runnable jobs as skipped control flow', () => {
+    const baseResult = {
+      docsAdded: 0,
+      docsUpdated: 0,
+      docsDeleted: 0,
+      docsUnchanged: 0,
+      docsSkipped: 0,
+      docsFailed: 0,
+      processingDispatch: { requested: 0, accepted: 0, failed: 0 },
+    }
+
+    expect(classifyConnectorSyncResult({ ...baseResult, skipReason: 'sync_superseded' })).toBe(
+      'skipped'
+    )
+    expect(
+      classifyConnectorSyncResult({ ...baseResult, skipReason: 'connector_not_syncable' })
+    ).toBe('skipped')
+  })
+
+  it('never treats a provider error that collides with a skip reason as control flow', () => {
+    expect(
+      classifyConnectorSyncResult({
+        docsAdded: 0,
+        docsUpdated: 0,
+        docsDeleted: 0,
+        docsUnchanged: 0,
+        docsSkipped: 0,
+        docsFailed: 0,
+        processingDispatch: { requested: 0, accepted: 0, failed: 0 },
+        error: 'sync_in_progress',
+      })
+    ).toBe('failed')
+  })
+
+  it('classifies a persisted connector error as a failed task', () => {
+    expect(
+      classifyConnectorSyncResult({
+        docsAdded: 0,
+        docsUpdated: 0,
+        docsDeleted: 0,
+        docsUnchanged: 0,
+        docsSkipped: 0,
+        docsFailed: 0,
+        processingDispatch: { requested: 0, accepted: 0, failed: 0 },
+        error: 'provider unavailable',
+      })
+    ).toBe('failed')
+  })
+
+  it('fails the Trigger run for a persisted connector error without a whole-sync retry', async () => {
+    mockAssertConnectorSyncPayload.mockReturnValue({
+      connectorId: 'connector-1',
+      requestId: 'request-1',
+      billingAttribution: BILLING_ATTRIBUTION,
+    })
+    mockExecuteSync.mockResolvedValue({
+      docsAdded: 0,
+      docsUpdated: 0,
+      docsDeleted: 0,
+      docsUnchanged: 0,
+      docsSkipped: 0,
+      docsFailed: 0,
+      processingDispatch: { requested: 0, accepted: 0, failed: 0 },
+      error: 'provider unavailable',
+    })
+
+    const run = executeConnectorSyncJob({
+      connectorId: 'connector-1',
+      requestId: 'request-1',
+      billingAttribution: BILLING_ATTRIBUTION,
+    })
+
+    await expect(run).rejects.toBeInstanceOf(AbortTaskRunError)
+    await expect(run).rejects.toThrow('Connector sync failed for connector-1: provider unavailable')
+  })
+
+  it('returns an explicit skipped outcome for a superseded task', async () => {
+    mockAssertConnectorSyncPayload.mockReturnValue({
+      connectorId: 'connector-1',
+      requestId: 'request-1',
+      billingAttribution: BILLING_ATTRIBUTION,
+    })
+    mockExecuteSync.mockResolvedValue({
+      docsAdded: 0,
+      docsUpdated: 0,
+      docsDeleted: 0,
+      docsUnchanged: 0,
+      docsSkipped: 0,
+      docsFailed: 0,
+      processingDispatch: { requested: 0, accepted: 0, failed: 0 },
+      skipReason: 'dispatch_superseded',
+    })
+
+    await expect(
+      executeConnectorSyncJob({
+        connectorId: 'connector-1',
+        requestId: 'request-1',
+        billingAttribution: BILLING_ATTRIBUTION,
+      })
+    ).resolves.toMatchObject({
+      success: false,
+      outcome: 'skipped',
+      skipReason: 'dispatch_superseded',
+    })
+  })
 })

apps/sim/background/knowledge-connector-sync.ts

@@ -1,14 +1,40 @@
 import { createLogger } from '@sim/logger'
-import { task } from '@trigger.dev/sdk'
+import { AbortTaskRunError, task } from '@trigger.dev/sdk'
 import {
   assertConnectorSyncPayload,
   type ConnectorSyncPayload,
 } from '@/lib/knowledge/connectors/queue'
 import { executeSync } from '@/lib/knowledge/connectors/sync-engine'
 import { CONNECTOR_SYNC_MAX_DURATION_SECONDS } from '@/lib/knowledge/connectors/sync-limits'
+import type { SyncResult } from '@/connectors/types'
 
 const logger = createLogger('TriggerKnowledgeConnectorSync')
 
+export type ConnectorSyncTaskOutcome = 'completed' | 'partial' | 'skipped' | 'failed'
+
+/**
+ * Separates source-sync failures from expected queue/lock no-ops. Intentional
+ * source skips do not make a run partial; actual hydration, persistence, or
+ * processing-dispatch failures do.
+ */
+export function classifyConnectorSyncResult(result: SyncResult): ConnectorSyncTaskOutcome {
+  if (result.skipReason) return 'skipped'
+  if (result.error) return 'failed'
+  if (result.docsFailed > 0 || result.processingDispatch.failed > 0) return 'partial'
+  return 'completed'
+}
+
+function formatConnectorSyncFailure(
+  connectorId: string,
+  result: SyncResult,
+  outcome: Extract<ConnectorSyncTaskOutcome, 'partial' | 'failed'>
+): string {
+  if (outcome === 'failed') {
+    return `Connector sync failed for ${connectorId}: ${result.error}`
+  }
+  return `Connector sync partially failed for ${connectorId}: ${result.docsFailed} source failures, ${result.processingDispatch.failed} dispatch failures`
+}
+
 export async function executeConnectorSyncJob(payload: unknown) {
   const {
     connectorId,
@@ -33,15 +59,33 @@ export async function executeConnectorSyncJob(payload: unknown) {
 
     logger.info(`[${requestId}] Connector sync completed`, {
       connectorId,
+      outcome: classifyConnectorSyncResult(result),
       added: result.docsAdded,
       updated: result.docsUpdated,
       deleted: result.docsDeleted,
       unchanged: result.docsUnchanged,
+      skipped: result.docsSkipped,
       failed: result.docsFailed,
+      processingRequested: result.processingDispatch.requested,
+      processingAccepted: result.processingDispatch.accepted,
+      processingDispatchFailed: result.processingDispatch.failed,
     })
 
+    const outcome = classifyConnectorSyncResult(result)
+    if (outcome === 'failed' || outcome === 'partial') {
+      /**
+       * `executeSync` has already persisted its terminal state. Source failures
+       * preserve the previous incremental watermark so the next connector pass
+       * replays them; dispatch failures remain eligible for the stuck-document
+       * sweep. Retrying this whole task immediately would duplicate a large
+       * fan-out, so fail visibly without retrying the completed transaction.
+       */
+      throw new AbortTaskRunError(formatConnectorSyncFailure(connectorId, result, outcome))
+    }
+
     return {
-      success: !result.error,
+      success: outcome === 'completed',
+      outcome,
       connectorId,
       ...result,
     }

apps/sim/background/knowledge-processing.test.ts

@@ -1,17 +1,24 @@
 /**
  * @vitest-environment node
  */
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockAssertBillingAttributionSnapshot, mockProcessDocumentAsync, mockTask } = vi.hoisted(
-  () => ({
-    mockAssertBillingAttributionSnapshot: vi.fn(),
-    mockProcessDocumentAsync: vi.fn(),
-    mockTask: vi.fn((config) => config),
-  })
-)
+const {
+  mockAssertBillingAttributionSnapshot,
+  mockProcessDocumentAsync,
+  mockResolveTriggerRegion,
+  mockTask,
+  mockTrigger,
+} = vi.hoisted(() => ({
+  mockAssertBillingAttributionSnapshot: vi.fn(),
+  mockProcessDocumentAsync: vi.fn(),
+  mockResolveTriggerRegion: vi.fn(),
+  mockTask: vi.fn((config) => config),
+  mockTrigger: vi.fn(),
+}))
 
-vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
+vi.mock('@trigger.dev/sdk', () => ({ task: mockTask, tasks: { trigger: mockTrigger } }))
+vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: mockResolveTriggerRegion }))
 vi.mock('@/lib/billing/core/billing-attribution', () => ({
   assertBillingAttributionSnapshot: mockAssertBillingAttributionSnapshot,
 }))
@@ -19,7 +26,17 @@ vi.mock('@/lib/knowledge/documents/service', () => ({
   processDocumentAsync: mockProcessDocumentAsync,
 }))
 
-import { runDocumentProcessing } from '@/background/knowledge-processing'
+import { EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client'
+import { EMBEDDING_QUOTA_CIRCUIT_TTL_MS } from '@/lib/embeddings/quota-circuit'
+import {
+  PermanentDocumentProcessingError,
+  UsageLimitDocumentProcessingError,
+} from '@/lib/knowledge/documents/document-processing-error'
+import { MAX_QUOTA_CONTINUATION_ATTEMPTS } from '@/lib/knowledge/documents/processing-quota-continuation'
+import {
+  resolveQuotaContinuationDelayMs,
+  runDocumentProcessing,
+} from '@/background/knowledge-processing'
 
 const BILLING_ATTRIBUTION = {
   actorUserId: 'external-admin',
@@ -45,6 +62,7 @@ const BASE_PAYLOAD = {
   },
   processingOptions: {},
   requestId: 'request-1',
+  processingQueuedAt: '2026-08-24T22:00:00.000Z',
 }
 
 const WORKSPACE_PAYLOAD = {
@@ -55,6 +73,16 @@ const WORKSPACE_PAYLOAD = {
   billingAttribution: BILLING_ATTRIBUTION,
 }
 
+function mockQuotaExhaustion(error: EmbeddingQuotaExhaustedError): void {
+  mockProcessDocumentAsync.mockImplementation(async (...args: unknown[]) => {
+    const attemptContext = args[6] as {
+      scheduleQuotaContinuation?: () => Promise<Date>
+    }
+    await attemptContext.scheduleQuotaContinuation?.()
+    throw error
+  })
+}
+
 describe('knowledge processing worker', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -65,6 +93,12 @@ describe('knowledge processing worker', () => {
       return value
     })
     mockProcessDocumentAsync.mockResolvedValue(undefined)
+    mockResolveTriggerRegion.mockResolvedValue('us-east-1')
+    mockTrigger.mockResolvedValue({ id: 'quota-continuation-run' })
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
   })
 
   it('rejects workspace work without attribution before document processing starts', async () => {
@@ -79,6 +113,99 @@ describe('knowledge processing worker', () => {
     expect(mockProcessDocumentAsync).not.toHaveBeenCalled()
   })
 
+  it('rejects an invalid durable quota retry count before processing starts', async () => {
+    await expect(
+      runDocumentProcessing({
+        ...WORKSPACE_PAYLOAD,
+        quotaRetryCount: -1,
+      })
+    ).rejects.toThrow('Document processing quota retry count is invalid')
+    expect(mockProcessDocumentAsync).not.toHaveBeenCalled()
+  })
+
+  it('rejects an invalid queue-generation stamp before processing starts', async () => {
+    await expect(
+      runDocumentProcessing({
+        ...WORKSPACE_PAYLOAD,
+        processingQueuedAt: 'not-a-date',
+      })
+    ).rejects.toThrow('Document processing queue stamp is invalid')
+    expect(mockProcessDocumentAsync).not.toHaveBeenCalled()
+  })
+
+  it('rejects a queue token that is not the request generation', async () => {
+    await expect(
+      runDocumentProcessing({
+        ...WORKSPACE_PAYLOAD,
+        processingQueueToken: 'another-request',
+      })
+    ).rejects.toThrow('Document processing queue token is invalid')
+    expect(mockProcessDocumentAsync).not.toHaveBeenCalled()
+  })
+
+  it('rejects a new queue token without its canonical queue stamp', async () => {
+    const { processingQueuedAt: _processingQueuedAt, ...payloadWithoutStamp } = WORKSPACE_PAYLOAD
+
+    await expect(
+      runDocumentProcessing({
+        ...payloadWithoutStamp,
+        processingQueueToken: 'request-1',
+      })
+    ).rejects.toThrow('Document processing payload is missing its queue stamp')
+    expect(mockProcessDocumentAsync).not.toHaveBeenCalled()
+  })
+
+  it('rejects a new dispatch charge marker without a queue token', async () => {
+    await expect(
+      runDocumentProcessing({
+        ...WORKSPACE_PAYLOAD,
+        chargedAtDispatch: true,
+      })
+    ).rejects.toThrow('Document processing dispatch charge marker requires a queue token')
+    expect(mockProcessDocumentAsync).not.toHaveBeenCalled()
+  })
+
+  it('accepts a literal pre-rollout staging payload without synthesizing a generation stamp', async () => {
+    const { processingQueuedAt: _processingQueuedAt, ...stagingPayload } = WORKSPACE_PAYLOAD
+
+    await runDocumentProcessing(stagingPayload)
+
+    expect(mockProcessDocumentAsync).toHaveBeenLastCalledWith(
+      'knowledge-base-1',
+      'document-1',
+      BASE_PAYLOAD.docData,
+      {},
+      expect.objectContaining({ billingScope: 'workspace' }),
+      'request-1',
+      expect.objectContaining({
+        chargedAtDispatch: false,
+        scheduleQuotaContinuation: expect.any(Function),
+      })
+    )
+  })
+
+  it('propagates a new queue token while accepting legacy queuedAt-only payloads', async () => {
+    await runDocumentProcessing({
+      ...WORKSPACE_PAYLOAD,
+      processingQueueToken: 'request-1',
+      chargedAtDispatch: false,
+    })
+
+    expect(mockProcessDocumentAsync).toHaveBeenLastCalledWith(
+      expect.any(String),
+      expect.any(String),
+      expect.any(Object),
+      expect.any(Object),
+      expect.any(Object),
+      'request-1',
+      expect.objectContaining({
+        chargedAtDispatch: false,
+        processingQueueToken: 'request-1',
+        processingQueuedAt: new Date(BASE_PAYLOAD.processingQueuedAt),
+      })
+    )
+  })
+
   it('preserves the validated actor and payer snapshot through serialization', async () => {
     await runDocumentProcessing(structuredClone(WORKSPACE_PAYLOAD))
 
@@ -93,7 +220,12 @@ describe('knowledge processing worker', () => {
         workspaceId: 'workspace-1',
         billingAttribution: BILLING_ATTRIBUTION,
       },
-      BASE_PAYLOAD.requestId
+      BASE_PAYLOAD.requestId,
+      expect.objectContaining({
+        chargedAtDispatch: true,
+        processingQueuedAt: new Date(BASE_PAYLOAD.processingQueuedAt),
+        scheduleQuotaContinuation: expect.any(Function),
+      })
     )
   })
 
@@ -135,7 +267,228 @@ describe('knowledge processing worker', () => {
         actorUserId: 'legacy-owner',
         workspaceId: null,
       },
-      BASE_PAYLOAD.requestId
+      BASE_PAYLOAD.requestId,
+      expect.objectContaining({
+        chargedAtDispatch: true,
+        processingQueuedAt: new Date(BASE_PAYLOAD.processingQueuedAt),
+        scheduleQuotaContinuation: expect.any(Function),
+      })
+    )
+  })
+
+  it('reports elapsed processing time rather than an epoch timestamp', async () => {
+    vi.spyOn(Date, 'now').mockReturnValueOnce(1_000).mockReturnValueOnce(1_125)
+
+    const result = await runDocumentProcessing({
+      ...BASE_PAYLOAD,
+      billingScope: 'non-workspace',
+      actorUserId: 'legacy-owner',
+      workspaceId: null,
+    })
+
+    expect(result.processingTime).toBe(125)
+  })
+
+  it('returns a controlled terminal result for permanent document input failures', async () => {
+    mockProcessDocumentAsync.mockRejectedValue(
+      new PermanentDocumentProcessingError(
+        'archive_safety_limit',
+        'This file expands beyond the safe processing limit.'
+      )
+    )
+
+    await expect(
+      runDocumentProcessing({
+        ...BASE_PAYLOAD,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      })
+    ).resolves.toMatchObject({
+      success: false,
+      outcome: 'permanent_failure',
+      code: 'archive_safety_limit',
+      error: 'This file expands beyond the safe processing limit.',
+    })
+  })
+
+  it('reports a mutable usage-limit outcome without requesting an immediate retry', async () => {
+    mockProcessDocumentAsync.mockRejectedValue(
+      new UsageLimitDocumentProcessingError('Usage limit exceeded. Upgrade to continue.')
+    )
+
+    await expect(
+      runDocumentProcessing({
+        ...BASE_PAYLOAD,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      })
+    ).resolves.toMatchObject({
+      success: false,
+      outcome: 'usage_limit',
+      error: 'Usage limit exceeded. Upgrade to continue.',
+    })
+  })
+
+  it('preserves normal retries for transient failures', async () => {
+    const transientError = new Error('Database connection timed out')
+    mockProcessDocumentAsync.mockRejectedValue(transientError)
+
+    await expect(
+      runDocumentProcessing({
+        ...BASE_PAYLOAD,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      })
+    ).rejects.toBe(transientError)
+  })
+
+  it('durably continues quota exhaustion beyond the task attempt budget', async () => {
+    vi.spyOn(Date, 'now').mockReturnValue(1_000)
+    const quotaError = new EmbeddingQuotaExhaustedError('openai')
+    mockQuotaExhaustion(quotaError)
+
+    await expect(
+      runDocumentProcessing({
+        ...BASE_PAYLOAD,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      })
+    ).resolves.toMatchObject({ success: false, outcome: 'quota_deferred' })
+
+    expect(mockTrigger).toHaveBeenCalledWith(
+      'knowledge-process-document',
+      expect.objectContaining({
+        documentId: 'document-1',
+        requestId: 'request-1',
+        processingQueuedAt: BASE_PAYLOAD.processingQueuedAt,
+        quotaRetryCount: 1,
+      }),
+      expect.objectContaining({
+        delay: expect.any(Date),
+        idempotencyKey: 'knowledge-quota-document-1-request-1-1',
+        region: 'us-east-1',
+      })
+    )
+    const delay = mockTrigger.mock.calls[0]?.[2]?.delay as Date
+    expect(delay.getTime()).toBeGreaterThanOrEqual(1_000 + EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 0.8)
+    expect(delay.getTime()).toBeLessThanOrEqual(1_000 + EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 1.2)
+  })
+
+  it('ends a quota chain after the bounded continuation horizon', async () => {
+    mockQuotaExhaustion(new EmbeddingQuotaExhaustedError('openai'))
+
+    await expect(
+      runDocumentProcessing({
+        ...BASE_PAYLOAD,
+        quotaRetryCount: MAX_QUOTA_CONTINUATION_ATTEMPTS,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      })
+    ).resolves.toMatchObject({ success: false, outcome: 'quota_exhausted' })
+
+    expect(mockTrigger).not.toHaveBeenCalled()
+    expect(mockProcessDocumentAsync).toHaveBeenCalledWith(
+      expect.any(String),
+      expect.any(String),
+      expect.any(Object),
+      expect.any(Object),
+      expect.any(Object),
+      expect.any(String),
+      expect.objectContaining({ quotaContinuationExhausted: true })
+    )
+  })
+
+  it('keeps the task failed when the durable continuation handoff fails', async () => {
+    mockQuotaExhaustion(new EmbeddingQuotaExhaustedError('openai'))
+    const dispatchError = new Error('Trigger dispatch unavailable')
+    mockTrigger.mockRejectedValue(dispatchError)
+
+    await expect(
+      runDocumentProcessing({
+        ...BASE_PAYLOAD,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      })
+    ).rejects.toBe(dispatchError)
+  })
+
+  it('continues an existing quota chain with the same indexing pass identity', async () => {
+    mockQuotaExhaustion(new EmbeddingQuotaExhaustedError('openai'))
+
+    await runDocumentProcessing({
+      ...BASE_PAYLOAD,
+      quotaRetryCount: 3,
+      billingScope: 'non-workspace',
+      actorUserId: 'legacy-owner',
+      workspaceId: null,
+    })
+
+    expect(mockTrigger).toHaveBeenCalledWith(
+      'knowledge-process-document',
+      expect.objectContaining({ requestId: 'request-1', quotaRetryCount: 4 }),
+      expect.objectContaining({
+        idempotencyKey: 'knowledge-quota-document-1-request-1-4',
+      })
+    )
+  })
+
+  it('does not refund the original dispatch charge again on a task retry', async () => {
+    await runDocumentProcessing(
+      {
+        ...BASE_PAYLOAD,
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      },
+      2
+    )
+
+    expect(mockProcessDocumentAsync).toHaveBeenCalledWith(
+      'knowledge-base-1',
+      'document-1',
+      BASE_PAYLOAD.docData,
+      {},
+      {
+        billingScope: 'non-workspace',
+        actorUserId: 'legacy-owner',
+        workspaceId: null,
+      },
+      BASE_PAYLOAD.requestId,
+      expect.objectContaining({
+        chargedAtDispatch: false,
+        processingQueuedAt: new Date(BASE_PAYLOAD.processingQueuedAt),
+        scheduleQuotaContinuation: expect.any(Function),
+      })
+    )
+  })
+
+  it('preserves the queue generation without refunding a quota continuation run', async () => {
+    await runDocumentProcessing({
+      ...BASE_PAYLOAD,
+      quotaRetryCount: 3,
+      billingScope: 'non-workspace',
+      actorUserId: 'legacy-owner',
+      workspaceId: null,
+    })
+
+    expect(mockProcessDocumentAsync).toHaveBeenLastCalledWith(
+      'knowledge-base-1',
+      'document-1',
+      BASE_PAYLOAD.docData,
+      {},
+      expect.objectContaining({ billingScope: 'non-workspace' }),
+      BASE_PAYLOAD.requestId,
+      expect.objectContaining({
+        chargedAtDispatch: false,
+        processingQueuedAt: new Date(BASE_PAYLOAD.processingQueuedAt),
+        scheduleQuotaContinuation: expect.any(Function),
+      })
     )
   })
 })
@@ -152,4 +505,17 @@ describe('knowledge-process-document task configuration', () => {
 
     expect(processDocument.retry?.outOfMemory?.machine).toBe('large-2x')
   })
+
+  it('backs durable quota continuations off to a bounded polling interval', () => {
+    const first = resolveQuotaContinuationDelayMs(1)
+    const second = resolveQuotaContinuationDelayMs(2)
+    const capped = resolveQuotaContinuationDelayMs(Number.MAX_SAFE_INTEGER)
+
+    expect(first).toBeGreaterThanOrEqual(EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 0.8)
+    expect(first).toBeLessThanOrEqual(EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 1.2)
+    expect(second).toBeGreaterThanOrEqual(EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 2 * 0.8)
+    expect(second).toBeLessThanOrEqual(EMBEDDING_QUOTA_CIRCUIT_TTL_MS * 2 * 1.2)
+    expect(capped).toBeGreaterThanOrEqual(6 * 60 * 60 * 1000 * 0.8)
+    expect(capped).toBeLessThanOrEqual(6 * 60 * 60 * 1000)
+  })
 })

apps/sim/background/knowledge-processing.ts

@@ -1,16 +1,32 @@
 import { createLogger } from '@sim/logger'
 import { task } from '@trigger.dev/sdk'
 import { env, envNumber } from '@/lib/core/config/env'
+import { EMBEDDING_QUOTA_EXHAUSTED_MESSAGE, isEmbeddingQuotaExhaustion } from '@/lib/embeddings'
+import {
+  isPermanentDocumentProcessingError,
+  isUsageLimitDocumentProcessingError,
+} from '@/lib/knowledge/documents/document-processing-error'
 import {
   assertDocumentProcessingPayload,
   type DocumentProcessingBillingContext,
   type DocumentProcessingPayload,
 } from '@/lib/knowledge/documents/processing-payload'
+import {
+  canScheduleDocumentProcessingQuotaContinuation,
+  MAX_QUOTA_CONTINUATION_ATTEMPTS,
+  resolveQuotaContinuationDelayMs,
+  scheduleDocumentProcessingQuotaContinuation,
+} from '@/lib/knowledge/documents/processing-quota-continuation'
 import { processDocumentAsync } from '@/lib/knowledge/documents/service'
 
 const logger = createLogger('TriggerKnowledgeProcessing')
+export { resolveQuotaContinuationDelayMs }
 
-export async function runDocumentProcessing(rawPayload: DocumentProcessingPayload) {
+export async function runDocumentProcessing(
+  rawPayload: DocumentProcessingPayload,
+  attemptNumber = 1
+) {
+  const startedAt = Date.now()
   const payload = assertDocumentProcessingPayload(rawPayload)
   const { knowledgeBaseId, documentId, docData, processingOptions, requestId } = payload
   const billingContext: DocumentProcessingBillingContext =
@@ -26,6 +42,7 @@ export async function runDocumentProcessing(rawPayload: DocumentProcessingPayloa
           actorUserId: payload.actorUserId,
           workspaceId: null,
         }
+  const canScheduleQuotaContinuation = canScheduleDocumentProcessingQuotaContinuation(payload)
 
   logger.info(`[${requestId}] Starting Trigger.dev processing for document: ${docData.filename}`)
 
@@ -36,7 +53,24 @@ export async function runDocumentProcessing(rawPayload: DocumentProcessingPayloa
       docData,
       processingOptions,
       billingContext,
-      requestId
+      requestId,
+      {
+        chargedAtDispatch:
+          (payload.chargedAtDispatch ?? payload.processingQueuedAt !== undefined) &&
+          attemptNumber === 1 &&
+          payload.quotaRetryCount === undefined,
+        ...(payload.processingQueueToken
+          ? { processingQueueToken: payload.processingQueueToken }
+          : {}),
+        ...(payload.processingQueuedAt
+          ? { processingQueuedAt: new Date(payload.processingQueuedAt) }
+          : {}),
+        ...(canScheduleQuotaContinuation
+          ? {
+              scheduleQuotaContinuation: () => scheduleDocumentProcessingQuotaContinuation(payload),
+            }
+          : { quotaContinuationExhausted: true }),
+      }
     )
 
     logger.info(`[${requestId}] Successfully processed document: ${docData.filename}`)
@@ -45,9 +79,54 @@ export async function runDocumentProcessing(rawPayload: DocumentProcessingPayloa
       success: true,
       documentId,
       filename: docData.filename,
-      processingTime: Date.now(),
+      processingTime: Date.now() - startedAt,
     }
   } catch (error) {
+    if (isUsageLimitDocumentProcessingError(error)) {
+      logger.warn(`[${requestId}] Document processing is blocked by the current usage limit`, {
+        filename: docData.filename,
+      })
+      return {
+        success: false,
+        outcome: 'usage_limit' as const,
+        documentId,
+        filename: docData.filename,
+        error: error.message,
+        processingTime: Date.now() - startedAt,
+      }
+    }
+    if (isEmbeddingQuotaExhaustion(error)) {
+      const outcome = canScheduleQuotaContinuation ? 'quota_deferred' : 'quota_exhausted'
+      logger.warn(`[${requestId}] Embedding quota is exhausted`, {
+        filename: docData.filename,
+        quotaRetryCount: payload.quotaRetryCount ?? 0,
+        continuationLimit: MAX_QUOTA_CONTINUATION_ATTEMPTS,
+        outcome,
+      })
+      return {
+        success: false,
+        outcome,
+        documentId,
+        filename: docData.filename,
+        error: EMBEDDING_QUOTA_EXHAUSTED_MESSAGE,
+        processingTime: Date.now() - startedAt,
+      }
+    }
+    if (isPermanentDocumentProcessingError(error)) {
+      logger.warn(`[${requestId}] Document cannot be processed without changing its content`, {
+        code: error.code,
+        filename: docData.filename,
+      })
+      return {
+        success: false,
+        outcome: 'permanent_failure' as const,
+        documentId,
+        filename: docData.filename,
+        code: error.code,
+        error: error.message,
+        processingTime: Date.now() - startedAt,
+      }
+    }
     logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error)
     throw error
   }
@@ -64,12 +143,8 @@ export const processDocument = task({
     maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
     /**
      * `maxAttempts` does not cover an out-of-memory kill — Trigger.dev retries
-     * `TASK_PROCESS_OOM_KILLED` only when a larger preset is named here. Eleven
-     * documents were killed in one afternoon and every one recorded
-     * `attempt_count = 1`, so each was left `failed` with no retry at all. The
-     * escalation is a safety net, not the fix: the workbook parser's allocation
-     * no longer scales with a sheet's declared range, and fleet p99 memory is
-     * 691 MB against this machine's 8 GB.
+     * `TASK_PROCESS_OOM_KILLED` only when a larger preset is named here. The
+     * escalation is a safety net after parser allocations have been bounded.
      */
     outOfMemory: { machine: 'large-2x' },
   },
@@ -77,5 +152,6 @@ export const processDocument = task({
     concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
     name: 'document-processing-queue',
   },
-  run: runDocumentProcessing,
+  run: (payload: DocumentProcessingPayload, { ctx }) =>
+    runDocumentProcessing(payload, ctx.attempt.number),
 })

apps/sim/connectors/fireflies/fireflies.test.ts

@@ -3,36 +3,33 @@
  */
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() }))
-
-vi.mock('@/lib/knowledge/documents/utils', () => ({
-  fetchWithRetry: mockFetchWithRetry,
-  VALIDATE_RETRY_OPTIONS: {},
-}))
 vi.mock('@/components/icons', () => ({ FirefliesIcon: () => null }))
 
 import { firefliesConnector } from '@/connectors/fireflies/fireflies'
 
+beforeEach(() => {
+  vi.useRealTimers()
+  vi.unstubAllGlobals()
+})
+
 interface GraphQLCall {
   query: string
   variables: Record<string, unknown>
 }
 
 /** Replays the given GraphQL bodies in order and records what was sent. */
-function mockGraphQL(responses: { status?: number; body: unknown }[]) {
+function mockGraphQL(
+  responses: { status?: number; body: unknown; headers?: Record<string, string> }[]
+) {
   const calls: GraphQLCall[] = []
   let index = 0
-  mockFetchWithRetry.mockImplementation(async (_url: string, options: RequestInit) => {
-    calls.push(JSON.parse(String(options.body)))
+  const fetchMock = vi.fn(async (_url: string | URL | Request, options?: RequestInit) => {
+    calls.push(JSON.parse(String(options?.body)))
     const route = responses[Math.min(index++, responses.length - 1)]
     const status = route.status ?? 200
-    return {
-      ok: status >= 200 && status < 300,
-      status,
-      json: async () => route.body,
-      text: async () => JSON.stringify(route.body),
-    } as unknown as Response
+    return new Response(JSON.stringify(route.body), { status, headers: route.headers })
   })
+  vi.stubGlobal('fetch', fetchMock)
   return calls
 }
 
@@ -83,17 +80,54 @@ describe('fireflies listDocuments', () => {
     expect(calls[0].query).not.toContain('50')
   })
 
-  it('leaves listingCapped unset when the source is genuinely exhausted', async () => {
+  it.each([
+    [
+      { is_live: true, meeting_info: { summary_status: 'processing' } },
+      { is_live: false, meeting_info: { summary_status: 'processing' } },
+    ],
+    [
+      { is_live: false, meeting_info: { summary_status: 'processing' } },
+      { is_live: false, meeting_info: { summary_status: 'processed' } },
+    ],
+  ])('changes the listing hash when transcript lifecycle state changes', async (before, after) => {
+    const calls = mockGraphQL([
+      { body: { data: { transcripts: [transcript('t0', before)] } } },
+      { body: { data: { transcripts: [transcript('t0', after)] } } },
+    ])
+
+    const first = await firefliesConnector.listDocuments('key', {}, undefined, {})
+    const second = await firefliesConnector.listDocuments('key', {}, undefined, {})
+
+    expect(first.documents[0].contentHash).not.toBe(second.documents[0].contentHash)
+    expect(calls[0].query).toContain('is_live')
+    expect(calls[0].query).toContain('summary_status')
+  })
+
+  it.each(['-1', '1.5', 'Infinity', '9007199254740992', 'opaque'])(
+    'rejects invalid pagination cursor %s before calling Fireflies',
+    async (cursor) => {
+      const fetchMock = vi.fn()
+      vi.stubGlobal('fetch', fetchMock)
+
+      await expect(firefliesConnector.listDocuments('key', {}, cursor, {})).rejects.toThrow(
+        'Invalid Fireflies connector pagination cursor'
+      )
+      expect(fetchMock).not.toHaveBeenCalled()
+    }
+  )
+
+  it('marks offset pagination unsafe for deletion reconciliation even when exhausted', async () => {
     mockGraphQL([page(3)])
     const syncContext: Record<string, unknown> = {}
 
     const result = await firefliesConnector.listDocuments('key', {}, undefined, syncContext)
 
     expect(result.hasMore).toBe(false)
+    expect(result.reconciliationSafe).toBe(false)
     expect(syncContext.listingCapped).toBeUndefined()
   })
 
-  it('leaves listingCapped unset when maxTranscripts lands exactly on exhaustion', async () => {
+  it('keeps deletion reconciliation disabled when maxTranscripts lands on exhaustion', async () => {
     mockGraphQL([page(3)])
     const syncContext: Record<string, unknown> = {}
 
@@ -105,6 +139,7 @@ describe('fireflies listDocuments', () => {
     )
 
     expect(result.documents).toHaveLength(3)
+    expect(result.reconciliationSafe).toBe(false)
     expect(syncContext.listingCapped).toBeUndefined()
   })
 
@@ -124,48 +159,227 @@ describe('fireflies listDocuments', () => {
     expect(syncContext.listingCapped).toBe(true)
   })
 
+  it.each(['1.5', 'Infinity'])(
+    'rejects invalid maxTranscripts %s before calling Fireflies',
+    async (maxTranscripts) => {
+      const fetchMock = vi.fn()
+      vi.stubGlobal('fetch', fetchMock)
+
+      await expect(
+        firefliesConnector.listDocuments('key', { maxTranscripts }, undefined, {})
+      ).rejects.toThrow(/positive safe integer/)
+      expect(fetchMock).not.toHaveBeenCalled()
+    }
+  )
+
   it('throws on a GraphQL errors[] payload rather than reporting an empty listing', async () => {
     mockGraphQL([
-      { body: { data: {}, errors: [{ message: 'Rate limited', code: 'too_many_requests' }] } },
+      { body: { data: {}, errors: [{ message: 'Invalid input', code: 'invalid_arguments' }] } },
     ])
 
     await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow(
-      /too_many_requests/
+      /invalid_arguments/
     )
   })
 
   it('throws rather than reporting an empty listing when a 200 body is unreadable', async () => {
-    mockFetchWithRetry.mockResolvedValue({
-      ok: true,
-      status: 200,
-      json: async () => {
-        throw new SyntaxError('Unexpected token < in JSON at position 0')
-      },
-      text: async () => '<html>gateway</html>',
-    } as unknown as Response)
+    vi.useFakeTimers()
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => new Response('<html>gateway</html>'))
+    )
     const syncContext: Record<string, unknown> = {}
 
-    await expect(
+    const pending = expect(
       firefliesConnector.listDocuments('key', {}, undefined, syncContext)
     ).rejects.toThrow(/malformed/i)
+    await vi.runAllTimersAsync()
+    await pending
   })
 
   it('throws rather than reporting an empty listing when a 200 carries no data', async () => {
+    vi.useFakeTimers()
     mockGraphQL([{ body: {} }])
 
+    const pending = expect(
+      firefliesConnector.listDocuments('key', {}, undefined, {})
+    ).rejects.toThrow(/malformed/i)
+    await vi.runAllTimersAsync()
+    await pending
+  })
+
+  it('throws rather than reporting an empty listing when transcripts are missing', async () => {
+    mockGraphQL([{ body: { data: {} } }])
+
     await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow(
-      /malformed/i
+      'Fireflies API returned malformed transcript-list data'
     )
   })
 
-  it('surfaces the errors[] message on a non-2xx response', async () => {
+  it('rejects malformed transcript rows instead of silently filtering them', async () => {
+    mockGraphQL([{ body: { data: { transcripts: [{}] } } }])
+
+    await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow(
+      'Fireflies API returned malformed transcript metadata'
+    )
+  })
+
+  it('retries one malformed page without discarding the sync', async () => {
+    vi.useFakeTimers()
+    mockGraphQL([{ body: {} }, page(2)])
+
+    const pending = firefliesConnector.listDocuments('key', {}, undefined, {})
+    await vi.runAllTimersAsync()
+    const result = await pending
+
+    expect(result.documents).toHaveLength(2)
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
+  it.each([
+    ['HTTP', { status: 500, body: { message: 'Temporary upstream failure' } }],
+    [
+      'GraphQL',
+      {
+        body: {
+          errors: [{ message: 'Temporary resolver failure', extensions: { status: 500 } }],
+        },
+      },
+    ],
+  ])('retries a transient %s 5xx response', async (_kind, failure) => {
+    vi.useFakeTimers()
+    mockGraphQL([failure, page(2)])
+
+    const pending = firefliesConnector.listDocuments('key', {}, undefined, {})
+    await vi.runAllTimersAsync()
+    const result = await pending
+
+    expect(result.documents).toHaveLength(2)
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
+  it.each([
+    [520, 60],
+    [522, 120],
+  ])('respects Retry-After and retries Cloudflare %i', async (status, retryAfterSeconds) => {
+    vi.useFakeTimers()
+    mockGraphQL([
+      {
+        status,
+        body: { diagnostic: 'temporary edge failure' },
+        headers: { 'retry-after': String(retryAfterSeconds) },
+      },
+      page(2),
+    ])
+
+    const pending = firefliesConnector.listDocuments('key', {}, undefined, {})
+    await vi.advanceTimersByTimeAsync(retryAfterSeconds * 1000 - 1)
+    expect(global.fetch).toHaveBeenCalledTimes(1)
+    await vi.advanceTimersByTimeAsync(1)
+    const result = await pending
+
+    expect(result.documents).toHaveLength(2)
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
+  it('respects Retry-After when a retryable error body exceeds the diagnostic limit', async () => {
+    vi.useFakeTimers()
+    const retryAfterSeconds = 60
+    mockGraphQL([
+      {
+        status: 503,
+        body: { diagnostic: 'body is not materialized' },
+        headers: {
+          'content-length': String(16 * 1024 * 1024 + 1),
+          'retry-after': String(retryAfterSeconds),
+        },
+      },
+      page(2),
+    ])
+
+    const pending = firefliesConnector.listDocuments('key', {}, undefined, {})
+    await vi.advanceTimersByTimeAsync(retryAfterSeconds * 1000 - 1)
+    expect(global.fetch).toHaveBeenCalledTimes(1)
+    await vi.advanceTimersByTimeAsync(1)
+    const result = await pending
+
+    expect(result.documents).toHaveLength(2)
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
+  it('parses an HTTP 429 GraphQL retry timestamp before retrying', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z'))
+    mockGraphQL([
+      {
+        status: 429,
+        body: {
+          errors: [
+            {
+              message: 'Rate limited',
+              code: 'too_many_requests',
+              extensions: { status: 429, metadata: { retryAfter: Date.now() + 60_000 } },
+            },
+          ],
+        },
+      },
+      page(1),
+    ])
+
+    const pending = firefliesConnector.listDocuments('key', {}, undefined, {})
+    await vi.advanceTimersByTimeAsync(60_000)
+    await expect(pending).resolves.toMatchObject({ documents: [{ externalId: 't0' }] })
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
+  it('surfaces a validated errors[] code without the provider message', async () => {
     mockGraphQL([
       { status: 403, body: { errors: [{ message: 'Upgrade required', code: 'paid_required' }] } },
     ])
 
     await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow(
-      /Upgrade required/
+      /paid_required/
     )
+    expect(global.fetch).toHaveBeenCalledTimes(1)
+  })
+
+  it('bounds non-retryable GraphQL error responses', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(
+        async () =>
+          new Response('x'.repeat(17 * 1024 * 1024), { status: 403, statusText: 'Forbidden' })
+      )
+    )
+
+    await expect(firefliesConnector.listDocuments('key', {}, undefined, {})).rejects.toThrow(
+      /exceeded the diagnostic limit/i
+    )
+    expect(global.fetch).toHaveBeenCalledTimes(1)
+  })
+})
+
+describe('fireflies validateConfig', () => {
+  it.each(['1.5', 'Infinity'])(
+    'rejects invalid maxTranscripts %s without an API request',
+    async (maxTranscripts) => {
+      const fetchMock = vi.fn()
+      vi.stubGlobal('fetch', fetchMock)
+
+      await expect(firefliesConnector.validateConfig!('key', { maxTranscripts })).resolves.toEqual({
+        valid: false,
+        error: 'Max transcripts must be a positive safe integer, or 0 for unlimited',
+      })
+      expect(fetchMock).not.toHaveBeenCalled()
+    }
+  )
+
+  it('accepts a valid integer maxTranscripts', async () => {
+    mockGraphQL([{ body: { data: { user: { user_id: 'user-1' } } } }])
+
+    await expect(
+      firefliesConnector.validateConfig!('key', { maxTranscripts: '25' })
+    ).resolves.toEqual({ valid: true })
   })
 })
 
@@ -174,6 +388,17 @@ describe('fireflies getDocument', () => {
     vi.clearAllMocks()
   })
 
+  it.each([null, {}, transcript('different')])(
+    'rejects malformed or mismatched successful transcript metadata',
+    async (value) => {
+      mockGraphQL([{ body: { data: { transcript: value } } }])
+
+      await expect(firefliesConnector.getDocument('key', {}, 't0')).rejects.toThrow(
+        'Fireflies API returned malformed transcript metadata'
+      )
+    }
+  )
+
   it('declares the transcript id as String! and reuses the stub contentHash', async () => {
     const calls = mockGraphQL([
       page(1),
@@ -211,6 +436,109 @@ describe('fireflies getDocument', () => {
     expect(full?.metadata?.duration).toBe(45)
   })
 
+  it('formats the documented string keyword shape and tolerates legacy arrays', async () => {
+    mockGraphQL([
+      {
+        body: {
+          data: { transcript: transcript('t0', { summary: { keywords: 'alpha, beta' } }) },
+        },
+      },
+      {
+        body: {
+          data: { transcript: transcript('t1', { summary: { keywords: ['alpha', 'beta'] } }) },
+        },
+      },
+    ])
+
+    const documented = await firefliesConnector.getDocument('key', {}, 't0')
+    const legacy = await firefliesConnector.getDocument('key', {}, 't1')
+
+    expect(documented?.content).toContain('Keywords: alpha, beta')
+    expect(legacy?.content).toContain('Keywords: alpha, beta')
+  })
+
+  it('surfaces extracted transcript content beyond its byte budget as skipped', async () => {
+    mockGraphQL([
+      {
+        body: {
+          data: {
+            transcript: transcript('t0', {
+              sentences: [{ speaker_name: 'Ada', text: 'x'.repeat(9 * 1024 * 1024) }],
+            }),
+          },
+        },
+      },
+    ])
+
+    const result = await firefliesConnector.getDocument('key', {}, 't0')
+
+    expect(result).toMatchObject({ content: '', contentDeferred: false })
+    expect(result?.skippedReason).toContain('8MB')
+  })
+
+  it('admits a hydration envelope exactly at the wire limit before applying the content cap', async () => {
+    const maxResponseBytes = 16 * 1024 * 1024
+    const responseBody = (text: string) => ({
+      data: {
+        transcript: transcript('boundary', {
+          sentences: [{ speaker_name: 'Ada', text }],
+        }),
+      },
+    })
+    const emptyEnvelopeBytes = Buffer.byteLength(JSON.stringify(responseBody('')), 'utf8')
+    const serialized = JSON.stringify(
+      responseBody('x'.repeat(maxResponseBytes - emptyEnvelopeBytes))
+    )
+    expect(Buffer.byteLength(serialized, 'utf8')).toBe(maxResponseBytes)
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => new Response(serialized))
+    )
+
+    const result = await firefliesConnector.getDocument('key', {}, 'boundary')
+
+    expect(result).toMatchObject({ content: '', contentDeferred: false })
+    expect(result?.skippedReason).toContain('8MB extracted-content limit')
+  })
+
+  it('surfaces JSON escape expansion beyond the wire cap as a visible skip', async () => {
+    const escapedText = '\u0000'.repeat(3 * 1024 * 1024)
+    const responseBody = JSON.stringify({
+      data: {
+        transcript: transcript('escaped', {
+          sentences: [{ speaker_name: 'Ada', text: escapedText }],
+        }),
+      },
+    })
+    expect(Buffer.byteLength(escapedText, 'utf8')).toBeLessThan(8 * 1024 * 1024)
+    expect(Buffer.byteLength(responseBody, 'utf8')).toBeGreaterThan(16 * 1024 * 1024)
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => new Response(responseBody))
+    )
+
+    const result = await firefliesConnector.getDocument('key', {}, 'escaped')
+
+    expect(result).toMatchObject({
+      externalId: 'escaped',
+      content: '',
+      contentDeferred: false,
+      skippedReason:
+        'Transcript response exceeds the 16MB safe hydration limit and was not indexed',
+    })
+  })
+
+  it('does not convert an oversized provider error response into a skipped transcript', async () => {
+    vi.stubGlobal(
+      'fetch',
+      vi.fn(async () => new Response('x'.repeat(17 * 1024 * 1024), { status: 403 }))
+    )
+
+    await expect(firefliesConnector.getDocument('key', {}, 'provider-error')).rejects.toThrow(
+      /HTTP error: 403/
+    )
+  })
+
   it('falls back to organizer_email when the deprecated host_email is absent', async () => {
     mockGraphQL([{ body: { data: { transcript: transcript('t0') } } }])
 

apps/sim/connectors/fireflies/fireflies.ts

@@ -1,14 +1,54 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
-import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { isPlainRecord } from '@sim/utils/object'
+import { isPayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits'
+import {
+  isRetryableError,
+  type RetryOptions,
+  resolveRetryDelayMs,
+  retryWithExponentialBackoff,
+  VALIDATE_RETRY_OPTIONS,
+} from '@/lib/knowledge/documents/utils'
 import { firefliesConnectorMeta } from '@/connectors/fireflies/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
-import { parseTagDate } from '@/connectors/utils'
+import {
+  ConnectorFileTooLargeError,
+  computeContentHash,
+  markSkipped,
+  parseOptionalUnlimitedSafeInteger,
+  parseTagDate,
+} from '@/connectors/utils'
 
 const logger = createLogger('FirefliesConnector')
 
 const FIREFLIES_GRAPHQL_URL = 'https://api.fireflies.ai/graphql'
 const TRANSCRIPTS_PER_PAGE = 50
+const FIREFLIES_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
+const FIREFLIES_MAX_EXTRACTED_CONTENT_BYTES = 8 * 1024 * 1024
+const FIREFLIES_DEFAULT_MAX_RETRY_DELAY_MS = 120_000
+const FIREFLIES_DEFAULT_MAX_RETRIES = 2
+const FIREFLIES_EXTRACTED_CONTENT_SKIP_REASON =
+  'Transcript exceeds the 8MB extracted-content limit and was not indexed'
+const FIREFLIES_RESPONSE_SKIP_REASON =
+  'Transcript response exceeds the 16MB safe hydration limit and was not indexed'
+const MAX_TRANSCRIPTS_VALIDATION_ERROR =
+  'Max transcripts must be a positive safe integer, or 0 for unlimited'
+
+function parseMaxTranscripts(value: unknown): number {
+  return parseOptionalUnlimitedSafeInteger(value, MAX_TRANSCRIPTS_VALIDATION_ERROR)
+}
+
+function parsePaginationCursor(cursor?: string): number {
+  if (cursor === undefined) return 0
+  if (!/^\d+$/.test(cursor)) {
+    throw new Error('Invalid Fireflies connector pagination cursor')
+  }
+  const parsed = Number(cursor)
+  if (!Number.isSafeInteger(parsed)) {
+    throw new Error('Invalid Fireflies connector pagination cursor')
+  }
+  return parsed
+}
 
 interface FirefliesTranscript {
   id: string
@@ -22,15 +62,42 @@ interface FirefliesTranscript {
   participants?: string[]
   transcript_url?: string
   speakers?: { name: string }[]
+  is_live?: boolean
+  meeting_info?: {
+    summary_status?: string
+  }
   sentences?: { speaker_name: string; text: string }[]
   summary?: {
-    keywords?: string[]
+    /** Fireflies documents a string; tolerate the historical array shape defensively. */
+    keywords?: string | string[]
     action_items?: string
     overview?: string
     short_summary?: string
   }
 }
 
+function isFirefliesTranscript(value: unknown): value is FirefliesTranscript {
+  return (
+    isPlainRecord(value) &&
+    typeof value.id === 'string' &&
+    value.id.length > 0 &&
+    typeof value.title === 'string' &&
+    typeof value.date === 'number' &&
+    Number.isFinite(value.date) &&
+    typeof value.duration === 'number' &&
+    Number.isFinite(value.duration)
+  )
+}
+
+interface FirefliesGraphQLError {
+  message?: string
+  code?: string
+  extensions?: {
+    status?: number
+    metadata?: { retryAfter?: number }
+  }
+}
+
 /**
  * Carries the Fireflies GraphQL error `code` so callers can tell a genuinely missing
  * object (`object_not_found`) from a transient fault (`too_many_requests`, 5xx).
@@ -38,13 +105,40 @@ interface FirefliesTranscript {
 class FirefliesApiError extends Error {
   constructor(
     message: string,
-    readonly code?: string
+    readonly code?: string,
+    readonly status?: number,
+    readonly retryAfterMs?: number
   ) {
     super(message)
     this.name = 'FirefliesApiError'
   }
 }
 
+class FirefliesMalformedResponseError extends Error {
+  constructor(status: number) {
+    super(`Fireflies API returned a malformed response with no data (HTTP ${status})`)
+    this.name = 'FirefliesMalformedResponseError'
+  }
+}
+
+const FIREFLIES_RETRYABLE_ERROR_CODES = new Set([
+  'too_many_requests',
+  'request_timeout',
+  'invariant_violation',
+])
+
+function isRetryableFirefliesError(error: unknown): boolean {
+  if (error instanceof FirefliesMalformedResponseError) return true
+  if (error instanceof FirefliesApiError) {
+    return (
+      (error.status !== undefined && error.status >= 500 && error.status <= 599) ||
+      Boolean(error.code && FIREFLIES_RETRYABLE_ERROR_CODES.has(error.code)) ||
+      isRetryableError(error)
+    )
+  }
+  return isRetryableError(error)
+}
+
 /**
  * Executes a GraphQL query against the Fireflies API.
  */
@@ -52,109 +146,173 @@ async function firefliesGraphQL(
   accessToken: string,
   query: string,
   variables: Record<string, unknown> = {},
-  retryOptions?: Parameters<typeof fetchWithRetry>[2]
+  retryOptions?: RetryOptions
 ): Promise<Record<string, unknown>> {
-  const response = await fetchWithRetry(
-    FIREFLIES_GRAPHQL_URL,
-    {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${accessToken}`,
-      },
-      body: JSON.stringify({ query, variables }),
+  return retryWithExponentialBackoff(
+    async () => {
+      /** One retry layer owns transport, HTTP, and GraphQL semantic failures. */
+      const response = await fetch(FIREFLIES_GRAPHQL_URL, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: `Bearer ${accessToken}`,
+        },
+        body: JSON.stringify({ query, variables }),
+      })
+
+      /**
+       * Fireflies reports failures as an `errors` array in the body, and does so on
+       * non-2xx responses too (`object_not_found` → 404, `too_many_requests` → 429,
+       * `paid_required` → 403). Read the body first so the caller sees the actual
+       * reason instead of a bare status code.
+       */
+      let rawBody: string
+      try {
+        rawBody = await readResponseTextWithLimit(response, {
+          maxBytes: FIREFLIES_MAX_RESPONSE_BYTES,
+          label: 'Fireflies GraphQL response',
+        })
+      } catch (error) {
+        if (!response.ok && isPayloadSizeLimitError(error)) {
+          throw new FirefliesApiError(
+            `Fireflies API HTTP error: ${response.status} (response body exceeded the diagnostic limit)`,
+            undefined,
+            response.status,
+            resolveRetryDelayMs(response.headers)
+          )
+        }
+        throw error
+      }
+
+      let parsedBody: unknown = null
+      try {
+        parsedBody = JSON.parse(rawBody)
+      } catch {
+        parsedBody = null
+      }
+
+      const data = parsedBody as {
+        data?: Record<string, unknown> | null
+        errors?: FirefliesGraphQLError[]
+      } | null
+      const headerRetryAfterMs = resolveRetryDelayMs(response.headers)
+
+      const firstError = data?.errors?.[0]
+      if (firstError) {
+        const safeCode =
+          typeof firstError.code === 'string' && /^[A-Za-z0-9_.-]{1,64}$/.test(firstError.code)
+            ? firstError.code
+            : undefined
+        const code = safeCode ? ` (${safeCode})` : ''
+        const retryAt = firstError.extensions?.metadata?.retryAfter
+        const graphQLRetryAfterMs =
+          typeof retryAt === 'number' && retryAt > Date.now() ? retryAt - Date.now() : undefined
+        throw new FirefliesApiError(
+          `Fireflies API error${code}`,
+          safeCode,
+          firstError.extensions?.status ?? response.status,
+          graphQLRetryAfterMs ?? headerRetryAfterMs
+        )
+      }
+
+      if (!response.ok) {
+        throw new FirefliesApiError(
+          `Fireflies API HTTP error: ${response.status} (response body omitted)`,
+          undefined,
+          response.status,
+          headerRetryAfterMs
+        )
+      }
+
+      /**
+       * A 2xx carrying neither `errors` nor a `data` object is unreadable — an
+       * unparseable body, a truncated response, or a proxy interstitial. It must
+       * raise rather than degrade to an empty result: `listDocuments` would
+       * otherwise report a confident empty listing and reconcile every stored
+       * document as deleted. Retrying at the page boundary preserves prior pages.
+       */
+      if (!data || typeof data.data !== 'object' || data.data === null) {
+        throw new FirefliesMalformedResponseError(response.status)
+      }
+
+      return data.data
     },
-    retryOptions
+    {
+      maxRetries: retryOptions?.maxRetries ?? FIREFLIES_DEFAULT_MAX_RETRIES,
+      initialDelayMs: retryOptions?.initialDelayMs ?? 1000,
+      maxDelayMs: retryOptions?.maxDelayMs ?? FIREFLIES_DEFAULT_MAX_RETRY_DELAY_MS,
+      maxRetryAfterMs:
+        retryOptions?.maxRetryAfterMs ??
+        retryOptions?.maxDelayMs ??
+        FIREFLIES_DEFAULT_MAX_RETRY_DELAY_MS,
+      backoffMultiplier: retryOptions?.backoffMultiplier,
+      retryCondition: isRetryableFirefliesError,
+    }
   )
-
-  /**
-   * Fireflies reports failures as an `errors` array in the body, and does so on
-   * non-2xx responses too (`object_not_found` → 404, `too_many_requests` → 429,
-   * `paid_required` → 403). Read the body first so the caller sees the actual
-   * reason instead of a bare status code.
-   */
-  const data = (await response.json().catch(() => null)) as {
-    data?: Record<string, unknown> | null
-    errors?: { message?: string; code?: string }[]
-  } | null
-
-  const firstError = data?.errors?.[0]
-  if (firstError) {
-    const code = firstError.code ? ` (${firstError.code})` : ''
-    throw new FirefliesApiError(
-      `Fireflies API error${code}: ${firstError.message || 'Unknown GraphQL error'}`,
-      firstError.code
-    )
-  }
-
-  if (!response.ok) {
-    throw new Error(`Fireflies API HTTP error: ${response.status}`)
-  }
-
-  /**
-   * A 2xx carrying neither `errors` nor a `data` object is unreadable — an
-   * unparseable body, a truncated response, a proxy interstitial. It must raise
-   * rather than degrade to an empty result: `listDocuments` would otherwise
-   * report a confident empty listing and the sync engine would reconcile every
-   * stored document as deleted.
-   */
-  if (!data || typeof data.data !== 'object' || data.data === null) {
-    throw new Error('Fireflies API returned a malformed response with no data')
-  }
-
-  return data.data
 }
 
-/**
- * Formats transcript sentences into plain text content.
- */
 function formatTranscriptContent(transcript: FirefliesTranscript): string {
   const parts: string[] = []
+  let totalBytes = 0
+  const push = (part: string) => {
+    const addedBytes = Buffer.byteLength(part, 'utf8') + (parts.length > 0 ? 1 : 0)
+    if (totalBytes + addedBytes > FIREFLIES_MAX_EXTRACTED_CONTENT_BYTES) {
+      throw new ConnectorFileTooLargeError(FIREFLIES_MAX_EXTRACTED_CONTENT_BYTES)
+    }
+    parts.push(part)
+    totalBytes += addedBytes
+  }
 
   if (transcript.title) {
-    parts.push(`Meeting: ${transcript.title}`)
+    push(`Meeting: ${transcript.title}`)
   }
 
   if (transcript.date) {
-    parts.push(`Date: ${new Date(transcript.date).toISOString()}`)
+    push(`Date: ${new Date(transcript.date).toISOString()}`)
   }
 
   if (transcript.duration) {
-    parts.push(`Duration: ${Math.round(transcript.duration)} minutes`)
+    push(`Duration: ${Math.round(transcript.duration)} minutes`)
   }
 
   const host = transcript.host_email || transcript.organizer_email
   if (host) {
-    parts.push(`Host: ${host}`)
+    push(`Host: ${host}`)
   }
 
   if (transcript.participants && transcript.participants.length > 0) {
-    parts.push(`Participants: ${transcript.participants.join(', ')}`)
+    push(`Participants: ${transcript.participants.join(', ')}`)
   }
 
   const overview = transcript.summary?.overview || transcript.summary?.short_summary
   if (overview) {
-    parts.push('')
-    parts.push('--- Overview ---')
-    parts.push(overview)
+    push('')
+    push('--- Overview ---')
+    push(overview)
   }
 
   if (transcript.summary?.action_items) {
-    parts.push('')
-    parts.push('--- Action Items ---')
-    parts.push(transcript.summary.action_items)
+    push('')
+    push('--- Action Items ---')
+    push(transcript.summary.action_items)
   }
 
-  if (transcript.summary?.keywords && transcript.summary.keywords.length > 0) {
-    parts.push('')
-    parts.push(`Keywords: ${transcript.summary.keywords.join(', ')}`)
+  const keywords = transcript.summary?.keywords
+  const formattedKeywords = Array.isArray(keywords)
+    ? keywords.filter((keyword): keyword is string => typeof keyword === 'string').join(', ')
+    : typeof keywords === 'string'
+      ? keywords
+      : ''
+  if (formattedKeywords) {
+    push('')
+    push(`Keywords: ${formattedKeywords}`)
   }
 
   if (transcript.sentences && transcript.sentences.length > 0) {
-    parts.push('')
-    parts.push('--- Transcript ---')
+    push('')
+    push('--- Transcript ---')
     for (const sentence of transcript.sentences) {
-      parts.push(`${sentence.speaker_name}: ${sentence.text}`)
+      push(`${sentence.speaker_name}: ${sentence.text}`)
     }
   }
 
@@ -166,9 +324,21 @@ function formatTranscriptContent(transcript: FirefliesTranscript): string {
  * `getDocument`, so the metadata-derived `contentHash` is byte-identical on both
  * paths and a hydrated transcript is never seen as changed.
  */
-function transcriptToStub(transcript: FirefliesTranscript): ExternalDocument {
+async function transcriptToStub(transcript: FirefliesTranscript): Promise<ExternalDocument> {
   const meetingDate = transcript.date ? new Date(transcript.date).toISOString() : undefined
   const speakerNames = transcript.speakers?.map((s) => s.name).filter(Boolean) ?? []
+  const lifecycleHash = await computeContentHash(
+    JSON.stringify({
+      date: transcript.date ?? null,
+      duration: transcript.duration ?? null,
+      title: transcript.title ?? null,
+      host: transcript.host_email || transcript.organizer_email || null,
+      participants: transcript.participants ?? [],
+      speakers: speakerNames,
+      isLive: transcript.is_live ?? null,
+      summaryStatus: transcript.meeting_info?.summary_status ?? null,
+    })
+  )
 
   return {
     externalId: transcript.id,
@@ -177,7 +347,7 @@ function transcriptToStub(transcript: FirefliesTranscript): ExternalDocument {
     contentDeferred: true,
     mimeType: 'text/plain',
     sourceUrl: transcript.transcript_url || undefined,
-    contentHash: `fireflies:${transcript.id}:${transcript.date ?? ''}:${transcript.duration ?? ''}`,
+    contentHash: `fireflies:v2:${transcript.id}:${lifecycleHash}`,
     metadata: {
       hostEmail: transcript.host_email || transcript.organizer_email,
       duration: transcript.duration,
@@ -188,6 +358,20 @@ function transcriptToStub(transcript: FirefliesTranscript): ExternalDocument {
   }
 }
 
+function oversizedTranscriptResponseStub(externalId: string): ExternalDocument {
+  return markSkipped(
+    {
+      externalId,
+      title: `Fireflies transcript ${externalId}`,
+      content: '',
+      contentDeferred: true,
+      mimeType: 'text/plain',
+      contentHash: `fireflies:oversized-response:${externalId}`,
+    },
+    FIREFLIES_RESPONSE_SKIP_REASON
+  )
+}
+
 export const firefliesConnector: ConnectorConfig = {
   ...firefliesConnectorMeta,
 
@@ -198,9 +382,9 @@ export const firefliesConnector: ConnectorConfig = {
     syncContext?: Record<string, unknown>
   ): Promise<ExternalDocumentList> => {
     const hostEmail = (sourceConfig.hostEmail as string) || ''
-    const maxTranscripts = sourceConfig.maxTranscripts ? Number(sourceConfig.maxTranscripts) : 0
+    const maxTranscripts = parseMaxTranscripts(sourceConfig.maxTranscripts)
 
-    const skip = cursor ? Number(cursor) : 0
+    const skip = parsePaginationCursor(cursor)
     const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0
 
     /**
@@ -214,7 +398,9 @@ export const firefliesConnector: ConnectorConfig = {
     let listingCeiling = syncContext?.firefliesListingCeiling as string | undefined
     if (!listingCeiling) {
       listingCeiling = new Date().toISOString()
-      if (syncContext) syncContext.firefliesListingCeiling = listingCeiling
+      if (syncContext) {
+        syncContext.firefliesListingCeiling = listingCeiling
+      }
     }
 
     /**
@@ -268,27 +454,36 @@ export const firefliesConnector: ConnectorConfig = {
           speakers {
             name
           }
+          is_live
+          meeting_info {
+            summary_status
+          }
         }
       }`,
       variables
     )
 
-    const transcripts = (
-      Array.isArray(data.transcripts) ? data.transcripts : []
-    ) as FirefliesTranscript[]
+    if (!Array.isArray(data.transcripts)) {
+      throw new Error('Fireflies API returned malformed transcript-list data')
+    }
+    if (!data.transcripts.every(isFirefliesTranscript)) {
+      throw new Error('Fireflies API returned malformed transcript metadata')
+    }
+    const transcripts = data.transcripts as FirefliesTranscript[]
 
-    const allStubs = transcripts.filter((t) => Boolean(t?.id)).map(transcriptToStub)
+    const allStubs = await Promise.all(
+      transcripts.filter((t) => Boolean(t?.id)).map(transcriptToStub)
+    )
     const documents = maxTranscripts > 0 ? allStubs.slice(0, remaining) : allStubs
 
     const totalFetched = prevFetched + documents.length
     if (syncContext) syncContext.totalDocsFetched = totalFetched
 
     /**
-     * `listingCapped` blocks the sync engine's deletion reconciliation, so it is
-     * set only when the cap actually hid transcripts that still exist — either
-     * the probe row came back, or the page came back full. A cap that lands
-     * exactly on source exhaustion leaves a short page and stays reconcilable,
-     * otherwise deleted meetings could never be removed from the KB.
+     * Record a configured cap only when it actually hid transcripts — either the
+     * probe row came back, or the page came back full. The separate
+     * `reconciliationSafe: false` result reflects the provider's offset pagination
+     * regardless of whether this user-configured cap was reached.
      */
     const moreAvailable = allStubs.length > documents.length || transcripts.length === pageSize
     const hitLimit = maxTranscripts > 0 && totalFetched >= maxTranscripts
@@ -306,6 +501,12 @@ export const firefliesConnector: ConnectorConfig = {
        */
       nextCursor: hasMore ? String(skip + transcripts.length) : undefined,
       hasMore,
+      /**
+       * Fireflies exposes offset pagination without a documented stable sort or
+       * snapshot cursor. `toDate` excludes new transcripts, but a deletion during
+       * the walk can still shift later offsets and make a live item appear absent.
+       */
+      reconciliationSafe: false,
     }
   },
 
@@ -330,6 +531,10 @@ export const firefliesConnector: ConnectorConfig = {
             speakers {
               name
             }
+            is_live
+            meeting_info {
+              summary_status
+            }
             sentences {
               speaker_name
               text
@@ -345,14 +550,25 @@ export const firefliesConnector: ConnectorConfig = {
         { id: externalId }
       )
 
-      const transcript = data.transcript as FirefliesTranscript | null
-      if (!transcript?.id) return null
+      const transcript = data.transcript
+      if (!isFirefliesTranscript(transcript) || transcript.id !== externalId) {
+        throw new Error('Fireflies API returned malformed transcript metadata')
+      }
 
-      const stub = transcriptToStub(transcript)
+      const stub = await transcriptToStub(transcript)
+      let content: string
+      try {
+        content = formatTranscriptContent(transcript)
+      } catch (error) {
+        if (error instanceof ConnectorFileTooLargeError) {
+          return markSkipped(stub, FIREFLIES_EXTRACTED_CONTENT_SKIP_REASON)
+        }
+        throw error
+      }
 
       return {
         ...stub,
-        content: formatTranscriptContent(transcript),
+        content,
         contentDeferred: false,
         metadata: {
           ...stub.metadata,
@@ -360,6 +576,13 @@ export const firefliesConnector: ConnectorConfig = {
         },
       }
     } catch (error) {
+      if (isPayloadSizeLimitError(error)) {
+        logger.info('Skipping Fireflies transcript with oversized hydration response', {
+          externalId,
+          maxResponseBytes: FIREFLIES_MAX_RESPONSE_BYTES,
+        })
+        return oversizedTranscriptResponseStub(externalId)
+      }
       /**
        * Only `object_not_found` means the transcript is genuinely gone. Every other
        * failure — `too_many_requests`, `paid_required`, transport faults — is rethrown so
@@ -382,9 +605,10 @@ export const firefliesConnector: ConnectorConfig = {
     accessToken: string,
     sourceConfig: Record<string, unknown>
   ): Promise<{ valid: boolean; error?: string }> => {
-    const maxTranscripts = sourceConfig.maxTranscripts as string | undefined
-    if (maxTranscripts && (Number.isNaN(Number(maxTranscripts)) || Number(maxTranscripts) < 0)) {
-      return { valid: false, error: 'Max transcripts must be a non-negative number' }
+    try {
+      parseMaxTranscripts(sourceConfig.maxTranscripts)
+    } catch (error) {
+      return { valid: false, error: toError(error).message }
     }
 
     try {

apps/sim/connectors/fireflies/meta.ts

@@ -14,6 +14,12 @@ export const firefliesConnectorMeta: ConnectorMeta = {
     placeholder: 'Enter your Fireflies API key',
   },
 
+  /**
+   * Fireflies exposes no transcript modification timestamp, so an explicit
+   * full resync must rehydrate content even when list metadata is unchanged.
+   */
+  rehydrateOnFullSync: true,
+
   configFields: [
     {
       id: 'hostEmail',

apps/sim/connectors/google-docs/google-docs.test.ts

@@ -0,0 +1,707 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/components/icons', () => ({
+  GoogleDocsIcon: () => null,
+}))
+
+import { googleDocsConnector } from '@/connectors/google-docs/google-docs'
+
+const ACCESS_TOKEN = 'token-123'
+const DOCUMENT_ID = 'document-abc'
+const DRIVE_FILE = {
+  id: DOCUMENT_ID,
+  name: 'Product plan',
+  mimeType: 'application/vnd.google-apps.document',
+  modifiedTime: '2026-08-24T12:00:00.000Z',
+  createdTime: '2026-08-01T12:00:00.000Z',
+  webViewLink: `https://docs.google.com/document/d/${DOCUMENT_ID}/edit`,
+  owners: [{ displayName: 'Ada Lovelace' }],
+}
+
+function stubFetchDocument(docsResponse: Response) {
+  const fetchMock = vi.fn(async (input: string | URL | Request) => {
+    const url = new URL(typeof input === 'string' ? input : input.toString())
+
+    if (url.hostname === 'www.googleapis.com') {
+      return new Response(JSON.stringify({ ...DRIVE_FILE, trashed: false }), { status: 200 })
+    }
+    if (url.hostname === 'docs.googleapis.com') return docsResponse
+
+    throw new Error(`Unexpected fetch to ${url.toString()}`)
+  })
+
+  vi.stubGlobal('fetch', fetchMock)
+  return fetchMock
+}
+
+describe('googleDocsConnector', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  describe('getDocument', () => {
+    it.each([{}, { ...DRIVE_FILE, id: 'different-document' }])(
+      'rejects malformed Drive metadata instead of replacing retained content',
+      async (metadata) => {
+        vi.stubGlobal(
+          'fetch',
+          vi.fn(async () => new Response(JSON.stringify(metadata), { status: 200 }))
+        )
+
+        await expect(
+          googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+        ).rejects.toThrow('Google Drive API returned malformed file metadata')
+      }
+    )
+
+    it('authoritatively skips a listed document that changed type', async () => {
+      vi.stubGlobal(
+        'fetch',
+        vi.fn(
+          async () =>
+            new Response(
+              JSON.stringify({ ...DRIVE_FILE, mimeType: 'application/pdf', trashed: false }),
+              { status: 200 }
+            )
+        )
+      )
+
+      await expect(
+        googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+      ).resolves.toMatchObject({
+        content: '',
+        skippedReason: 'File is no longer a Google Doc',
+        skippedExistingDisposition: 'replace',
+      })
+    })
+
+    it('requests the tab response view and extracts nested tab content', async () => {
+      const fetchMock = stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            tabs: [
+              {
+                documentTab: {
+                  body: {
+                    content: [
+                      {
+                        paragraph: {
+                          paragraphStyle: { namedStyleType: 'HEADING_1' },
+                          elements: [{ textRun: { content: 'Overview\n' } }],
+                        },
+                      },
+                      {
+                        table: {
+                          tableRows: [
+                            {
+                              tableCells: [
+                                {
+                                  content: [
+                                    {
+                                      paragraph: {
+                                        elements: [{ textRun: { content: 'Table cell\n' } }],
+                                      },
+                                    },
+                                  ],
+                                },
+                              ],
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                },
+                childTabs: [
+                  {
+                    documentTab: {
+                      body: {
+                        content: [
+                          {
+                            paragraph: {
+                              elements: [
+                                {
+                                  richLink: {
+                                    richLinkProperties: {
+                                      title: 'Linked specification',
+                                      uri: 'https://example.com/spec',
+                                    },
+                                  },
+                                },
+                              ],
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  },
+                ],
+              },
+            ],
+          }),
+          { status: 200 }
+        )
+      )
+
+      const document = await googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+
+      expect(document?.content).toBe('# Overview\nTable cell\nLinked specification')
+      expect(document?.contentDeferred).toBe(false)
+
+      const docsCall = fetchMock.mock.calls.find(([input]) =>
+        input.toString().startsWith('https://docs.googleapis.com/')
+      )
+      expect(docsCall).toBeDefined()
+
+      const docsUrl = new URL(docsCall?.[0].toString() ?? '')
+      expect(docsUrl.pathname).toBe(`/v1/documents/${DOCUMENT_ID}`)
+      expect(docsUrl.searchParams.get('includeTabsContent')).toBe('true')
+      expect(docsUrl.searchParams.get('fields')).toBe('tabs')
+    })
+
+    it('omits the structured Google API error message when hydration fails', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            error: {
+              code: 400,
+              message: 'Invalid field selection tabs',
+              status: 'INVALID_ARGUMENT',
+            },
+          }),
+          { status: 400 }
+        )
+      )
+
+      await expect(googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)).rejects.toThrow(
+        `Failed to fetch Google Doc content ${DOCUMENT_ID}: 400`
+      )
+    })
+
+    it('omits a structured Google API error message from an envelope larger than 2KB', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            padding: 'x'.repeat(3000),
+            error: {
+              code: 400,
+              message: 'Invalid field selection tabs',
+              status: 'INVALID_ARGUMENT',
+            },
+          }),
+          { status: 400 }
+        )
+      )
+
+      await expect(googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)).rejects.toThrow(
+        `Failed to fetch Google Doc content ${DOCUMENT_ID}: 400`
+      )
+    })
+
+    it('omits credentials from bounded Google API diagnostics', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            error: {
+              code: 403,
+              message: 'Authorization: Bearer production-secret',
+              status: 'PERMISSION_DENIED',
+            },
+          }),
+          { status: 403 }
+        )
+      )
+
+      await expect(googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)).rejects.toThrow(
+        `Failed to fetch Google Doc content ${DOCUMENT_ID}: 403`
+      )
+    })
+
+    it('redacts canonical credential keys from an unexpected Google error envelope', async () => {
+      const secret = 'opaque-client-secret-that-must-not-escape'
+      stubFetchDocument(
+        new Response(JSON.stringify({ metadata: { client_secret: secret } }), { status: 400 })
+      )
+
+      const error = await googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID).then(
+        () => undefined,
+        (caught) => caught as Error
+      )
+
+      expect(error?.message).toBe(`Failed to fetch Google Doc content ${DOCUMENT_ID}: 400`)
+      expect(error?.message).not.toContain(secret)
+    })
+
+    it('returns only the status when an error envelope exceeds the diagnostic limit', async () => {
+      const secret = 'over-limit-provider-secret-that-must-not-escape'
+      stubFetchDocument(
+        new Response(`${'x'.repeat(64 * 1024)}${secret}`, {
+          status: 400,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      )
+
+      const error = await googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID).then(
+        () => undefined,
+        (caught) => caught as Error
+      )
+
+      expect(error?.message).toBe(`Failed to fetch Google Doc content ${DOCUMENT_ID}: 400`)
+      expect(error?.message).not.toContain('response body omitted')
+      expect(error?.message).not.toContain(secret)
+    })
+
+    it('marks an empty tab response as an authoritative skip', async () => {
+      stubFetchDocument(new Response(JSON.stringify({ tabs: [] }), { status: 200 }))
+
+      await expect(
+        googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+      ).resolves.toMatchObject({
+        content: '',
+        contentDeferred: false,
+        skippedExistingDisposition: 'replace',
+        skippedReason: 'Document contains no extractable text',
+      })
+    })
+
+    it('rejects a successful response that omits the required tabs array', async () => {
+      stubFetchDocument(new Response('{}', { status: 200 }))
+
+      await expect(googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)).rejects.toThrow(
+        'Google Docs API returned a malformed document response'
+      )
+    })
+
+    it('rejects an empty tab object instead of replacing retained content', async () => {
+      stubFetchDocument(new Response(JSON.stringify({ tabs: [{}] }), { status: 200 }))
+
+      await expect(googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)).rejects.toThrow(
+        'Google Docs API returned a malformed document response'
+      )
+    })
+
+    it('rejects malformed nested tab content instead of dropping it', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            tabs: [
+              {
+                documentTab: { body: { content: [] } },
+                childTabs: [{}],
+              },
+            ],
+          }),
+          { status: 200 }
+        )
+      )
+
+      await expect(googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)).rejects.toThrow(
+        'Google Docs API returned a malformed document response'
+      )
+    })
+
+    it('maps an oversized chunked hydration response to a visible skip', async () => {
+      const chunk = new Uint8Array(34 * 1024 * 1024)
+      let chunksSent = 0
+      let streamCancelled = false
+      const stream = new ReadableStream<Uint8Array>({
+        pull(controller) {
+          chunksSent += 1
+          controller.enqueue(chunk)
+        },
+        cancel() {
+          streamCancelled = true
+        },
+      })
+      stubFetchDocument(new Response(stream, { status: 200 }))
+
+      await expect(
+        googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+      ).resolves.toMatchObject({
+        externalId: DOCUMENT_ID,
+        content: '',
+        contentDeferred: false,
+        skippedReason: 'File exceeds the 100MB size limit and was not indexed',
+      })
+      expect(chunksSent).toBeGreaterThanOrEqual(3)
+      expect(streamCancelled).toBe(true)
+    })
+
+    it('extracts headers, footers, and footnotes from every tab', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            tabs: [
+              {
+                documentTab: {
+                  body: { content: [] },
+                  headers: {
+                    header1: {
+                      content: [
+                        { paragraph: { elements: [{ textRun: { content: 'Header text\n' } }] } },
+                      ],
+                    },
+                    dateHeader: {
+                      content: [
+                        {
+                          paragraph: {
+                            elements: [
+                              {
+                                dateElement: {
+                                  dateElementProperties: { displayText: 'Aug 24, 2026' },
+                                },
+                              },
+                            ],
+                          },
+                        },
+                      ],
+                    },
+                  },
+                  footers: {
+                    footer1: {
+                      content: [
+                        { paragraph: { elements: [{ textRun: { content: 'Footer text\n' } }] } },
+                      ],
+                    },
+                  },
+                  footnotes: {
+                    footnote1: {
+                      content: [
+                        {
+                          paragraph: { elements: [{ textRun: { content: 'Footnote text\n' } }] },
+                        },
+                      ],
+                    },
+                  },
+                },
+              },
+              {
+                documentTab: {
+                  body: { content: [] },
+                  headers: {
+                    siblingHeader: {
+                      content: [
+                        {
+                          paragraph: {
+                            elements: [{ textRun: { content: 'Sibling header\n' } }],
+                          },
+                        },
+                      ],
+                    },
+                  },
+                  footnotes: {
+                    siblingFootnote: {
+                      content: [
+                        {
+                          paragraph: {
+                            elements: [{ textRun: { content: 'Sibling footnote\n' } }],
+                          },
+                        },
+                      ],
+                    },
+                  },
+                },
+              },
+            ],
+          }),
+          { status: 200 }
+        )
+      )
+
+      await expect(
+        googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+      ).resolves.toMatchObject({
+        content:
+          'Header text\nAug 24, 2026\nFooter text\nFootnote text\nSibling header\nSibling footnote',
+        contentDeferred: false,
+      })
+    })
+
+    it('does not authoritatively replace prior content for an embedded-object-only doc', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            tabs: [
+              {
+                documentTab: {
+                  body: { content: [] },
+                  inlineObjects: { image1: { inlineObjectProperties: {} } },
+                },
+              },
+            ],
+          }),
+          { status: 200 }
+        )
+      )
+
+      const document = await googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+
+      expect(document).toMatchObject({
+        content: '',
+        contentDeferred: false,
+        skippedReason: 'Document contains non-text elements but no extractable text',
+      })
+      expect(document?.skippedExistingDisposition).toBeUndefined()
+    })
+
+    it('does not authoritatively replace prior content for an equation-only doc', async () => {
+      stubFetchDocument(
+        new Response(
+          JSON.stringify({
+            tabs: [
+              {
+                documentTab: {
+                  body: {
+                    content: [{ paragraph: { elements: [{ equation: {} }] } }],
+                  },
+                },
+              },
+            ],
+          }),
+          { status: 200 }
+        )
+      )
+
+      const document = await googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+
+      expect(document).toMatchObject({
+        content: '',
+        skippedReason: 'Document contains non-text elements but no extractable text',
+      })
+      expect(document?.skippedExistingDisposition).toBeUndefined()
+    })
+
+    it('keeps the metadata hash identical between listing and hydration', async () => {
+      let driveRequestCount = 0
+      vi.stubGlobal(
+        'fetch',
+        vi.fn(async (input: string | URL | Request) => {
+          const url = new URL(typeof input === 'string' ? input : input.toString())
+          if (url.hostname === 'www.googleapis.com' && url.pathname === '/drive/v3/files') {
+            return new Response(JSON.stringify({ files: [DRIVE_FILE] }), { status: 200 })
+          }
+          if (url.hostname === 'www.googleapis.com') {
+            driveRequestCount += 1
+            return new Response(JSON.stringify({ ...DRIVE_FILE, trashed: false }), { status: 200 })
+          }
+          if (url.hostname === 'docs.googleapis.com') {
+            return new Response(
+              JSON.stringify({
+                tabs: [
+                  {
+                    documentTab: {
+                      body: {
+                        content: [
+                          { paragraph: { elements: [{ textRun: { content: 'Content\n' } }] } },
+                        ],
+                      },
+                    },
+                  },
+                ],
+              }),
+              { status: 200 }
+            )
+          }
+          throw new Error(`Unexpected fetch to ${url.toString()}`)
+        })
+      )
+
+      const listing = await googleDocsConnector.listDocuments(ACCESS_TOKEN, {})
+      const hydrated = await googleDocsConnector.getDocument(ACCESS_TOKEN, {}, DOCUMENT_ID)
+
+      expect(listing.documents[0]?.contentDeferred).toBe(true)
+      expect(hydrated?.contentHash).toBe(listing.documents[0]?.contentHash)
+      expect(driveRequestCount).toBe(1)
+    })
+  })
+
+  describe('listDocuments', () => {
+    it('rejects a malformed successful Drive list envelope', async () => {
+      vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}', { status: 200 })))
+
+      await expect(
+        googleDocsConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {})
+      ).rejects.toThrow('Google Drive API returned malformed file-list metadata')
+    })
+
+    it('accepts a discriminator-only empty Drive list envelope', async () => {
+      vi.stubGlobal(
+        'fetch',
+        vi
+          .fn()
+          .mockResolvedValue(
+            new Response(JSON.stringify({ kind: 'drive#fileList' }), { status: 200 })
+          )
+      )
+
+      await expect(
+        googleDocsConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {})
+      ).resolves.toMatchObject({ documents: [], hasMore: false })
+    })
+
+    it.each([
+      {
+        files: [{ name: 'Missing ID', mimeType: DRIVE_FILE.mimeType, modifiedTime: '2026-01-01' }],
+      },
+      { files: [], nextPageToken: 123 },
+      { files: [], incompleteSearch: 'true' },
+    ])('rejects malformed Drive list metadata', async (body) => {
+      vi.stubGlobal(
+        'fetch',
+        vi.fn().mockResolvedValue(new Response(JSON.stringify(body), { status: 200 }))
+      )
+
+      await expect(
+        googleDocsConnector.listDocuments(ACCESS_TOKEN, {}, undefined, {})
+      ).rejects.toThrow('Google Drive API returned malformed file-list metadata')
+    })
+
+    it('does not issue another Drive request after a lowered cap is already exhausted', async () => {
+      const fetchMock = vi.fn()
+      vi.stubGlobal('fetch', fetchMock)
+      const syncContext: Record<string, unknown> = { totalDocsFetched: 5 }
+
+      const result = await googleDocsConnector.listDocuments(
+        ACCESS_TOKEN,
+        { maxDocs: '2' },
+        'stale-page-token',
+        syncContext
+      )
+
+      expect(result).toEqual({ documents: [], hasMore: false })
+      expect(syncContext.listingCapped).toBe(true)
+      expect(fetchMock).not.toHaveBeenCalled()
+    })
+
+    it('suppresses reconciliation when Drive reports an incomplete search', async () => {
+      const fetchMock = vi.fn().mockResolvedValue(
+        new Response(
+          JSON.stringify({
+            files: [DRIVE_FILE],
+            incompleteSearch: true,
+          }),
+          { status: 200 }
+        )
+      )
+      vi.stubGlobal('fetch', fetchMock)
+      const syncContext: Record<string, unknown> = {}
+
+      const result = await googleDocsConnector.listDocuments(
+        ACCESS_TOKEN,
+        {},
+        undefined,
+        syncContext
+      )
+
+      expect(result.documents.map((document) => document.externalId)).toEqual([DOCUMENT_ID])
+      expect(result.hasMore).toBe(false)
+      expect(result.reconciliationSafe).toBe(false)
+      expect(syncContext.listingCapped).toBe(true)
+    })
+
+    it('keeps an exactly exhausted cap authoritative when Drive is exhausted', async () => {
+      const files = [DRIVE_FILE, { ...DRIVE_FILE, id: 'document-def', name: 'Launch plan' }]
+      const fetchMock = vi
+        .fn()
+        .mockResolvedValue(new Response(JSON.stringify({ files }), { status: 200 }))
+      vi.stubGlobal('fetch', fetchMock)
+      const syncContext: Record<string, unknown> = {}
+
+      const result = await googleDocsConnector.listDocuments(
+        ACCESS_TOKEN,
+        { maxDocs: '2' },
+        undefined,
+        syncContext
+      )
+
+      expect(result.documents.map((document) => document.externalId)).toEqual([
+        DOCUMENT_ID,
+        'document-def',
+      ])
+      expect(result).toMatchObject({ hasMore: false })
+      expect(result.nextCursor).toBeUndefined()
+      expect(syncContext.listingCapped).toBeUndefined()
+      expect(new URL(String(fetchMock.mock.calls[0][0])).searchParams.get('pageSize')).toBe('2')
+    })
+
+    it('suppresses reconciliation when the cap is reached with another Drive page', async () => {
+      const files = [DRIVE_FILE, { ...DRIVE_FILE, id: 'document-def', name: 'Launch plan' }]
+      const fetchMock = vi.fn().mockResolvedValue(
+        new Response(
+          JSON.stringify({
+            files,
+            nextPageToken: 'next-page-token',
+          }),
+          { status: 200 }
+        )
+      )
+      vi.stubGlobal('fetch', fetchMock)
+      const syncContext: Record<string, unknown> = {}
+
+      const result = await googleDocsConnector.listDocuments(
+        ACCESS_TOKEN,
+        { maxDocs: '2' },
+        undefined,
+        syncContext
+      )
+
+      expect(result.documents).toHaveLength(2)
+      expect(result.hasMore).toBe(false)
+      expect(result.nextCursor).toBeUndefined()
+      expect(syncContext.listingCapped).toBe(true)
+    })
+
+    it.each(['1.5', 'Infinity', 1.5, Number.POSITIVE_INFINITY])(
+      'rejects invalid persisted maxDocs %s before calling Google Drive',
+      async (maxDocs) => {
+        const fetchMock = vi.fn()
+        vi.stubGlobal('fetch', fetchMock)
+
+        await expect(googleDocsConnector.listDocuments(ACCESS_TOKEN, { maxDocs })).rejects.toThrow(
+          'Max documents must be a positive safe integer, or 0 for unlimited'
+        )
+        expect(fetchMock).not.toHaveBeenCalled()
+      }
+    )
+
+    it.each([undefined, null, '', '   ', 0, '0'])(
+      'keeps omitted or explicit unlimited maxDocs %s valid at runtime',
+      async (maxDocs) => {
+        const fetchMock = vi
+          .fn()
+          .mockResolvedValue(new Response(JSON.stringify({ files: [] }), { status: 200 }))
+        vi.stubGlobal('fetch', fetchMock)
+
+        await expect(
+          googleDocsConnector.listDocuments(ACCESS_TOKEN, { maxDocs })
+        ).resolves.toMatchObject({ documents: [], hasMore: false })
+        expect(new URL(String(fetchMock.mock.calls[0][0])).searchParams.get('pageSize')).toBe('100')
+      }
+    )
+  })
+
+  describe('validateConfig', () => {
+    it.each(['1.5', 'Infinity', 1.5, Number.POSITIVE_INFINITY])(
+      'rejects invalid maxDocs %s before calling Google Drive',
+      async (maxDocs) => {
+        const fetchMock = vi.fn()
+        vi.stubGlobal('fetch', fetchMock)
+
+        const result = await googleDocsConnector.validateConfig(ACCESS_TOKEN, { maxDocs })
+
+        expect(result).toEqual({
+          valid: false,
+          error: 'Max documents must be a positive safe integer, or 0 for unlimited',
+        })
+        expect(fetchMock).not.toHaveBeenCalled()
+      }
+    )
+  })
+})

apps/sim/connectors/google-docs/google-docs.ts

@@ -1,6 +1,11 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
-import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { isPlainRecord } from '@sim/utils/object'
+import {
+  fetchWithRetry,
+  readBoundedHttpErrorBody,
+  VALIDATE_RETRY_OPTIONS,
+} from '@/lib/knowledge/documents/utils'
 import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import {
@@ -9,6 +14,7 @@ import {
   joinTagArray,
   markSkipped,
   parseMultiValue,
+  parseOptionalUnlimitedSafeInteger,
   parseTagDate,
   readBodyWithLimit,
   sizeLimitSkipReason,
@@ -30,6 +36,19 @@ const MAX_DOCS_RESPONSE_BYTES = 100 * 1024 * 1024
 const PAGE_SIZE = 100
 
 const GOOGLE_DOC_MIME_TYPE = 'application/vnd.google-apps.document'
+const MAX_DOCS_VALIDATION_ERROR =
+  'Max documents must be a positive safe integer, or 0 for unlimited'
+
+function parseMaxDocs(value: unknown): number {
+  return parseOptionalUnlimitedSafeInteger(value, MAX_DOCS_VALIDATION_ERROR)
+}
+
+/**
+ * `includeTabsContent=true` switches the Docs response to the tab representation.
+ * Request only that representation because the legacy top-level `body` is empty
+ * in tab mode. Google accepts `fields=tabs`.
+ */
+const GOOGLE_DOC_CONTENT_FIELDS = 'tabs'
 
 /**
  * Represents a Google Drive file entry returned by the Drive API.
@@ -44,6 +63,40 @@ interface DriveFile {
   owners?: { displayName?: string; emailAddress?: string }[]
 }
 
+function isDriveFileMetadata(
+  value: unknown,
+  expectedId: string
+): value is DriveFile & { trashed?: boolean } {
+  return (
+    isPlainRecord(value) &&
+    value.id === expectedId &&
+    typeof value.name === 'string' &&
+    typeof value.mimeType === 'string' &&
+    (value.trashed === undefined || typeof value.trashed === 'boolean')
+  )
+}
+
+function isDriveFileListItem(value: unknown): value is DriveFile {
+  return (
+    isPlainRecord(value) &&
+    typeof value.id === 'string' &&
+    value.id.length > 0 &&
+    typeof value.name === 'string' &&
+    typeof value.mimeType === 'string' &&
+    typeof value.modifiedTime === 'string'
+  )
+}
+
+function parseDriveFileMetadata(
+  value: unknown,
+  expectedId: string
+): DriveFile & { trashed?: boolean } {
+  if (!isDriveFileMetadata(value, expectedId)) {
+    throw new Error('Google Drive API returned malformed file metadata')
+  }
+  return value
+}
+
 /**
  * A single element inside a paragraph. Only the variants that carry readable
  * text are modeled — `pageBreak`, `columnBreak`, `horizontalRule`, `equation`,
@@ -53,6 +106,9 @@ interface DocsParagraphElement {
   textRun?: { content?: string }
   richLink?: { richLinkProperties?: { title?: string; uri?: string } }
   person?: { personProperties?: { name?: string; email?: string } }
+  dateElement?: { dateElementProperties?: { displayText?: string } }
+  equation?: Record<string, unknown>
+  inlineObjectElement?: { inlineObjectId?: string }
 }
 
 /**
@@ -81,29 +137,93 @@ interface DocsStructuralElement {
   }
 }
 
+interface DocsContentRegion {
+  content?: DocsStructuralElement[]
+}
+
 /**
- * A tab of a Google Doc. Tabs may nest arbitrarily deep via `childTabs`.
+ * A Google Doc tab. Tabs may nest through `childTabs`; non-text objects are
+ * tracked separately so an object-only document is not mistaken for empty text.
  */
 interface DocsTab {
   documentTab?: {
-    body?: { content?: DocsStructuralElement[] }
+    body?: DocsContentRegion
+    headers?: Record<string, DocsContentRegion>
+    footers?: Record<string, DocsContentRegion>
+    footnotes?: Record<string, DocsContentRegion>
+    inlineObjects?: Record<string, unknown>
+    positionedObjects?: Record<string, unknown>
   }
   childTabs?: DocsTab[]
 }
 
 /**
- * Represents the response from the Google Docs API for a single document. With
- * `includeTabsContent=true` the content lands in `tabs` and the legacy `body`
- * field is left empty; `body` is retained only as a fallback in case the request
- * is ever served without tab content.
+ * Represents the tab-based response from the Google Docs API for one document.
+ * With `includeTabsContent=true`, the legacy top-level text fields are empty.
  */
 interface DocsDocument {
-  body?: {
-    content?: DocsStructuralElement[]
-  }
   tabs?: DocsTab[]
 }
 
+function isDocsTab(value: unknown): value is DocsTab {
+  if (!isPlainRecord(value)) return false
+  if (value.childTabs !== undefined) {
+    if (!Array.isArray(value.childTabs) || !value.childTabs.every(isDocsTab)) return false
+  }
+  if (value.documentTab !== undefined) {
+    if (!isPlainRecord(value.documentTab)) return false
+    const body = value.documentTab.body
+    if (
+      body !== undefined &&
+      (!isPlainRecord(body) || (body.content !== undefined && !Array.isArray(body.content)))
+    ) {
+      return false
+    }
+    return true
+  }
+  return Array.isArray(value.childTabs) && value.childTabs.length > 0
+}
+
+function parseDriveFileListResponse(value: unknown): {
+  files: DriveFile[]
+  incompleteSearch: boolean
+  nextPageToken?: string
+} {
+  if (!isPlainRecord(value)) {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  const rawFiles = value.files
+  if (rawFiles === undefined && value.kind !== 'drive#fileList') {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  if (
+    rawFiles !== undefined &&
+    (!Array.isArray(rawFiles) || rawFiles.some((file) => !isDriveFileListItem(file)))
+  ) {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  if (
+    value.nextPageToken !== undefined &&
+    (typeof value.nextPageToken !== 'string' || value.nextPageToken.length === 0)
+  ) {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  if (value.incompleteSearch !== undefined && typeof value.incompleteSearch !== 'boolean') {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  return {
+    files: rawFiles ?? [],
+    incompleteSearch: value.incompleteSearch === true,
+    ...(typeof value.nextPageToken === 'string' ? { nextPageToken: value.nextPageToken } : {}),
+  }
+}
+
+/** Describes a Google API failure without leaking an unbounded response body. */
+async function describeGoogleApiFailure(response: Response): Promise<string> {
+  await readBoundedHttpErrorBody(response)
+  return String(response.status)
+}
+
 /**
  * Maps a Google Docs heading style to a Markdown heading prefix.
  */
@@ -141,6 +261,9 @@ function paragraphElementText(element: DocsParagraphElement): string {
     const { name, email } = element.person.personProperties
     return name || email || ''
   }
+  if (element.dateElement?.dateElementProperties?.displayText) {
+    return element.dateElement.dateElementProperties.displayText
+  }
   return ''
 }
 
@@ -189,31 +312,99 @@ function extractTextFromStructuralElements(elements: DocsStructuralElement[]): s
  * Collects the text of a tab and every descendant tab, depth-first in the order
  * the Docs API returns them.
  */
-function extractTextFromTabs(tabs: DocsTab[]): string[] {
-  const parts: string[] = []
+interface DocsExtraction {
+  parts: string[]
+  hasUnresolvedContent: boolean
+}
 
-  for (const tab of tabs) {
-    const content = tab.documentTab?.body?.content
-    if (content) parts.push(...extractTextFromStructuralElements(content))
-    if (tab.childTabs?.length) parts.push(...extractTextFromTabs(tab.childTabs))
+function hasUnresolvedStructuralContent(elements: DocsStructuralElement[]): boolean {
+  for (const element of elements) {
+    if (
+      element.paragraph?.elements?.some(
+        (paragraphElement) =>
+          Boolean(paragraphElement.equation) || Boolean(paragraphElement.inlineObjectElement)
+      )
+    ) {
+      return true
+    }
+
+    for (const row of element.table?.tableRows ?? []) {
+      for (const cell of row.tableCells ?? []) {
+        if (cell.content && hasUnresolvedStructuralContent(cell.content)) return true
+      }
+    }
+
+    if (
+      element.tableOfContents?.content &&
+      hasUnresolvedStructuralContent(element.tableOfContents.content)
+    ) {
+      return true
+    }
   }
 
-  return parts
+  return false
+}
+
+function appendRegionText(parts: string[], regions?: Record<string, DocsContentRegion>): boolean {
+  if (!regions) return false
+  let hasUnresolvedContent = false
+  for (const region of Object.values(regions)) {
+    if (region.content) {
+      parts.push(...extractTextFromStructuralElements(region.content))
+      hasUnresolvedContent ||= hasUnresolvedStructuralContent(region.content)
+    }
+  }
+  return hasUnresolvedContent
+}
+
+function extractTextFromTabs(tabs: DocsTab[]): DocsExtraction {
+  const parts: string[] = []
+  let hasUnresolvedContent = false
+
+  for (const tab of tabs) {
+    const documentTab = tab.documentTab
+    const content = documentTab?.body?.content
+    if (content) {
+      parts.push(...extractTextFromStructuralElements(content))
+      hasUnresolvedContent ||= hasUnresolvedStructuralContent(content)
+    }
+    if (appendRegionText(parts, documentTab?.headers)) hasUnresolvedContent = true
+    if (appendRegionText(parts, documentTab?.footers)) hasUnresolvedContent = true
+    if (appendRegionText(parts, documentTab?.footnotes)) hasUnresolvedContent = true
+
+    if (
+      Object.keys(documentTab?.inlineObjects ?? {}).length > 0 ||
+      Object.keys(documentTab?.positionedObjects ?? {}).length > 0
+    ) {
+      hasUnresolvedContent = true
+    }
+
+    if (tab.childTabs?.length) {
+      const child = extractTextFromTabs(tab.childTabs)
+      parts.push(...child.parts)
+      hasUnresolvedContent ||= child.hasUnresolvedContent
+    }
+  }
+
+  return { parts, hasUnresolvedContent }
 }
 
 /**
  * Extracts plain text from a Google Docs API document response. `tabs` is the
  * source of truth because `includeTabsContent=true` moves all content there and
- * leaves `body` empty; `body` is read only when `tabs` yields nothing, so a
- * response served without tab content still indexes instead of coming back blank.
+ * leaves the legacy top-level fields empty. The unresolved-content bit prevents
+ * an image- or equation-only document from being mistaken for an empty one.
  */
-function extractTextFromDocument(doc: DocsDocument): string {
-  const parts = doc.tabs?.length ? extractTextFromTabs(doc.tabs) : []
-  if (parts.length === 0 && doc.body?.content) {
-    parts.push(...extractTextFromStructuralElements(doc.body.content))
+function extractTextFromDocument(doc: DocsDocument): {
+  content: string
+  hasUnresolvedContent: boolean
+} {
+  if (!doc.tabs?.length) return { content: '', hasUnresolvedContent: false }
+  const extracted = extractTextFromTabs(doc.tabs)
+  return {
+    content: extracted.parts.join('\n').trim(),
+    hasUnresolvedContent: extracted.hasUnresolvedContent,
   }
-
-  return parts.join('\n').trim()
 }
 
 /**
@@ -224,10 +415,13 @@ function extractTextFromDocument(doc: DocsDocument): string {
  * {@link MAX_DOCS_RESPONSE_BYTES} so it surfaces as a visible skipped row rather
  * than being buffered whole.
  */
-async function fetchDocContent(accessToken: string, documentId: string): Promise<string> {
+async function fetchDocContent(
+  accessToken: string,
+  documentId: string
+): Promise<{ content: string; hasUnresolvedContent: boolean }> {
   const params = new URLSearchParams({
     includeTabsContent: 'true',
-    fields: 'body.content,tabs',
+    fields: GOOGLE_DOC_CONTENT_FIELDS,
   })
   const url = `https://docs.googleapis.com/v1/documents/${encodeURIComponent(documentId)}?${params.toString()}`
 
@@ -240,14 +434,19 @@ async function fetchDocContent(accessToken: string, documentId: string): Promise
   })
 
   if (!response.ok) {
-    throw new Error(`Failed to fetch Google Doc content ${documentId}: ${response.status}`)
+    throw new Error(
+      `Failed to fetch Google Doc content ${documentId}: ${await describeGoogleApiFailure(response)}`
+    )
   }
 
   const buffer = await readBodyWithLimit(response, MAX_DOCS_RESPONSE_BYTES)
   if (!buffer) throw new ConnectorFileTooLargeError(MAX_DOCS_RESPONSE_BYTES)
 
-  const doc = JSON.parse(buffer.toString('utf8')) as DocsDocument
-  return extractTextFromDocument(doc)
+  const parsed: unknown = JSON.parse(buffer.toString('utf8'))
+  if (!isPlainRecord(parsed) || !Array.isArray(parsed.tabs) || !parsed.tabs.every(isDocsTab)) {
+    throw new Error('Google Docs API returned a malformed document response')
+  }
+  return extractTextFromDocument({ tabs: parsed.tabs })
 }
 
 /**
@@ -302,10 +501,14 @@ export const googleDocsConnector: ConnectorConfig = {
   ): Promise<ExternalDocumentList> => {
     const query = buildQuery(sourceConfig)
 
-    const maxDocs = sourceConfig.maxDocs ? Number(sourceConfig.maxDocs) : 0
+    const maxDocs = parseMaxDocs(sourceConfig.maxDocs)
     const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
     /** Last-page precision: never ask Drive for more files than the cap still allows. */
     const remaining = maxDocs > 0 ? Math.max(0, maxDocs - previouslyFetched) : 0
+    if (maxDocs > 0 && remaining === 0) {
+      if (syncContext) syncContext.listingCapped = true
+      return { documents: [], hasMore: false }
+    }
     const pageSize = remaining > 0 ? Math.min(PAGE_SIZE, remaining) : PAGE_SIZE
 
     /**
@@ -323,7 +526,7 @@ export const googleDocsConnector: ConnectorConfig = {
       pageSize: String(pageSize),
       orderBy: 'modifiedTime desc',
       fields:
-        'nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
+        'kind,nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
       supportsAllDrives: 'true',
       includeItemsFromAllDrives: 'true',
     })
@@ -345,16 +548,16 @@ export const googleDocsConnector: ConnectorConfig = {
     })
 
     if (!response.ok) {
-      const errorText = await response.text()
+      const failure = await describeGoogleApiFailure(response)
       logger.error('Failed to list Google Docs', {
         status: response.status,
-        error: errorText,
+        error: failure,
       })
-      throw new Error(`Failed to list Google Docs: ${response.status}`)
+      throw new Error(`Failed to list Google Docs: ${failure}`)
     }
 
-    const data = await response.json()
-    const files = (data.files || []) as DriveFile[]
+    const data = parseDriveFileListResponse(await response.json())
+    const files = data.files
 
     /**
      * Drive sets `incompleteSearch` when it could not search every corpus (it
@@ -394,6 +597,7 @@ export const googleDocsConnector: ConnectorConfig = {
       documents,
       nextCursor: hitLimit ? undefined : nextPageToken,
       hasMore: hitLimit ? false : Boolean(nextPageToken),
+      reconciliationSafe: incompleteSearch ? false : undefined,
     }
   },
 
@@ -415,17 +619,35 @@ export const googleDocsConnector: ConnectorConfig = {
 
     if (!response.ok) {
       if (response.status === 404) return null
-      throw new Error(`Failed to get Google Doc metadata: ${response.status}`)
+      throw new Error(
+        `Failed to get Google Doc metadata: ${await describeGoogleApiFailure(response)}`
+      )
     }
 
-    const file = (await response.json()) as DriveFile & { trashed?: boolean }
+    const file = parseDriveFileMetadata(await response.json(), externalId)
 
     if (file.trashed) return null
-    if (file.mimeType !== GOOGLE_DOC_MIME_TYPE) return null
+    if (file.mimeType !== GOOGLE_DOC_MIME_TYPE) {
+      return {
+        ...markSkipped(fileToStub(file), 'File is no longer a Google Doc'),
+        skippedExistingDisposition: 'replace',
+      }
+    }
 
     try {
-      const content = await fetchDocContent(accessToken, file.id)
-      if (!content.trim()) return null
+      const { content, hasUnresolvedContent } = await fetchDocContent(accessToken, file.id)
+      if (!content.trim()) {
+        if (hasUnresolvedContent) {
+          return markSkipped(
+            fileToStub(file),
+            'Document contains non-text elements but no extractable text'
+          )
+        }
+        return {
+          ...markSkipped(fileToStub(file), 'Document contains no extractable text'),
+          skippedExistingDisposition: 'replace',
+        }
+      }
 
       return { ...fileToStub(file), content, contentDeferred: false }
     } catch (error) {
@@ -452,13 +674,10 @@ export const googleDocsConnector: ConnectorConfig = {
     sourceConfig: Record<string, unknown>
   ): Promise<{ valid: boolean; error?: string }> => {
     const folderIds = parseMultiValue(sourceConfig.folderId)
-    const maxDocs = sourceConfig.maxDocs as string | undefined
-
-    if (maxDocs && (Number.isNaN(Number(maxDocs)) || Number(maxDocs) <= 0)) {
-      return { valid: false, error: 'Max documents must be a positive number' }
-    }
 
     try {
+      parseMaxDocs(sourceConfig.maxDocs)
+
       if (folderIds.length > 0) {
         for (const folderId of folderIds) {
           const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
@@ -483,7 +702,7 @@ export const googleDocsConnector: ConnectorConfig = {
             }
             return {
               valid: false,
-              error: `Failed to access folder "${folderId}": ${response.status}`,
+              error: `Failed to access folder "${folderId}": ${await describeGoogleApiFailure(response)}`,
             }
           }
 
@@ -514,7 +733,10 @@ export const googleDocsConnector: ConnectorConfig = {
         )
 
         if (!response.ok) {
-          return { valid: false, error: `Failed to access Google Docs: ${response.status}` }
+          return {
+            valid: false,
+            error: `Failed to access Google Docs: ${await describeGoogleApiFailure(response)}`,
+          }
         }
       }
 

apps/sim/connectors/google-drive/google-drive-errors.ts

@@ -0,0 +1,142 @@
+import { readBodyWithLimit } from '@/connectors/utils'
+
+const GOOGLE_ERROR_BODY_MAX_BYTES = 64 * 1024
+const GOOGLE_ERROR_REASON_MAX_COUNT = 16
+
+const EXPORT_TOO_LARGE_REASONS = new Set(['exportSizeLimitExceeded'])
+const PERMISSION_REASONS = new Set([
+  'appNotAuthorizedToFile',
+  'insufficientFilePermissions',
+  'teamDriveMembershipRequired',
+])
+const POLICY_REASONS = new Set(['domainPolicy', 'download_restricted_for_revision'])
+const UNSUPPORTED_EXPORT_REASONS = new Set(['fileNotDownloadable', 'fileNotExportable'])
+const QUOTA_REASONS = new Set(['dailyLimitExceeded', 'quotaExceeded'])
+const TRANSIENT_REASONS = new Set([
+  'backendError',
+  'internalError',
+  'rateLimitExceeded',
+  'sharingRateLimitExceeded',
+  'userRateLimitExceeded',
+])
+
+export type GoogleDriveErrorKind =
+  | 'authorization'
+  | 'export_too_large'
+  | 'not_found'
+  | 'permission'
+  | 'policy'
+  | 'quota'
+  | 'transient'
+  | 'unknown'
+  | 'unsupported_export'
+
+interface GoogleErrorEntry {
+  reason?: string
+}
+
+interface ParsedGoogleErrorBody {
+  error?: {
+    errors?: GoogleErrorEntry[]
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}
+
+function optionalString(value: unknown): string | undefined {
+  return typeof value === 'string' && value.trim() ? value.trim() : undefined
+}
+
+function parseErrorBody(value: unknown): ParsedGoogleErrorBody | undefined {
+  if (!isRecord(value) || !isRecord(value.error)) return undefined
+
+  const entries = Array.isArray(value.error.errors)
+    ? value.error.errors.flatMap((entry): GoogleErrorEntry[] => {
+        if (!isRecord(entry)) return []
+        return [
+          {
+            reason: optionalString(entry.reason),
+          },
+        ]
+      })
+    : undefined
+
+  return {
+    error: {
+      errors: entries,
+    },
+  }
+}
+
+function normalizeReason(reason: string): string | undefined {
+  const normalized = reason.trim()
+  return /^[A-Za-z][A-Za-z0-9_.-]{0,99}$/.test(normalized) ? normalized : undefined
+}
+
+function classifyGoogleDriveError(
+  status: number,
+  reasons: readonly string[]
+): GoogleDriveErrorKind {
+  if (status === 429 || status >= 500) return 'transient'
+  if (reasons.some((reason) => EXPORT_TOO_LARGE_REASONS.has(reason))) {
+    return 'export_too_large'
+  }
+  if (status === 404 || reasons.includes('notFound')) return 'not_found'
+  if (status === 401 || reasons.includes('authError')) return 'authorization'
+  if (reasons.some((reason) => PERMISSION_REASONS.has(reason))) return 'permission'
+  if (reasons.some((reason) => POLICY_REASONS.has(reason))) return 'policy'
+  if (reasons.some((reason) => UNSUPPORTED_EXPORT_REASONS.has(reason))) {
+    return 'unsupported_export'
+  }
+  if (reasons.some((reason) => QUOTA_REASONS.has(reason))) return 'quota'
+  if (reasons.some((reason) => TRANSIENT_REASONS.has(reason))) {
+    return 'transient'
+  }
+  return 'unknown'
+}
+
+export class GoogleDriveApiError extends Error {
+  retryAfterMs?: number
+
+  constructor(
+    readonly status: number,
+    readonly reasons: readonly string[],
+    readonly kind: GoogleDriveErrorKind
+  ) {
+    const reasonSuffix = reasons.length > 0 ? ` (${reasons.join(', ')})` : ''
+    super(`Google Drive API request failed with HTTP ${status}${reasonSuffix}`)
+    this.name = 'GoogleDriveApiError'
+  }
+}
+
+/**
+ * Parses Google's structured error envelope without retaining or logging the raw
+ * response body. Error payloads are byte-bounded, free-form provider messages
+ * are omitted, and only validated machine-readable reason tokens survive.
+ */
+export async function readGoogleDriveApiError(response: Response): Promise<GoogleDriveApiError> {
+  const body = await readBodyWithLimit(response, GOOGLE_ERROR_BODY_MAX_BYTES).catch(() => null)
+  let parsedBody: ParsedGoogleErrorBody | undefined
+
+  if (body) {
+    try {
+      parsedBody = parseErrorBody(JSON.parse(body.toString('utf8')))
+    } catch {
+      parsedBody = undefined
+    }
+  }
+
+  const entries = parsedBody?.error?.errors ?? []
+  const rawReasons = [...new Set(entries.flatMap((entry) => (entry.reason ? [entry.reason] : [])))]
+  const reasons = [...new Set(rawReasons.flatMap((reason) => normalizeReason(reason) ?? []))].slice(
+    0,
+    GOOGLE_ERROR_REASON_MAX_COUNT
+  )
+  return new GoogleDriveApiError(
+    response.status,
+    reasons,
+    classifyGoogleDriveError(response.status, rawReasons)
+  )
+}

apps/sim/connectors/google-drive/google-drive.test.ts

@@ -0,0 +1,470 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import * as XLSX from 'xlsx'
+
+const { mockFetch } = vi.hoisted(() => ({ mockFetch: vi.fn() }))
+
+vi.mock('@/components/icons', () => ({ GoogleDriveIcon: () => null }))
+
+afterEach(() => {
+  vi.useRealTimers()
+  vi.unstubAllGlobals()
+})
+
+import { googleDriveConnector } from '@/connectors/google-drive/google-drive'
+import {
+  GoogleDriveApiError,
+  readGoogleDriveApiError,
+} from '@/connectors/google-drive/google-drive-errors'
+import { CONNECTOR_MAX_FILE_BYTES } from '@/connectors/utils'
+
+const FILE_ID = 'drive-file-1'
+const GOOGLE_DOCUMENT_MIME_TYPE = 'application/vnd.google-apps.document'
+const GOOGLE_SPREADSHEET_MIME_TYPE = 'application/vnd.google-apps.spreadsheet'
+
+function jsonResponse(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { 'Content-Type': 'application/json' },
+  })
+}
+
+function driveErrorResponse(reason: string, message: string, status = 403): Response {
+  return jsonResponse(
+    {
+      error: {
+        code: status,
+        errors: [{ domain: 'global', reason, message }],
+        message,
+      },
+    },
+    status
+  )
+}
+
+function fileMetadata(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: FILE_ID,
+    name: 'Product plan',
+    mimeType: GOOGLE_DOCUMENT_MIME_TYPE,
+    modifiedTime: '2026-08-20T12:00:00Z',
+    webViewLink: `https://drive.google.com/file/d/${FILE_ID}/view`,
+    ...overrides,
+  }
+}
+
+async function hydrateWithExportResponse(exportResponse: Response) {
+  mockFetch
+    .mockResolvedValueOnce(jsonResponse(fileMetadata()))
+    .mockResolvedValueOnce(exportResponse)
+  return googleDriveConnector.getDocument('token', {}, FILE_ID)
+}
+
+describe('Google Drive API error parsing', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.stubGlobal('fetch', mockFetch)
+  })
+
+  it.each([
+    ['exportSizeLimitExceeded', 'export_too_large'],
+    ['insufficientFilePermissions', 'permission'],
+    ['appNotAuthorizedToFile', 'permission'],
+    ['domainPolicy', 'policy'],
+    ['fileNotExportable', 'unsupported_export'],
+    ['dailyLimitExceeded', 'quota'],
+    ['rateLimitExceeded', 'transient'],
+    ['userRateLimitExceeded', 'transient'],
+  ] as const)('classifies %s as %s', async (reason, kind) => {
+    const error = await readGoogleDriveApiError(driveErrorResponse(reason, 'Provider message'))
+
+    expect(error).toBeInstanceOf(GoogleDriveApiError)
+    expect(error.kind).toBe(kind)
+    expect(error.reasons).toEqual([reason])
+  })
+
+  it('classifies retryable statuses even without a structured reason', async () => {
+    const error = await readGoogleDriveApiError(
+      new Response('upstream unavailable', { status: 503 })
+    )
+
+    expect(error.kind).toBe('transient')
+    expect(error.message).not.toContain('upstream unavailable')
+  })
+
+  it('omits provider messages from diagnostics', async () => {
+    const message = `Authorization: Bearer private-token\ncontext ${'x'.repeat(700)}`
+    const error = await readGoogleDriveApiError(
+      driveErrorResponse('insufficientFilePermissions', message)
+    )
+
+    expect(error.message).not.toContain('private-token')
+  })
+
+  it('bounds and redacts provider reasons without losing classification', async () => {
+    const secret = 'sk-provider-controlled-secret-value'
+    const reasons = [
+      ...Array.from({ length: 20 }, (_, index) => `${index}-${secret}-${'x'.repeat(200)}`),
+      'quotaExceeded',
+    ]
+    const error = await readGoogleDriveApiError(
+      jsonResponse(
+        {
+          error: {
+            errors: reasons.map((reason) => ({ reason })),
+            message: 'Provider message',
+          },
+        },
+        403
+      )
+    )
+
+    expect(error.kind).toBe('quota')
+    expect(error.reasons).toEqual(['quotaExceeded'])
+    expect(JSON.stringify(error.reasons)).not.toContain(secret)
+    expect(error.message).not.toContain(secret)
+  })
+
+  it('discards an error envelope that exceeds the diagnostic body limit', async () => {
+    const sentinel = 'sk-over-cap-provider-secret-value'
+    const body = JSON.stringify({
+      error: {
+        errors: [{ reason: 'quotaExceeded' }],
+        message: `${'x'.repeat(64 * 1024)}${sentinel}`,
+      },
+    })
+
+    const error = await readGoogleDriveApiError(
+      new Response(body, { status: 403, headers: { 'Content-Type': 'application/json' } })
+    )
+
+    expect(error.kind).toBe('unknown')
+    expect(error.reasons).toEqual([])
+    expect(error.message).not.toContain(sentinel)
+  })
+})
+
+describe('Google Drive metadata hydration', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.stubGlobal('fetch', mockFetch)
+  })
+
+  it.each([{}, { ...fileMetadata(), id: 'different-file' }])(
+    'rejects malformed metadata instead of replacing retained content',
+    async (metadata) => {
+      mockFetch.mockResolvedValueOnce(jsonResponse(metadata))
+
+      await expect(googleDriveConnector.getDocument('token', {}, FILE_ID)).rejects.toThrow(
+        'Google Drive API returned malformed file metadata'
+      )
+    }
+  )
+})
+
+describe('Google Drive export failures', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.stubGlobal('fetch', mockFetch)
+  })
+
+  it('records the documented export size limit as a terminal skipped document', async () => {
+    const document = await hydrateWithExportResponse(
+      driveErrorResponse('exportSizeLimitExceeded', 'Export exceeds the 10 MB limit')
+    )
+
+    expect(document?.contentDeferred).toBe(false)
+    expect(document?.skippedReason).toContain('10MB size limit')
+    expect(mockFetch).toHaveBeenCalledTimes(2)
+  })
+
+  it.each([
+    ['notFound', 'File not found.', 404],
+    [
+      'insufficientFilePermissions',
+      'The user does not have sufficient permissions for this file.',
+      403,
+    ],
+    ['domainPolicy', 'The domain administrators have disabled Drive apps.', 403],
+    ['fileNotExportable', 'This file cannot be exported.', 403],
+  ])(
+    'propagates recoverable %s failures instead of persisting a sticky same-hash skip',
+    async (reason, message, status) => {
+      await expect(
+        hydrateWithExportResponse(driveErrorResponse(reason, message, status))
+      ).rejects.toMatchObject({ name: 'GoogleDriveApiError', status })
+      expect(mockFetch).toHaveBeenCalledTimes(2)
+    }
+  )
+
+  it('retries a body-classified 403 rate limit and succeeds', async () => {
+    vi.useFakeTimers()
+    mockFetch
+      .mockResolvedValueOnce(jsonResponse(fileMetadata()))
+      .mockResolvedValueOnce(
+        driveErrorResponse('userRateLimitExceeded', 'User Rate Limit Exceeded')
+      )
+      .mockResolvedValueOnce(new Response('recovered content', { status: 200 }))
+
+    const documentPromise = googleDriveConnector.getDocument('token', {}, FILE_ID)
+    await vi.runAllTimersAsync()
+
+    await expect(documentPromise).resolves.toMatchObject({
+      content: 'recovered content',
+    })
+    expect(mockFetch).toHaveBeenCalledTimes(3)
+  })
+
+  it('retains header-classified 403 rate-limit retries when the body has no known reason', async () => {
+    vi.useFakeTimers()
+    mockFetch
+      .mockResolvedValueOnce(jsonResponse(fileMetadata()))
+      .mockResolvedValueOnce(
+        new Response(JSON.stringify({ error: { message: 'Temporarily throttled' } }), {
+          status: 403,
+          headers: { 'Content-Type': 'application/json', 'Retry-After': '1' },
+        })
+      )
+      .mockResolvedValueOnce(new Response('recovered content', { status: 200 }))
+
+    const documentPromise = googleDriveConnector.getDocument('token', {}, FILE_ID)
+    await vi.runAllTimersAsync()
+
+    await expect(documentPromise).resolves.toMatchObject({ content: 'recovered content' })
+    expect(mockFetch).toHaveBeenCalledTimes(3)
+  })
+
+  it.each([
+    [429, 'insufficientFilePermissions'],
+    [503, 'exportSizeLimitExceeded'],
+  ])(
+    'retries HTTP %i even when the provider reason is classified as terminal',
+    async (status, reason) => {
+      vi.useFakeTimers()
+      mockFetch
+        .mockResolvedValueOnce(jsonResponse(fileMetadata()))
+        .mockResolvedValueOnce(driveErrorResponse(reason, 'Conflicting provider reason', status))
+        .mockResolvedValueOnce(new Response('recovered content', { status: 200 }))
+
+      const documentPromise = googleDriveConnector.getDocument('token', {}, FILE_ID)
+      await vi.runAllTimersAsync()
+
+      await expect(documentPromise).resolves.toMatchObject({ content: 'recovered content' })
+      expect(mockFetch).toHaveBeenCalledTimes(3)
+    }
+  )
+
+  it('propagates unknown 403 responses instead of misclassifying them as permanent', async () => {
+    await expect(
+      hydrateWithExportResponse(driveErrorResponse('newGoogleReason', 'Undocumented failure'))
+    ).rejects.toMatchObject({
+      name: 'GoogleDriveApiError',
+      status: 403,
+      kind: 'unknown',
+      reasons: ['newGoogleReason'],
+    })
+  })
+
+  it('hands the complete XLSX workbook to the shared parser instead of exporting only sheet one', async () => {
+    const workbook = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(workbook, XLSX.utils.aoa_to_sheet([]), 'Empty first sheet')
+    XLSX.utils.book_append_sheet(
+      workbook,
+      XLSX.utils.aoa_to_sheet([
+        ['month', 'revenue'],
+        ['Jan', 100],
+      ]),
+      'Revenue'
+    )
+    const workbookBytes = Buffer.from(XLSX.write(workbook, { bookType: 'xlsx', type: 'buffer' }))
+
+    mockFetch
+      .mockResolvedValueOnce(
+        jsonResponse(
+          fileMetadata({ name: 'Revenue model', mimeType: GOOGLE_SPREADSHEET_MIME_TYPE })
+        )
+      )
+      .mockResolvedValueOnce(new Response(workbookBytes))
+
+    const document = await googleDriveConnector.getDocument('token', {}, FILE_ID)
+    const exportUrl = String(mockFetch.mock.calls[1][0])
+
+    expect(exportUrl).toContain(
+      'mimeType=application%2Fvnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    )
+    expect(document?.content).toBe('')
+    expect(document?.mimeType).toBe(
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    )
+    expect(document?.sourceFile).toMatchObject({
+      fileName: 'Revenue model.xlsx',
+      mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    })
+    expect(document?.sourceFile?.bytes).toEqual(workbookBytes)
+    expect(document?.skippedReason).toBeUndefined()
+    expect(document?.contentHash).toBe('gdrive:v2:drive-file-1:2026-08-20T12:00:00Z')
+  })
+
+  it('marks an empty export as an authoritative skip', async () => {
+    const document = await hydrateWithExportResponse(new Response('   '))
+
+    expect(document).toMatchObject({
+      content: '',
+      contentDeferred: false,
+      skippedExistingDisposition: 'replace',
+      skippedReason: 'Document contains no extractable text',
+    })
+  })
+
+  it('authoritatively skips a listed file that changed to an unsupported type', async () => {
+    mockFetch.mockResolvedValueOnce(
+      jsonResponse(fileMetadata({ name: 'diagram.png', mimeType: 'image/png' }))
+    )
+
+    await expect(googleDriveConnector.getDocument('token', {}, FILE_ID)).resolves.toMatchObject({
+      content: '',
+      skippedReason: 'File is no longer an indexable document',
+      skippedExistingDisposition: 'replace',
+    })
+    expect(mockFetch).toHaveBeenCalledTimes(1)
+  })
+})
+
+describe('Google Drive connector limits', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    vi.stubGlobal('fetch', mockFetch)
+  })
+
+  it('does not let an oversized skipped file consume the maxFiles budget', async () => {
+    mockFetch
+      .mockResolvedValueOnce(
+        jsonResponse({
+          files: [
+            fileMetadata({
+              id: 'oversized',
+              name: 'oversized.txt',
+              mimeType: 'text/plain',
+              size: String(CONNECTOR_MAX_FILE_BYTES + 1),
+            }),
+          ],
+          nextPageToken: 'next-page',
+        })
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({
+          files: [
+            fileMetadata({
+              id: 'indexable',
+              name: 'notes.txt',
+              mimeType: 'text/plain',
+              size: '12',
+            }),
+          ],
+        })
+      )
+
+    const syncContext: Record<string, unknown> = {}
+    const first = await googleDriveConnector.listDocuments(
+      'token',
+      { maxFiles: '1' },
+      undefined,
+      syncContext
+    )
+    const second = await googleDriveConnector.listDocuments(
+      'token',
+      { maxFiles: '1' },
+      first.nextCursor,
+      syncContext
+    )
+
+    expect(first.documents[0].skippedReason).toBeDefined()
+    expect(first.hasMore).toBe(true)
+    expect(second.documents.map((document) => document.externalId)).toEqual(['indexable'])
+    expect(syncContext.totalDocsFetched).toBe(1)
+  })
+
+  it('makes an incomplete cross-corpus search non-authoritative', async () => {
+    mockFetch.mockResolvedValueOnce(
+      jsonResponse({ files: [fileMetadata()], incompleteSearch: true })
+    )
+    const syncContext: Record<string, unknown> = {}
+
+    const result = await googleDriveConnector.listDocuments('token', {}, undefined, syncContext)
+
+    expect(result.documents.map((document) => document.externalId)).toEqual([FILE_ID])
+    expect(result.reconciliationSafe).toBe(false)
+    expect(syncContext.listingCapped).toBe(true)
+  })
+
+  it('rejects a malformed successful file-list envelope', async () => {
+    mockFetch.mockResolvedValueOnce(jsonResponse({}))
+
+    await expect(googleDriveConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow(
+      'Google Drive API returned malformed file-list metadata'
+    )
+  })
+
+  it('accepts a discriminator-only empty file-list envelope', async () => {
+    mockFetch.mockResolvedValueOnce(jsonResponse({ kind: 'drive#fileList' }))
+
+    await expect(
+      googleDriveConnector.listDocuments('token', {}, undefined, {})
+    ).resolves.toMatchObject({ documents: [], hasMore: false })
+  })
+
+  it.each([
+    { files: [{ name: 'Missing ID', mimeType: 'text/plain', modifiedTime: '2026-01-01' }] },
+    { files: [], nextPageToken: 123 },
+    { files: [], incompleteSearch: 'true' },
+  ])('rejects malformed file-list metadata', async (body) => {
+    mockFetch.mockResolvedValueOnce(jsonResponse(body))
+
+    await expect(googleDriveConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow(
+      'Google Drive API returned malformed file-list metadata'
+    )
+  })
+
+  it.each(['1.5', 'Infinity', 1.5, Number.POSITIVE_INFINITY])(
+    'rejects invalid persisted maxFiles %s before listing from Drive',
+    async (maxFiles) => {
+      await expect(googleDriveConnector.listDocuments('token', { maxFiles })).rejects.toThrow(
+        'Max files must be a positive safe integer, or 0 for unlimited'
+      )
+      expect(mockFetch).not.toHaveBeenCalled()
+    }
+  )
+
+  it.each([undefined, null, '', '   ', 0, '0'])(
+    'keeps omitted or explicit unlimited maxFiles %s valid at runtime',
+    async (maxFiles) => {
+      mockFetch.mockResolvedValueOnce(jsonResponse({ files: [] }))
+
+      await expect(
+        googleDriveConnector.listDocuments('token', { maxFiles })
+      ).resolves.toMatchObject({ documents: [], hasMore: false })
+      expect(String(mockFetch.mock.calls[0][0])).toContain('pageSize=100')
+    }
+  )
+
+  it('uses a valid persisted maxFiles cap at runtime', async () => {
+    mockFetch.mockResolvedValueOnce(jsonResponse({ files: [] }))
+
+    await googleDriveConnector.listDocuments('token', { maxFiles: '25' })
+
+    expect(String(mockFetch.mock.calls[0][0])).toContain('pageSize=25')
+  })
+
+  it.each(['1.5', 'Infinity', 1.5, Number.POSITIVE_INFINITY])(
+    'rejects invalid maxFiles %s during validation without calling Drive',
+    async (maxFiles) => {
+      await expect(googleDriveConnector.validateConfig('token', { maxFiles })).resolves.toEqual({
+        valid: false,
+        error: 'Max files must be a positive safe integer, or 0 for unlimited',
+      })
+      expect(mockFetch).not.toHaveBeenCalled()
+    }
+  )
+})

apps/sim/connectors/google-drive/google-drive.ts

@@ -1,6 +1,18 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
-import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { isPlainRecord } from '@sim/utils/object'
+import {
+  attachRetryHeaders,
+  isRetryableError,
+  type RetryOptions,
+  resolveRetryDelayMs,
+  retryWithExponentialBackoff,
+  VALIDATE_RETRY_OPTIONS,
+} from '@/lib/knowledge/documents/utils'
+import {
+  GoogleDriveApiError,
+  readGoogleDriveApiError,
+} from '@/connectors/google-drive/google-drive-errors'
 import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import {
@@ -8,20 +20,25 @@ import {
   CONNECTOR_MAX_FILE_BYTES,
   ConnectorFileTooLargeError,
   htmlToPlainText,
+  isSkippedDocument,
   joinTagArray,
   markSkipped,
   parseMultiValue,
+  parseOptionalUnlimitedSafeInteger,
   parseTagDate,
   readBodyWithLimit,
   sizeLimitSkipReason,
   stubOrSkipBySize,
+  takeIndexableWithinCap,
 } from '@/connectors/utils'
 
 const logger = createLogger('GoogleDriveConnector')
 
-const GOOGLE_WORKSPACE_MIME_TYPES: Record<string, string> = {
+const XLSX_MIME_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+
+const GOOGLE_WORKSPACE_EXPORTS: Record<string, string> = {
   'application/vnd.google-apps.document': 'text/plain',
-  'application/vnd.google-apps.spreadsheet': 'text/csv',
+  'application/vnd.google-apps.spreadsheet': XLSX_MIME_TYPE,
   'application/vnd.google-apps.presentation': 'text/plain',
 }
 
@@ -37,49 +54,88 @@ const SUPPORTED_TEXT_MIME_TYPES = [
 // Google Drive's `files.export` API rejects exports over 10 MB (exportSizeLimitExceeded),
 // so this is a hard external limit for Google Workspace docs — not the connector cap.
 const MAX_EXPORT_SIZE = 10 * 1024 * 1024
+const MAX_FILES_VALIDATION_ERROR = 'Max files must be a positive safe integer, or 0 for unlimited'
+
+function parseMaxFiles(value: unknown): number {
+  return parseOptionalUnlimitedSafeInteger(value, MAX_FILES_VALIDATION_ERROR)
+}
+
+function googleDriveErrorLogFields(error: unknown): Record<string, unknown> {
+  if (error instanceof GoogleDriveApiError) {
+    return {
+      error: error.message,
+      status: error.status,
+      reasons: error.reasons,
+    }
+  }
+  return { error: toError(error).message }
+}
 
 function isGoogleWorkspaceFile(mimeType: string): boolean {
-  return mimeType in GOOGLE_WORKSPACE_MIME_TYPES
+  return mimeType in GOOGLE_WORKSPACE_EXPORTS
 }
 
 function isSupportedTextFile(mimeType: string): boolean {
   return SUPPORTED_TEXT_MIME_TYPES.some((t) => mimeType.startsWith(t))
 }
 
+/** Retries Google errors whose structured body identifies a transient rejection. */
+async function fetchGoogleDriveWithRetry(
+  url: string,
+  options: RequestInit,
+  retryOptions: RetryOptions = {}
+): Promise<Response> {
+  return retryWithExponentialBackoff(
+    async () => {
+      const response = await fetch(url, options)
+      if (response.ok) return response
+
+      const error = await readGoogleDriveApiError(response)
+      attachRetryHeaders(error, response.headers)
+      const waitMs = resolveRetryDelayMs(response.headers)
+      if (waitMs !== undefined) error.retryAfterMs = waitMs
+      throw error
+    },
+    {
+      ...retryOptions,
+      retryCondition: (error) =>
+        error instanceof GoogleDriveApiError
+          ? error.kind === 'transient' || isRetryableError(error)
+          : (retryOptions.retryCondition?.(error) ?? isRetryableError(error)),
+    }
+  )
+}
+
 async function exportGoogleWorkspaceFile(
   accessToken: string,
   fileId: string,
   sourceMimeType: string
-): Promise<string> {
-  const exportMimeType = GOOGLE_WORKSPACE_MIME_TYPES[sourceMimeType]
+): Promise<Buffer> {
+  const exportMimeType = GOOGLE_WORKSPACE_EXPORTS[sourceMimeType]
   if (!exportMimeType) {
     throw new Error(`Unsupported Google Workspace MIME type: ${sourceMimeType}`)
   }
 
   const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}/export?mimeType=${encodeURIComponent(exportMimeType)}`
 
-  const response = await fetchWithRetry(url, {
-    method: 'GET',
-    headers: { Authorization: `Bearer ${accessToken}` },
-  })
-
-  if (!response.ok) {
-    // Google rejects exports over its 10 MB limit with a 403 exportSizeLimitExceeded
-    // before streaming any bytes — surface that as an oversize skip, not a hard error.
-    if (response.status === 403) {
-      const body = await response.text().catch(() => '')
-      if (body.includes('exportSizeLimitExceeded')) {
-        throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
-      }
+  let response: Response
+  try {
+    response = await fetchGoogleDriveWithRetry(url, {
+      method: 'GET',
+      headers: { Authorization: `Bearer ${accessToken}` },
+    })
+  } catch (error) {
+    if (error instanceof GoogleDriveApiError && error.kind === 'export_too_large') {
+      throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
     }
-    throw new Error(`Failed to export file ${fileId}: ${response.status}`)
+    throw error
   }
 
   const buffer = await readBodyWithLimit(response, MAX_EXPORT_SIZE)
   if (!buffer) {
     throw new ConnectorFileTooLargeError(MAX_EXPORT_SIZE)
   }
-  return buffer.toString('utf8')
+  return buffer
 }
 
 async function downloadTextFile(accessToken: string, fileId: string): Promise<string> {
@@ -88,15 +144,11 @@ async function downloadTextFile(accessToken: string, fileId: string): Promise<st
   // metadata fetch in getDocument already does. (`files.export` takes no such param.)
   const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?alt=media&supportsAllDrives=true`
 
-  const response = await fetchWithRetry(url, {
+  const response = await fetchGoogleDriveWithRetry(url, {
     method: 'GET',
     headers: { Authorization: `Bearer ${accessToken}` },
   })
 
-  if (!response.ok) {
-    throw new Error(`Failed to download file ${fileId}: ${response.status}`)
-  }
-
   // Stream with a hard byte cap so a file with missing/under-reported listing
   // size metadata is never fully buffered into memory. Oversized files raise
   // DriveFileTooLargeError so getDocument can surface them as skipped (failed) rows.
@@ -107,23 +159,37 @@ async function downloadTextFile(accessToken: string, fileId: string): Promise<st
   return buffer.toString('utf8')
 }
 
-async function fetchFileContent(
-  accessToken: string,
-  fileId: string,
-  mimeType: string
-): Promise<string> {
-  if (isGoogleWorkspaceFile(mimeType)) {
-    return exportGoogleWorkspaceFile(accessToken, fileId, mimeType)
+type FilePayload = Pick<ExternalDocument, 'content' | 'mimeType' | 'sourceFile'>
+
+function xlsxFileName(name: string): string {
+  return name.toLowerCase().endsWith('.xlsx') ? name : `${name}.xlsx`
+}
+
+async function fetchFilePayload(accessToken: string, file: DriveFile): Promise<FilePayload> {
+  if (GOOGLE_WORKSPACE_EXPORTS[file.mimeType]) {
+    const bytes = await exportGoogleWorkspaceFile(accessToken, file.id, file.mimeType)
+    if (file.mimeType === 'application/vnd.google-apps.spreadsheet') {
+      return {
+        content: '',
+        mimeType: XLSX_MIME_TYPE,
+        sourceFile: {
+          bytes,
+          fileName: xlsxFileName(file.name || 'Untitled'),
+          mimeType: XLSX_MIME_TYPE,
+        },
+      }
+    }
+    return { content: bytes.toString('utf8'), mimeType: 'text/plain' }
   }
-  if (mimeType === 'text/html') {
-    const html = await downloadTextFile(accessToken, fileId)
-    return htmlToPlainText(html)
+  if (file.mimeType === 'text/html') {
+    const html = await downloadTextFile(accessToken, file.id)
+    return { content: htmlToPlainText(html), mimeType: 'text/plain' }
   }
-  if (isSupportedTextFile(mimeType)) {
-    return downloadTextFile(accessToken, fileId)
+  if (isSupportedTextFile(file.mimeType)) {
+    return { content: await downloadTextFile(accessToken, file.id), mimeType: 'text/plain' }
   }
 
-  throw new Error(`Unsupported MIME type for content extraction: ${mimeType}`)
+  throw new Error(`Unsupported MIME type for content extraction: ${file.mimeType}`)
 }
 
 interface DriveFile {
@@ -139,6 +205,74 @@ interface DriveFile {
   trashed?: boolean
 }
 
+interface DriveFileListResponse {
+  kind?: string
+  files?: DriveFile[]
+  incompleteSearch?: boolean
+  nextPageToken?: string
+}
+
+function parseDriveFileListResponse(
+  value: unknown
+): DriveFileListResponse & { files: DriveFile[] } {
+  if (!isPlainRecord(value)) {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  const rawFiles = value.files
+  if (rawFiles === undefined && value.kind !== 'drive#fileList') {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  if (
+    rawFiles !== undefined &&
+    (!Array.isArray(rawFiles) || rawFiles.some((file) => !isDriveFileListItem(file)))
+  ) {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  if (
+    value.nextPageToken !== undefined &&
+    (typeof value.nextPageToken !== 'string' || value.nextPageToken.length === 0)
+  ) {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  if (value.incompleteSearch !== undefined && typeof value.incompleteSearch !== 'boolean') {
+    throw new Error('Google Drive API returned malformed file-list metadata')
+  }
+  return {
+    kind: typeof value.kind === 'string' ? value.kind : undefined,
+    files: rawFiles ?? [],
+    incompleteSearch: value.incompleteSearch === true,
+    nextPageToken: typeof value.nextPageToken === 'string' ? value.nextPageToken : undefined,
+  }
+}
+
+function isDriveFileMetadata(value: unknown, expectedId: string): value is DriveFile {
+  return (
+    isPlainRecord(value) &&
+    value.id === expectedId &&
+    typeof value.name === 'string' &&
+    typeof value.mimeType === 'string' &&
+    (value.trashed === undefined || typeof value.trashed === 'boolean')
+  )
+}
+
+function isDriveFileListItem(value: unknown): value is DriveFile {
+  return (
+    isPlainRecord(value) &&
+    typeof value.id === 'string' &&
+    value.id.length > 0 &&
+    typeof value.name === 'string' &&
+    typeof value.mimeType === 'string' &&
+    typeof value.modifiedTime === 'string'
+  )
+}
+
+function parseDriveFileMetadata(value: unknown, expectedId: string): DriveFile {
+  if (!isDriveFileMetadata(value, expectedId)) {
+    throw new Error('Google Drive API returned malformed file metadata')
+  }
+  return value
+}
+
 function buildQuery(sourceConfig: Record<string, unknown>): string {
   const parts: string[] = ['trashed = false']
 
@@ -161,10 +295,7 @@ function buildQuery(sourceConfig: Record<string, unknown>): string {
       break
     default: {
       // Include Google Workspace files + plain text files, exclude folders
-      const allMimeTypes = [
-        ...Object.keys(GOOGLE_WORKSPACE_MIME_TYPES),
-        ...SUPPORTED_TEXT_MIME_TYPES,
-      ]
+      const allMimeTypes = [...Object.keys(GOOGLE_WORKSPACE_EXPORTS), ...SUPPORTED_TEXT_MIME_TYPES]
       parts.push(`(${allMimeTypes.map((t) => `mimeType = '${t}'`).join(' or ')})`)
       break
     }
@@ -174,6 +305,14 @@ function buildQuery(sourceConfig: Record<string, unknown>): string {
 }
 
 function fileToStub(file: DriveFile): ExternalDocument {
+  /**
+   * Sheets moved from a first-sheet-only CSV export to the complete XLSX source.
+   * The namespace forces one rehydration for existing rows whose old hash would
+   * otherwise preserve embeddings that omit every sheet after the first.
+   */
+  const hashNamespace =
+    file.mimeType === 'application/vnd.google-apps.spreadsheet' ? 'gdrive:v2' : 'gdrive'
+
   return {
     externalId: file.id,
     title: file.name || 'Untitled',
@@ -181,7 +320,7 @@ function fileToStub(file: DriveFile): ExternalDocument {
     contentDeferred: true,
     mimeType: 'text/plain',
     sourceUrl: file.webViewLink || `https://drive.google.com/file/d/${file.id}/view`,
-    contentHash: `gdrive:${file.id}:${file.modifiedTime ?? ''}`,
+    contentHash: `${hashNamespace}:${file.id}:${file.modifiedTime ?? ''}`,
     metadata: {
       originalMimeType: file.mimeType,
       modifiedTime: file.modifiedTime,
@@ -205,7 +344,7 @@ export const googleDriveConnector: ConnectorConfig = {
     const query = buildQuery(sourceConfig)
     const pageSize = 100
 
-    const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
+    const maxFiles = parseMaxFiles(sourceConfig.maxFiles)
     const previouslyFetched = (syncContext?.totalDocsFetched as number) ?? 0
 
     if (maxFiles > 0 && previouslyFetched >= maxFiles) {
@@ -220,7 +359,7 @@ export const googleDriveConnector: ConnectorConfig = {
       pageSize: String(effectivePageSize),
       orderBy: 'modifiedTime desc',
       fields:
-        'nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred)',
+        'kind,nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred)',
       supportsAllDrives: 'true',
       includeItemsFromAllDrives: 'true',
     })
@@ -233,25 +372,22 @@ export const googleDriveConnector: ConnectorConfig = {
 
     logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' })
 
-    const response = await fetchWithRetry(url, {
-      method: 'GET',
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        Accept: 'application/json',
-      },
-    })
-
-    if (!response.ok) {
-      const errorText = await response.text()
-      logger.error('Failed to list Google Drive files', {
-        status: response.status,
-        error: errorText,
+    let response: Response
+    try {
+      response = await fetchGoogleDriveWithRetry(url, {
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          Accept: 'application/json',
+        },
       })
-      throw new Error(`Failed to list Google Drive files: ${response.status}`)
+    } catch (error) {
+      logger.error('Failed to list Google Drive files', googleDriveErrorLogFields(error))
+      throw error
     }
 
-    const data = await response.json()
-    const files = (data.files || []) as DriveFile[]
+    const data = parseDriveFileListResponse(await response.json())
+    const files = data.files
 
     /**
      * Drive sets `incompleteSearch` when it could not search every corpus (it
@@ -261,17 +397,24 @@ export const googleDriveConnector: ConnectorConfig = {
      */
     const incompleteSearch = data.incompleteSearch === true
 
-    const documents = files
+    const pageDocuments = files
       .filter((f) => isGoogleWorkspaceFile(f.mimeType) || isSupportedTextFile(f.mimeType))
       .map((f) =>
         stubOrSkipBySize(fileToStub(f), Number(f.size) || undefined, CONNECTOR_MAX_FILE_BYTES)
       )
 
-    const totalFetched = previouslyFetched + documents.length
-    if (syncContext) syncContext.totalDocsFetched = totalFetched
-    const hitLimit = maxFiles > 0 && totalFetched >= maxFiles
+    const page = takeIndexableWithinCap(
+      pageDocuments,
+      isSkippedDocument,
+      maxFiles,
+      previouslyFetched
+    )
 
-    const nextPageToken = data.nextPageToken as string | undefined
+    const totalFetched = previouslyFetched + page.indexableCount
+    if (syncContext) syncContext.totalDocsFetched = totalFetched
+    const hitLimit = page.capReached
+
+    const nextPageToken = data.nextPageToken
 
     /**
      * Suppress deletion reconciliation only when the listing really is partial.
@@ -284,9 +427,10 @@ export const googleDriveConnector: ConnectorConfig = {
     }
 
     return {
-      documents,
+      documents: page.documents,
       nextCursor: hitLimit ? undefined : nextPageToken,
       hasMore: hitLimit ? false : Boolean(nextPageToken),
+      reconciliationSafe: incompleteSearch ? false : undefined,
     }
   },
 
@@ -299,42 +443,54 @@ export const googleDriveConnector: ConnectorConfig = {
       'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,trashed'
     const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(externalId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
 
-    const response = await fetchWithRetry(url, {
-      method: 'GET',
-      headers: {
-        Authorization: `Bearer ${accessToken}`,
-        Accept: 'application/json',
-      },
-    })
-
-    if (!response.ok) {
-      if (response.status === 404) return null
-      throw new Error(`Failed to get Google Drive file: ${response.status}`)
+    let response: Response
+    try {
+      response = await fetchGoogleDriveWithRetry(url, {
+        method: 'GET',
+        headers: {
+          Authorization: `Bearer ${accessToken}`,
+          Accept: 'application/json',
+        },
+      })
+    } catch (error) {
+      if (!(error instanceof GoogleDriveApiError)) throw error
+      if (error.kind === 'not_found') return null
+      throw error
     }
 
-    const file = (await response.json()) as DriveFile
+    const file = parseDriveFileMetadata(await response.json(), externalId)
 
     if (file.trashed) return null
 
     /**
-     * Mirrors the listing filter: a file re-uploaded under an unextractable type between
-     * listing and hydration has no content, which is an absence rather than a fetch
-     * failure. Returning null keeps it from being retried as a failure every sync.
+     * Mirrors the listing filter. The marker distinguishes a successfully
+     * verified unindexable file from an ambiguous null hydration.
      */
     if (!isGoogleWorkspaceFile(file.mimeType) && !isSupportedTextFile(file.mimeType)) {
       logger.info('Google Drive file has no extractable text type', {
         fileId: file.id,
         mimeType: file.mimeType,
       })
-      return null
+      return {
+        ...markSkipped(fileToStub(file), 'File is no longer an indexable document'),
+        skippedExistingDisposition: 'replace',
+      }
     }
 
     try {
-      const content = await fetchFileContent(accessToken, file.id, file.mimeType)
-      if (!content.trim()) return null
+      const payload = await fetchFilePayload(accessToken, file)
+      if (!payload.content.trim() && !payload.sourceFile?.bytes.length) {
+        return {
+          ...markSkipped(
+            { ...fileToStub(file), ...payload },
+            'Document contains no extractable text'
+          ),
+          skippedExistingDisposition: 'replace',
+        }
+      }
 
       const stub = fileToStub(file)
-      return { ...stub, content, contentDeferred: false }
+      return { ...stub, ...payload, contentDeferred: false }
     } catch (error) {
       if (error instanceof ConnectorFileTooLargeError) {
         logger.info('Skipping oversized Google Drive file', { fileId: file.id, name: file.name })
@@ -347,7 +503,7 @@ export const googleDriveConnector: ConnectorConfig = {
        */
       const err = toError(error)
       logger.warn(`Failed to fetch content for file: ${file.name} (${file.id})`, {
-        error: err.message,
+        ...googleDriveErrorLogFields(err),
       })
       throw err
     }
@@ -358,41 +514,42 @@ export const googleDriveConnector: ConnectorConfig = {
     sourceConfig: Record<string, unknown>
   ): Promise<{ valid: boolean; error?: string }> => {
     const folderIds = parseMultiValue(sourceConfig.folderId)
-    const maxFiles = sourceConfig.maxFiles as string | undefined
-
-    if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
-      return { valid: false, error: 'Max files must be a positive number' }
-    }
 
     // Verify access to Drive API
     try {
+      parseMaxFiles(sourceConfig.maxFiles)
+
       if (folderIds.length > 0) {
         // Verify each folder exists, is accessible, and is actually a folder
         for (const folderId of folderIds) {
           const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
-          const response = await fetchWithRetry(
-            url,
-            {
-              method: 'GET',
-              headers: {
-                Authorization: `Bearer ${accessToken}`,
-                Accept: 'application/json',
+          let response: Response
+          try {
+            response = await fetchGoogleDriveWithRetry(
+              url,
+              {
+                method: 'GET',
+                headers: {
+                  Authorization: `Bearer ${accessToken}`,
+                  Accept: 'application/json',
+                },
               },
-            },
-            VALIDATE_RETRY_OPTIONS
-          )
-
-          if (!response.ok) {
-            if (response.status === 404) {
+              VALIDATE_RETRY_OPTIONS
+            )
+          } catch (error) {
+            if (error instanceof GoogleDriveApiError) {
+              if (error.kind === 'not_found') {
+                return {
+                  valid: false,
+                  error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
+                }
+              }
               return {
                 valid: false,
-                error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
+                error: `Failed to access folder "${folderId}": ${error.message}`,
               }
             }
-            return {
-              valid: false,
-              error: `Failed to access folder "${folderId}": ${response.status}`,
-            }
+            throw error
           }
 
           const folder = await response.json()
@@ -404,20 +561,23 @@ export const googleDriveConnector: ConnectorConfig = {
         // Verify basic Drive access by listing one file
         const url =
           'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true'
-        const response = await fetchWithRetry(
-          url,
-          {
-            method: 'GET',
-            headers: {
-              Authorization: `Bearer ${accessToken}`,
-              Accept: 'application/json',
+        try {
+          await fetchGoogleDriveWithRetry(
+            url,
+            {
+              method: 'GET',
+              headers: {
+                Authorization: `Bearer ${accessToken}`,
+                Accept: 'application/json',
+              },
             },
-          },
-          VALIDATE_RETRY_OPTIONS
-        )
-
-        if (!response.ok) {
-          return { valid: false, error: `Failed to access Google Drive: ${response.status}` }
+            VALIDATE_RETRY_OPTIONS
+          )
+        } catch (error) {
+          if (error instanceof GoogleDriveApiError) {
+            return { valid: false, error: `Failed to access Google Drive: ${error.message}` }
+          }
+          throw error
         }
       }
 

apps/sim/connectors/notion/meta.ts

@@ -19,7 +19,7 @@ export const notionConnectorMeta: ConnectorMeta = {
       options: [
         { label: 'Entire workspace', id: 'workspace' },
         { label: 'Specific database', id: 'database' },
-        { label: 'Specific page (and children)', id: 'page' },
+        { label: 'Specific page (and direct child pages)', id: 'page' },
       ],
     },
     {

apps/sim/connectors/onedrive/onedrive.test.ts

@@ -7,11 +7,16 @@ const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() }
 
 vi.mock('@/lib/knowledge/documents/utils', () => ({
   fetchWithRetry: mockFetchWithRetry,
+  readBoundedHttpErrorBody: async (response: Response) => response.text(),
   VALIDATE_RETRY_OPTIONS: {},
 }))
 vi.mock('@/components/icons', () => ({ MicrosoftOneDriveIcon: () => null }))
 
 import { onedriveConnector } from '@/connectors/onedrive/onedrive'
+import {
+  encodeMicrosoftGraphTraversalCursor,
+  MICROSOFT_GRAPH_MAX_PENDING_FOLDERS,
+} from '@/connectors/utils'
 
 const GRAPH = 'https://graph.microsoft.com/v1.0'
 
@@ -52,15 +57,32 @@ function mockGraph(routes: Record<string, GraphRoute>) {
   return requested
 }
 
-const ROOT_URL = `${GRAPH}/me/drive/root/children?$top=200&$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference`
+const ROOT_URL = `${GRAPH}/me/drive/root/children?$top=200&$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`
 const childrenUrl = (id: string) =>
-  `${GRAPH}/me/drive/items/${id}/children?$top=200&$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference`
+  `${GRAPH}/me/drive/items/${id}/children?$top=200&$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`
 
 describe('onedrive listDocuments', () => {
   beforeEach(() => {
     vi.clearAllMocks()
   })
 
+  it('rejects a malformed successful list envelope', async () => {
+    mockGraph({ [ROOT_URL]: { body: {} } })
+
+    await expect(onedriveConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow(
+      'Microsoft Graph returned malformed OneDrive list metadata'
+    )
+  })
+
+  it.each([
+    { value: [{ id: 'f1', name: 'Missing facet' }] },
+    { value: [], '@odata.nextLink': 'https://evil.example/items' },
+  ])('rejects ambiguous or unsafe list metadata', async (body) => {
+    mockGraph({ [ROOT_URL]: { body } })
+
+    await expect(onedriveConnector.listDocuments('token', {}, undefined, {})).rejects.toThrow()
+  })
+
   it('walks nested folders within a single call', async () => {
     const requested = mockGraph({
       [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), folder('dir1', 'dir1')] } },
@@ -161,6 +183,33 @@ describe('onedrive listDocuments', () => {
     expect(syncContext.listingCapped).toBe(true)
   })
 
+  it('does not retain irrelevant folders after maxFiles has stopped traversal', async () => {
+    mockGraph({
+      [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), folder('overflow', 'overflow')] } },
+    })
+    const cursor = encodeMicrosoftGraphTraversalCursor(
+      {
+        folderStack: Array.from(
+          { length: MICROSOFT_GRAPH_MAX_PENDING_FOLDERS },
+          (_, index) => `pending-${index}`
+        ),
+      },
+      'OneDrive'
+    )
+    const syncContext: Record<string, unknown> = {}
+
+    const result = await onedriveConnector.listDocuments(
+      'token',
+      { maxFiles: '1' },
+      cursor,
+      syncContext
+    )
+
+    expect(result.documents.map((document) => document.externalId)).toEqual(['f1'])
+    expect(result.hasMore).toBe(false)
+    expect(syncContext.listingCapped).toBe(true)
+  })
+
   it('resumes from the cursor when the per-call request budget is exhausted', async () => {
     const routes: Record<string, GraphRoute> = {
       [ROOT_URL]: {
@@ -186,7 +235,7 @@ describe('onedrive listDocuments', () => {
   })
 
   it('encodes the configured folder path', async () => {
-    const url = `${GRAPH}/me/drive/root:/My%20Docs/Q1%20%26%20Q2:/children?$top=200&$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference`
+    const url = `${GRAPH}/me/drive/root:/My%20Docs/Q1%20%26%20Q2:/children?$top=200&$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`
     const requested = mockGraph({ [url]: { body: { value: [] } } })
 
     await onedriveConnector.listDocuments(
@@ -198,6 +247,47 @@ describe('onedrive listDocuments', () => {
 
     expect(requested[0]).toBe(url)
   })
+
+  it.each(['1.5', 'Infinity'])(
+    'rejects invalid maxFiles %s before calling Graph',
+    async (maxFiles) => {
+      const requested = mockGraph({})
+
+      await expect(
+        onedriveConnector.listDocuments('token', { maxFiles }, undefined, {})
+      ).rejects.toThrow(/positive safe integer/)
+      expect(requested).toHaveLength(0)
+    }
+  )
+})
+
+describe('onedrive validateConfig', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it.each(['1.5', 'Infinity'])(
+    'rejects invalid maxFiles %s without calling Graph',
+    async (maxFiles) => {
+      const requested = mockGraph({})
+
+      await expect(onedriveConnector.validateConfig!('token', { maxFiles })).resolves.toEqual({
+        valid: false,
+        error: 'Max files must be a positive safe integer, or 0 for unlimited',
+      })
+      expect(requested).toHaveLength(0)
+    }
+  )
+
+  it('accepts a valid integer maxFiles', async () => {
+    const validateRootUrl = `${GRAPH}/me/drive/root/children?$top=1&$select=id`
+    const requested = mockGraph({ [validateRootUrl]: { body: { value: [] } } })
+
+    await expect(onedriveConnector.validateConfig!('token', { maxFiles: '25' })).resolves.toEqual({
+      valid: true,
+    })
+    expect(requested).toEqual([validateRootUrl])
+  })
 })
 
 describe('onedrive getDocument', () => {
@@ -211,11 +301,56 @@ describe('onedrive getDocument', () => {
     expect(doc).toBeNull()
   })
 
+  it.each([{}, { id: 'f1', name: 'Missing facet' }, file('different', 'a.txt')])(
+    'rejects malformed metadata instead of replacing retained content',
+    async (metadata) => {
+      mockGraph({
+        [`${GRAPH}/me/drive/items/f1?$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`]:
+          {
+            body: metadata,
+          },
+      })
+
+      await expect(onedriveConnector.getDocument!('token', {}, 'f1')).rejects.toThrow(
+        'Microsoft Graph returned malformed OneDrive item metadata'
+      )
+    }
+  )
+
+  it('authoritatively skips a listed file that changed to a folder', async () => {
+    mockGraph({
+      [`${GRAPH}/me/drive/items/f1?$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`]:
+        {
+          body: folder('f1', 'Former document'),
+        },
+    })
+
+    await expect(onedriveConnector.getDocument!('token', {}, 'f1')).resolves.toMatchObject({
+      content: '',
+      skippedReason: 'File is no longer an indexable document',
+      skippedExistingDisposition: 'replace',
+    })
+  })
+
+  it('authoritatively skips a listed file that changed to a package', async () => {
+    mockGraph({
+      [`${GRAPH}/me/drive/items/f1?$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`]:
+        {
+          body: { id: 'f1', name: 'Former document', package: { type: 'oneNote' } },
+        },
+    })
+
+    await expect(onedriveConnector.getDocument!('token', {}, 'f1')).resolves.toMatchObject({
+      skippedExistingDisposition: 'replace',
+      skippedReason: 'File is no longer an indexable document',
+    })
+  })
+
   it('produces the same contentHash as the listing stub', async () => {
     const item = file('f1', 'a.txt')
     mockGraph({
       [ROOT_URL]: { body: { value: [item] } },
-      [`${GRAPH}/me/drive/items/f1?$select=id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference`]:
+      [`${GRAPH}/me/drive/items/f1?$select=id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference`]:
         { body: item },
     })
 
@@ -227,6 +362,7 @@ describe('onedrive getDocument', () => {
           ok: true,
           status: 200,
           body: null,
+          headers: new Headers({ 'content-length': '5' }),
           arrayBuffer: async () => new TextEncoder().encode('hello').buffer,
         } as unknown as Response
       }
@@ -244,4 +380,35 @@ describe('onedrive getDocument', () => {
     expect(fetched?.contentDeferred).toBe(false)
     expect(fetched?.content).toBe('hello')
   })
+
+  it('marks an empty file as an authoritative skip', async () => {
+    const item = file('empty', 'empty.txt', 0)
+    mockFetchWithRetry.mockImplementation(async (url: string) => {
+      if (url.endsWith('/content')) {
+        return {
+          ok: true,
+          status: 200,
+          body: null,
+          headers: new Headers({ 'content-length': '0' }),
+          arrayBuffer: async () => new ArrayBuffer(0),
+        } as unknown as Response
+      }
+      return {
+        ok: true,
+        status: 200,
+        json: async () => item,
+        text: async () => '',
+      } as unknown as Response
+    })
+
+    const document = await onedriveConnector.getDocument!('token', {}, 'empty')
+
+    expect(document).toMatchObject({
+      externalId: 'empty',
+      content: '',
+      contentDeferred: false,
+      skippedReason: 'Document contains no extractable text',
+      skippedExistingDisposition: 'replace',
+    })
+  })
 })

apps/sim/connectors/onedrive/onedrive.ts

@@ -1,17 +1,29 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
-import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import {
+  fetchWithRetry,
+  readBoundedHttpErrorBody,
+  VALIDATE_RETRY_OPTIONS,
+} from '@/lib/knowledge/documents/utils'
 import { onedriveConnectorMeta } from '@/connectors/onedrive/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import {
+  appendPendingMicrosoftGraphFolders,
+  assertMicrosoftGraphNextLink,
   CONNECTOR_MAX_FILE_BYTES,
   ConnectorFileTooLargeError,
   connectorFileExtension,
+  decodeMicrosoftGraphTraversalCursor,
+  encodeMicrosoftGraphTraversalCursor,
   extractConnectorText,
   hasIndexablePayload,
   isIndexableConnectorFile,
+  isMicrosoftGraphDriveItem,
   isSkippedDocument,
+  type MicrosoftGraphTraversalState,
   markSkipped,
+  parseMicrosoftGraphDriveItemList,
+  parseOptionalUnlimitedSafeInteger,
   parseTagDate,
   pipelineParsedMimeType,
   readBodyWithLimit,
@@ -34,7 +46,8 @@ const GRAPH_BASE_URL = `${GRAPH_API_ORIGIN}/v1.0`
  * The exact driveItem fields the stub is built from. Graph returns the full
  * driveItem otherwise, which is an order of magnitude larger per item.
  */
-const ITEM_SELECT = 'id,name,webUrl,size,file,folder,lastModifiedDateTime,createdBy,parentReference'
+const ITEM_SELECT =
+  'id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdBy,parentReference'
 
 /**
  * Requested page size for a children collection, matching Graph's own default.
@@ -59,11 +72,20 @@ const PAGE_SIZE = 200
  */
 const MAX_LIST_REQUESTS_PER_CALL = 25
 
+function parseMaxFiles(value: unknown): number {
+  return parseOptionalUnlimitedSafeInteger(
+    value,
+    'Max files must be a positive safe integer, or 0 for unlimited'
+  )
+}
+
 interface OneDriveItem {
   id: string
   name: string
   file?: { mimeType: string }
   folder?: { childCount: number }
+  package?: Record<string, unknown>
+  remoteItem?: Record<string, unknown>
   size?: number
   webUrl?: string
   lastModifiedDateTime?: string
@@ -71,9 +93,15 @@ interface OneDriveItem {
   parentReference?: { path?: string }
 }
 
-interface OneDriveListResponse {
-  value: OneDriveItem[]
-  '@odata.nextLink'?: string
+function isOneDriveItemMetadata(value: unknown, expectedId: string): value is OneDriveItem {
+  return isMicrosoftGraphDriveItem(value) && value.id === expectedId
+}
+
+function parseOneDriveItemMetadata(value: unknown, expectedId: string): OneDriveItem {
+  if (!isOneDriveItemMetadata(value, expectedId)) {
+    throw new Error('Microsoft Graph returned malformed OneDrive item metadata')
+  }
+  return value
 }
 
 /**
@@ -170,43 +198,21 @@ function buildListUrl(folderPath: string | undefined, folderId: string | undefin
 
 /**
  * Asserts a paging URL points at Microsoft Graph before it is followed with the
- * bearer token in the `Authorization` header. The `@odata.nextLink` this connector
- * follows is persisted into the sync cursor, so it round-trips through storage
- * rather than arriving straight off a TLS response — a tampered cursor must never
- * be able to redirect the access token to a third-party host. Mirrors
- * `assertGraphNextPageUrl` used by the Graph tool routes.
+ * bearer token in the `Authorization` header. Provider-controlled continuation
+ * state must never be able to redirect the access token to a third-party host.
+ * Mirrors `assertGraphNextPageUrl` used by the Graph tool routes.
  */
 function assertGraphNextLink(nextLink: string): string {
-  const url = new URL(nextLink.trim())
-  if (url.origin !== GRAPH_API_ORIGIN) {
-    throw new Error('Refusing to follow a non-Microsoft Graph @odata.nextLink')
-  }
-  return url.toString()
+  return assertMicrosoftGraphNextLink(nextLink)
 }
 
 /**
  * Depth-first traversal position carried across `listDocuments` calls.
  */
-interface OneDriveTraversalState {
-  /** Absolute `@odata.nextLink` for the current folder's next page, if any. */
-  nextLink?: string
-  /** Item id of the folder being listed; `undefined` means the configured root. */
-  currentFolder?: string
-  /** Subfolder ids discovered but not yet listed. */
-  folderStack: string[]
-}
+type OneDriveTraversalState = MicrosoftGraphTraversalState
 
 function decodeCursor(cursor: string): OneDriveTraversalState {
-  try {
-    const parsed = JSON.parse(cursor) as Partial<OneDriveTraversalState>
-    return {
-      nextLink: typeof parsed.nextLink === 'string' ? parsed.nextLink : undefined,
-      currentFolder: typeof parsed.currentFolder === 'string' ? parsed.currentFolder : undefined,
-      folderStack: Array.isArray(parsed.folderStack) ? parsed.folderStack : [],
-    }
-  } catch {
-    return { folderStack: [] }
-  }
+  return decodeMicrosoftGraphTraversalCursor(cursor, 'OneDrive')
 }
 
 export const onedriveConnector: ConnectorConfig = {
@@ -220,8 +226,7 @@ export const onedriveConnector: ConnectorConfig = {
   ): Promise<ExternalDocumentList> => {
     const folderPath = sourceConfig.folderPath as string | undefined
 
-    const parsedMaxFiles = Number(sourceConfig.maxFiles)
-    const maxFiles = Number.isFinite(parsedMaxFiles) && parsedMaxFiles > 0 ? parsedMaxFiles : 0
+    const maxFiles = parseMaxFiles(sourceConfig.maxFiles)
 
     const state: OneDriveTraversalState = cursor ? decodeCursor(cursor) : { folderStack: [] }
 
@@ -253,7 +258,7 @@ export const onedriveConnector: ConnectorConfig = {
       })
 
       if (!response.ok) {
-        const errorText = await response.text()
+        const errorText = await readBoundedHttpErrorBody(response)
         logger.error('Failed to list OneDrive files', {
           status: response.status,
           error: errorText,
@@ -261,10 +266,11 @@ export const onedriveConnector: ConnectorConfig = {
         throw new Error(`Failed to list OneDrive files: ${response.status}`)
       }
 
-      const data = (await response.json()) as OneDriveListResponse
-      const items = data.value || []
+      const data = parseMicrosoftGraphDriveItemList(await response.json(), 'OneDrive')
+      const items = data.value as OneDriveItem[]
 
       const files: OneDriveItem[] = []
+      const subfolders: string[] = []
       /**
        * Extensions this connector cannot index, tallied per page. A folder of
        * unsupported files otherwise syncs as "success, 0 documents", which reads
@@ -276,7 +282,7 @@ export const onedriveConnector: ConnectorConfig = {
 
       for (const item of items) {
         if (item.folder) {
-          state.folderStack.push(item.id)
+          subfolders.push(item.id)
         } else if (item.file) {
           if (isIndexableConnectorFile(item.name)) {
             // Keep oversized files; they are surfaced as skipped (failed) docs below.
@@ -305,7 +311,7 @@ export const onedriveConnector: ConnectorConfig = {
       documents.push(...take.documents)
       totalFetched += take.indexableCount
 
-      const nextLink = data['@odata.nextLink']
+      const nextLink = data.nextLink
 
       if (take.capReached) {
         done = true
@@ -319,10 +325,15 @@ export const onedriveConnector: ConnectorConfig = {
          * block deletion reconciliation for a complete listing.
          */
         cappedWithItemsLeft =
-          take.documents.length < stubs.length || Boolean(nextLink) || state.folderStack.length > 0
+          take.documents.length < stubs.length ||
+          Boolean(nextLink) ||
+          subfolders.length > 0 ||
+          state.folderStack.length > 0
         break
       }
 
+      appendPendingMicrosoftGraphFolders(state.folderStack, subfolders, 'OneDrive')
+
       if (nextLink) {
         state.nextLink = nextLink
         continue
@@ -353,7 +364,7 @@ export const onedriveConnector: ConnectorConfig = {
      */
     return {
       documents,
-      nextCursor: JSON.stringify(state),
+      nextCursor: encodeMicrosoftGraphTraversalCursor(state, 'OneDrive'),
       hasMore: true,
     }
   },
@@ -378,13 +389,23 @@ export const onedriveConnector: ConnectorConfig = {
       throw new Error(`Failed to get OneDrive file: ${response.status}`)
     }
 
-    const item = (await response.json()) as OneDriveItem
+    const item = parseOneDriveItemMetadata(await response.json(), externalId)
 
-    if (!item.file || !isIndexableConnectorFile(item.name)) return null
+    if (!item.file || !isIndexableConnectorFile(item.name)) {
+      return {
+        ...markSkipped(fileToStub(item), 'File is no longer an indexable document'),
+        skippedExistingDisposition: 'replace',
+      }
+    }
 
     try {
       const payload = await fetchFilePayload(accessToken, item.id, item.name)
-      if (!hasIndexablePayload(payload)) return null
+      if (!hasIndexablePayload(payload)) {
+        return {
+          ...markSkipped(fileToStub(item), 'Document contains no extractable text'),
+          skippedExistingDisposition: 'replace',
+        }
+      }
 
       const stub = fileToStub(item)
       return { ...stub, ...payload, contentDeferred: false }
@@ -410,10 +431,10 @@ export const onedriveConnector: ConnectorConfig = {
     sourceConfig: Record<string, unknown>
   ): Promise<{ valid: boolean; error?: string }> => {
     const folderPath = sourceConfig.folderPath as string | undefined
-    const maxFiles = sourceConfig.maxFiles as string | undefined
-
-    if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
-      return { valid: false, error: 'Max files must be a positive number' }
+    try {
+      parseMaxFiles(sourceConfig.maxFiles)
+    } catch (error) {
+      return { valid: false, error: toError(error).message }
     }
 
     try {

apps/sim/connectors/sharepoint/sharepoint.test.ts

@@ -7,6 +7,7 @@ const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() }
 
 vi.mock('@/lib/knowledge/documents/utils', () => ({
   fetchWithRetry: mockFetchWithRetry,
+  readBoundedHttpErrorBody: async (response: Response) => response.text(),
   VALIDATE_RETRY_OPTIONS: {},
 }))
 vi.mock('@/components/icons', () => ({ MicrosoftSharepointIcon: () => null }))
@@ -17,6 +18,11 @@ import {
   serverRelativePathFromUrl,
   sharepointConnector,
 } from '@/connectors/sharepoint/sharepoint'
+import {
+  appendPendingMicrosoftGraphFolders,
+  encodeMicrosoftGraphTraversalCursor,
+  MICROSOFT_GRAPH_MAX_PENDING_FOLDERS,
+} from '@/connectors/utils'
 
 const GRAPH = 'https://graph.microsoft.com/v1.0'
 const SITE_ID = 'contoso.sharepoint.com,site-guid,web-guid'
@@ -46,14 +52,17 @@ function mockGraph(routes: Record<string, GraphRoute>) {
     requested.push(url)
     const route = routes[url] ?? { status: 404 }
     const status = route.status ?? 200
+    const responseBytes = Buffer.from(
+      route.raw ? String(route.body ?? '') : JSON.stringify(route.body ?? {})
+    )
     return {
       ok: status >= 200 && status < 300,
       status,
+      headers: new Headers({ 'content-length': String(responseBytes.byteLength) }),
       json: async () => route.body,
       text: async () => JSON.stringify(route.body ?? {}),
       /** `readBodyWithLimit` falls back to this when there is no stream body. */
-      arrayBuffer: async () =>
-        Buffer.from(route.raw ? String(route.body ?? '') : JSON.stringify(route.body ?? {})),
+      arrayBuffer: async () => responseBytes,
     } as unknown as Response
   })
   return requested
@@ -339,7 +348,7 @@ describe('resolveFolderTarget', () => {
 })
 
 const ITEM_SELECT =
-  'id,name,webUrl,size,file,folder,lastModifiedDateTime,createdDateTime,createdBy,parentReference'
+  'id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdDateTime,createdBy,parentReference'
 
 /** File-shaped drive item for children listings. */
 function file(id: string, name: string) {
@@ -416,6 +425,33 @@ describe('listDocuments', () => {
     expect(syncContext.listingCapped).toBeUndefined()
   })
 
+  it('does not retain irrelevant folders after maxFiles has stopped traversal', async () => {
+    mockGraph(
+      childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), folder('overflow', 'Overflow')])
+    )
+    const cursor = encodeMicrosoftGraphTraversalCursor(
+      {
+        folderStack: Array.from(
+          { length: MICROSOFT_GRAPH_MAX_PENDING_FOLDERS },
+          (_, index) => `pending-${index}`
+        ),
+      },
+      'SharePoint'
+    )
+    const syncContext = listContext()
+
+    const result = await sharepointConnector.listDocuments(
+      'token',
+      { siteUrl: SITE_URL, maxFiles: '1' },
+      cursor,
+      syncContext
+    )
+
+    expect(result.documents.map((document) => document.externalId)).toEqual(['f1'])
+    expect(result.hasMore).toBe(false)
+    expect(syncContext.listingCapped).toBe(true)
+  })
+
   /**
    * The reported failure: a document library of Office SOPs synced as
    * "success, 0 documents" because the listing filter accepted only plain text,
@@ -465,6 +501,78 @@ describe('listDocuments', () => {
     expect(result.documents[0].contentHash).toBe('sharepoint:f1:2026-01-01T00:00:00Z')
     expect(result.documents[0].contentDeferred).toBe(true)
   })
+
+  it.each(['1.5', 'Infinity'])(
+    'rejects invalid maxFiles %s before calling Graph',
+    async (maxFiles) => {
+      const requested = mockGraph({})
+
+      await expect(
+        sharepointConnector.listDocuments(
+          'token',
+          { siteUrl: SITE_URL, maxFiles },
+          undefined,
+          listContext()
+        )
+      ).rejects.toThrow(/positive safe integer/)
+      expect(requested).toHaveLength(0)
+    }
+  )
+})
+
+describe('validateConfig', () => {
+  it.each(['1.5', 'Infinity'])(
+    'rejects invalid maxFiles %s without calling Graph',
+    async (maxFiles) => {
+      const requested = mockGraph({})
+
+      await expect(
+        sharepointConnector.validateConfig!('token', { siteUrl: SITE_URL, maxFiles })
+      ).resolves.toEqual({
+        valid: false,
+        error: 'Max files must be a positive safe integer, or 0 for unlimited',
+      })
+      expect(requested).toHaveLength(0)
+    }
+  )
+
+  it('accepts a valid integer maxFiles', async () => {
+    const siteRoute = `${GRAPH}/sites/${SITE_URL}`
+    const requested = mockGraph({
+      [siteRoute]: { body: { id: SITE_ID, displayName: 'Contoso' } },
+      ...defaultDriveRoute,
+    })
+
+    await expect(
+      sharepointConnector.validateConfig!('token', { siteUrl: SITE_URL, maxFiles: '25' })
+    ).resolves.toEqual({ valid: true })
+    expect(requested).toEqual([siteRoute, Object.keys(defaultDriveRoute)[0]])
+  })
+})
+
+describe('SharePoint traversal working-set bound', () => {
+  it('accepts discovered folders up to the pending-folder ceiling', () => {
+    const pending = Array.from(
+      { length: MICROSOFT_GRAPH_MAX_PENDING_FOLDERS - 2 },
+      (_, index) => `pending-${index}`
+    )
+
+    appendPendingMicrosoftGraphFolders(pending, ['last-1', 'last-2'], 'SharePoint')
+
+    expect(pending).toHaveLength(MICROSOFT_GRAPH_MAX_PENDING_FOLDERS)
+  })
+
+  it('stops before retaining a folder page beyond the ceiling', () => {
+    const pending = Array.from(
+      { length: MICROSOFT_GRAPH_MAX_PENDING_FOLDERS },
+      (_, index) => `pending-${index}`
+    )
+
+    expect(() => appendPendingMicrosoftGraphFolders(pending, ['overflow'], 'SharePoint')).toThrow(
+      /Narrow the connector/
+    )
+    expect(pending).toHaveLength(MICROSOFT_GRAPH_MAX_PENDING_FOLDERS)
+  })
 })
 
 describe('getDocument content extraction', () => {
@@ -492,6 +600,21 @@ describe('getDocument content extraction', () => {
     )
   }
 
+  it.each([{}, { id: 'f1', name: 'Missing facet' }, file('different', 'a.txt')])(
+    'rejects malformed metadata instead of replacing retained content',
+    async (metadata) => {
+      mockGraph({
+        [`${GRAPH}/drives/${DEFAULT_DRIVE_ID}/items/f1?$select=${ITEM_SELECT}`]: {
+          body: metadata,
+        },
+      })
+
+      await expect(get('f1')).rejects.toThrow(
+        'Microsoft Graph returned malformed SharePoint item metadata'
+      )
+    }
+  )
+
   /**
    * The connector hands an Office document over untouched so the shared pipeline
    * parses it — the same path an upload of the same file takes, which is what
@@ -532,6 +655,40 @@ describe('getDocument content extraction', () => {
     expect(doc?.sourceFile).toBeUndefined()
     expect(doc?.mimeType).toBe('text/plain')
   })
+
+  it('authoritatively skips a listed file that changed to a folder', async () => {
+    mockGraph({
+      [`${GRAPH}/drives/${DEFAULT_DRIVE_ID}/items/folder?$select=${ITEM_SELECT}`]: {
+        body: folder('folder', 'Former document'),
+      },
+    })
+
+    await expect(get('folder')).resolves.toMatchObject({
+      content: '',
+      skippedReason: 'File is no longer an indexable document',
+      skippedExistingDisposition: 'replace',
+    })
+  })
+
+  it('marks an empty file as an authoritative skip', async () => {
+    const item = file('empty', 'empty.txt')
+    mockFetchWithRetry.mockImplementation(async (url: string) => {
+      if (url.endsWith('/content')) return new Response(new Uint8Array(0))
+      return new Response(JSON.stringify(item), {
+        headers: { 'content-type': 'application/json' },
+      })
+    })
+
+    const document = await get('empty')
+
+    expect(document).toMatchObject({
+      externalId: 'empty',
+      content: '',
+      contentDeferred: false,
+      skippedReason: 'Document contains no extractable text',
+      skippedExistingDisposition: 'replace',
+    })
+  })
 })
 
 describe('serverRelativePathFromUrl', () => {

apps/sim/connectors/sharepoint/sharepoint.ts

@@ -1,17 +1,30 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
-import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
+import { isPlainRecord } from '@sim/utils/object'
+import {
+  fetchWithRetry,
+  readBoundedHttpErrorBody,
+  VALIDATE_RETRY_OPTIONS,
+} from '@/lib/knowledge/documents/utils'
 import { sharepointConnectorMeta } from '@/connectors/sharepoint/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import {
+  appendPendingMicrosoftGraphFolders,
+  assertMicrosoftGraphNextLink,
   CONNECTOR_MAX_FILE_BYTES,
   ConnectorFileTooLargeError,
   connectorFileExtension,
+  decodeMicrosoftGraphTraversalCursor,
+  encodeMicrosoftGraphTraversalCursor,
   extractConnectorText,
   hasIndexablePayload,
   isIndexableConnectorFile,
+  isMicrosoftGraphDriveItem,
   isSkippedDocument,
+  type MicrosoftGraphTraversalState,
   markSkipped,
+  parseMicrosoftGraphDriveItemList,
+  parseOptionalUnlimitedSafeInteger,
   parseTagDate,
   pipelineParsedMimeType,
   readBodyWithLimit,
@@ -35,7 +48,7 @@ const MAX_LOGGED_SKIPPED_EXTENSIONS = 10
  * driveItem otherwise, which is an order of magnitude larger per item.
  */
 const ITEM_SELECT =
-  'id,name,webUrl,size,file,folder,lastModifiedDateTime,createdDateTime,createdBy,parentReference'
+  'id,name,webUrl,size,file,folder,package,remoteItem,lastModifiedDateTime,createdDateTime,createdBy,parentReference'
 
 /**
  * Folder pages listed within a single `listDocuments` call. The sync engine caps
@@ -45,6 +58,22 @@ const ITEM_SELECT =
  */
 const MAX_LIST_REQUESTS_PER_CALL = 25
 
+/**
+ * Maximum breadth retained by the depth-first library walk.
+ *
+ * A library page can contribute 200 folders, and the sync engine may request
+ * 12,500 Graph pages in one run. Without this guard a folder-heavy tenant can
+ * retain millions of IDs and serialize them into a multi-megabyte cursor before
+ * yielding one document. Exceeding the bound fails visibly so users can scope
+ * the connector to a narrower library or folder.
+ */
+function parseMaxFiles(value: unknown): number {
+  return parseOptionalUnlimitedSafeInteger(
+    value,
+    'Max files must be a positive safe integer, or 0 for unlimited'
+  )
+}
+
 /** Microsoft Graph drive item shape (subset of fields we use). */
 interface DriveItem {
   id: string
@@ -53,15 +82,23 @@ interface DriveItem {
   size?: number
   file?: { mimeType?: string }
   folder?: { childCount?: number }
+  package?: Record<string, unknown>
+  remoteItem?: Record<string, unknown>
   lastModifiedDateTime?: string
   createdDateTime?: string
   createdBy?: { user?: { displayName?: string } }
   parentReference?: { path?: string; siteId?: string }
 }
 
-interface DriveItemListResponse {
-  value: DriveItem[]
-  '@odata.nextLink'?: string
+function isDriveItemMetadata(value: unknown, expectedId: string): value is DriveItem {
+  return isMicrosoftGraphDriveItem(value) && value.id === expectedId
+}
+
+function parseDriveItemMetadata(value: unknown, expectedId: string): DriveItem {
+  if (!isDriveItemMetadata(value, expectedId)) {
+    throw new Error('Microsoft Graph returned malformed SharePoint item metadata')
+  }
+  return value
 }
 
 /** Microsoft Graph drive (document library) shape (subset of fields we use). */
@@ -76,6 +113,24 @@ interface DriveListResponse {
   '@odata.nextLink'?: string
 }
 
+function parseDriveListResponse(value: unknown): DriveListResponse {
+  if (!isPlainRecord(value) || !Array.isArray(value.value)) {
+    throw new Error('Microsoft Graph returned malformed SharePoint drive-list metadata')
+  }
+  if (
+    !value.value.every(
+      (drive) => isPlainRecord(drive) && typeof drive.id === 'string' && drive.id.length > 0
+    )
+  ) {
+    throw new Error('Microsoft Graph returned malformed SharePoint drive metadata')
+  }
+  const nextLink =
+    value['@odata.nextLink'] === undefined
+      ? undefined
+      : assertMicrosoftGraphNextLink(value['@odata.nextLink'])
+  return { value: value.value as Drive[], ...(nextLink ? { '@odata.nextLink': nextLink } : {}) }
+}
+
 /** A configured folder path resolved to a concrete drive and starting folder. */
 interface ResolvedFolderTarget {
   driveId: string
@@ -94,11 +149,7 @@ type RetryOptions = Parameters<typeof fetchWithRetry>[2]
  * party. Mirrors `assertGraphNextPageUrl` used by the Graph tool routes.
  */
 function assertGraphUrl(url: string): string {
-  const parsed = new URL(url.trim())
-  if (parsed.origin !== GRAPH_API_ORIGIN) {
-    throw new Error('Refusing to follow a non-Microsoft Graph URL')
-  }
-  return parsed.toString()
+  return assertMicrosoftGraphNextLink(url)
 }
 
 /**
@@ -164,7 +215,7 @@ async function resolveSiteId(
   const response = await graphGet(url, accessToken, retryOptions)
 
   if (!response.ok) {
-    const errorText = await response.text()
+    const errorText = await readBoundedHttpErrorBody(response)
     throw new Error(
       `Failed to resolve SharePoint site "${siteUrl}": ${response.status} – ${errorText}`
     )
@@ -261,7 +312,7 @@ async function listFolderItems(
   driveId: string,
   folderId?: string,
   nextLink?: string
-): Promise<DriveItemListResponse> {
+): Promise<{ value: DriveItem[]; nextLink?: string }> {
   const url =
     nextLink ??
     (folderId
@@ -271,11 +322,12 @@ async function listFolderItems(
   const response = await graphGet(url, accessToken)
 
   if (!response.ok) {
-    const errorText = await response.text()
+    const errorText = await readBoundedHttpErrorBody(response)
     throw new Error(`Failed to list folder items: ${response.status} – ${errorText}`)
   }
 
-  return response.json() as Promise<DriveItemListResponse>
+  const data = parseMicrosoftGraphDriveItemList(await response.json(), 'SharePoint')
+  return { value: data.value as DriveItem[], nextLink: data.nextLink }
 }
 
 /**
@@ -378,12 +430,31 @@ async function listChildFolders(
       throw new Error(`Failed to list folder contents: ${response.status}`)
     }
 
-    const data = (await response.json()) as DriveItemListResponse
-    for (const item of data.value) {
+    const rawData: unknown = await response.json()
+    if (!isPlainRecord(rawData) || !Array.isArray(rawData.value)) {
+      throw new Error('Microsoft Graph returned malformed SharePoint folder-list metadata')
+    }
+    if (
+      !rawData.value.every(
+        (item) =>
+          isPlainRecord(item) &&
+          typeof item.id === 'string' &&
+          item.id.length > 0 &&
+          typeof item.name === 'string' &&
+          (item.folder === undefined || isPlainRecord(item.folder))
+      )
+    ) {
+      throw new Error('Microsoft Graph returned malformed SharePoint folder metadata')
+    }
+    const items = rawData.value as DriveItem[]
+    for (const item of items) {
       if (item.folder) folders.push(item)
     }
 
-    const nextLink = data['@odata.nextLink']
+    const nextLink =
+      rawData['@odata.nextLink'] === undefined
+        ? undefined
+        : assertMicrosoftGraphNextLink(rawData['@odata.nextLink'])
     if (!nextLink) break
     url = nextLink
   }
@@ -598,8 +669,8 @@ async function listSiteDrives(
   for (let page = 0; page < MAX_DRIVE_PAGES; page++) {
     const response = await graphGet(url, accessToken, retryOptions)
     if (!response.ok) break
-    const data = (await response.json()) as DriveListResponse
-    drives.push(...(data.value ?? []))
+    const data = parseDriveListResponse(await response.json())
+    drives.push(...data.value)
     const nextLink = data['@odata.nextLink']
     if (!nextLink) break
     url = nextLink
@@ -685,21 +756,14 @@ async function buildFolderNotFoundMessage(
  * Pagination state encoded as the cursor string.
  * We track a stack of folder IDs to traverse plus an optional @odata.nextLink.
  */
-interface PaginationState {
-  /** Folders still to be listed (depth-first) */
-  folderStack: string[]
-  /** Current folder being listed (undefined = root) */
-  currentFolder?: string
-  /** @odata.nextLink for the current folder page */
-  nextLink?: string
-}
+type PaginationState = MicrosoftGraphTraversalState
 
 function encodeCursor(state: PaginationState): string {
-  return Buffer.from(JSON.stringify(state)).toString('base64')
+  return encodeMicrosoftGraphTraversalCursor(state, 'SharePoint')
 }
 
 function decodeCursor(cursor: string): PaginationState {
-  return JSON.parse(Buffer.from(cursor, 'base64').toString('utf-8')) as PaginationState
+  return decodeMicrosoftGraphTraversalCursor(cursor, 'SharePoint')
 }
 
 export const sharepointConnector: ConnectorConfig = {
@@ -765,7 +829,7 @@ export const sharepointConnector: ConnectorConfig = {
     }
 
     const documents: ExternalDocument[] = []
-    const maxFiles = sourceConfig.maxFiles ? Number(sourceConfig.maxFiles) : 0
+    const maxFiles = parseMaxFiles(sourceConfig.maxFiles)
     let totalFetched = (syncContext?.totalDocsFetched as number) ?? 0
 
     /** Set when the walk stopped for good — either the cap hit or the source ran out. */
@@ -813,9 +877,6 @@ export const sharepointConnector: ConnectorConfig = {
         })
       }
 
-      // Push subfolders onto the stack for depth-first traversal
-      state.folderStack.push(...subfolders)
-
       // Convert files to lightweight stubs (no content download). Oversized files are
       // kept as skipped stubs but do not consume the max-files cap.
       const stubs = files.map((file) =>
@@ -825,7 +886,7 @@ export const sharepointConnector: ConnectorConfig = {
       documents.push(...take.documents)
       totalFetched += take.indexableCount
 
-      const nextLink = data['@odata.nextLink']
+      const nextLink = data.nextLink
 
       if (take.capReached) {
         stopPaging = true
@@ -838,10 +899,16 @@ export const sharepointConnector: ConnectorConfig = {
          * or in folders still on the stack.
          */
         cappedWithItemsLeft =
-          take.documents.length < stubs.length || Boolean(nextLink) || state.folderStack.length > 0
+          take.documents.length < stubs.length ||
+          Boolean(nextLink) ||
+          subfolders.length > 0 ||
+          state.folderStack.length > 0
         break
       }
 
+      /** A max-files stop must not validate folders beyond the requested scope. */
+      appendPendingMicrosoftGraphFolders(state.folderStack, subfolders, 'SharePoint')
+
       if (nextLink) {
         // More pages in the current folder
         state.nextLink = nextLink
@@ -924,15 +991,29 @@ export const sharepointConnector: ConnectorConfig = {
       throw new Error(`Failed to get SharePoint file: ${response.status}`)
     }
 
-    const item = (await response.json()) as DriveItem
+    const item = parseDriveItemMetadata(await response.json(), externalId)
 
     if (!item.file || !isIndexableConnectorFile(item.name)) {
-      return null
+      return {
+        ...markSkipped(
+          itemToStub(item, siteName ?? siteUrl),
+          'File is no longer an indexable document'
+        ),
+        skippedExistingDisposition: 'replace',
+      }
     }
 
     try {
       const payload = await fetchFilePayload(accessToken, driveId, item.id, item.name)
-      if (!hasIndexablePayload(payload)) return null
+      if (!hasIndexablePayload(payload)) {
+        return {
+          ...markSkipped(
+            itemToStub(item, siteName ?? siteUrl),
+            'Document contains no extractable text'
+          ),
+          skippedExistingDisposition: 'replace',
+        }
+      }
 
       const stub = itemToStub(item, siteName ?? siteUrl)
       return { ...stub, ...payload, contentDeferred: false }
@@ -965,9 +1046,10 @@ export const sharepointConnector: ConnectorConfig = {
       return { valid: false, error: 'Site URL is required' }
     }
 
-    const maxFiles = sourceConfig.maxFiles as string | undefined
-    if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
-      return { valid: false, error: 'Max files must be a positive number' }
+    try {
+      parseMaxFiles(sourceConfig.maxFiles)
+    } catch (error) {
+      return { valid: false, error: toError(error).message }
     }
 
     try {

apps/sim/connectors/types.ts

@@ -56,6 +56,11 @@ export interface ExternalDocument {
   sourceUrl?: string
   /** Hash of content for change detection (format varies by connector) */
   contentHash: string
+  /**
+   * Connector-owned hash to persist for a skipped hydration that must be retried
+   * even when the source's listing metadata is unchanged.
+   */
+  skippedRetryContentHash?: string
   /** When true, content is empty and will be fetched via getDocument for new/changed docs only */
   contentDeferred?: boolean
   /**
@@ -65,6 +70,12 @@ export interface ExternalDocument {
    * being silently dropped.
    */
   skippedReason?: string
+  /**
+   * Controls what happens when a previously indexed document is intentionally
+   * skipped. The default retains its last-known-good content; `replace` removes
+   * stale indexed content and persists the skipped state as authoritative.
+   */
+  skippedExistingDisposition?: 'replace'
   /** Additional source-specific metadata */
   metadata?: Record<string, unknown>
 }
@@ -76,8 +87,26 @@ export interface ExternalDocumentList {
   documents: ExternalDocument[]
   nextCursor?: string
   hasMore: boolean
+  /**
+   * Whether absence from this listing is authoritative enough for deletion
+   * reconciliation. Defaults to true. Offset-based or otherwise unstable
+   * provider pagination must set this to false.
+   */
+  reconciliationSafe?: boolean
 }
 
+export const SYNC_SKIP_REASONS = [
+  'connector_unavailable',
+  'knowledge_base_deleted',
+  'connector_not_syncable',
+  'dispatch_superseded',
+  'sync_in_progress',
+  'sync_superseded',
+  'connector_deleted_during_sync',
+] as const
+
+export type SyncSkipReason = (typeof SYNC_SKIP_REASONS)[number]
+
 /**
  * Result of a sync operation.
  */
@@ -86,7 +115,19 @@ export interface SyncResult {
   docsUpdated: number
   docsDeleted: number
   docsUnchanged: number
+  /** Source documents intentionally recorded without indexing, such as oversized files. */
+  docsSkipped: number
+  /** Source documents that failed during listing, hydration, or persistence. */
   docsFailed: number
+  /** Immediate hand-off outcome; eventual child results live on document rows and child runs. */
+  processingDispatch: {
+    requested: number
+    accepted: number
+    failed: number
+  }
+  /** Expected queue, lifecycle, or lock no-op. Never derived from error text. */
+  skipReason?: SyncSkipReason
+  /** Diagnostic for an actual failed sync. */
   error?: string
 }
 

apps/sim/connectors/utils.test.ts

@@ -61,12 +61,19 @@ import { s3Connector } from '@/connectors/s3/s3'
 import { sentryConnector } from '@/connectors/sentry/sentry'
 import { typeformConnector } from '@/connectors/typeform/typeform'
 import {
+  appendPendingMicrosoftGraphFolders,
+  assertMicrosoftGraphNextLink,
   ConnectorFileTooLargeError,
+  decodeMicrosoftGraphTraversalCursor,
+  encodeMicrosoftGraphTraversalCursor,
   extractConnectorText,
   hasIndexablePayload,
   htmlToPlainText,
   isIndexableConnectorFile,
   isSkippedDocument,
+  MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES,
+  MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES,
+  MICROSOFT_GRAPH_MAX_PENDING_FOLDERS,
   markSkipped,
   pipelineParsedMimeType,
   readBodyWithLimit,
@@ -1180,33 +1187,107 @@ describe('readBodyWithLimit', () => {
     expect(result?.byteLength).toBe(2048)
   })
 
-  it('returns null and cancels the stream once the cap is exceeded', async () => {
-    const onCancel = vi.fn()
+  it('returns null as soon as the streamed cap is exceeded', async () => {
     const chunk = new Uint8Array(1024).fill(65)
-    // Cap is 2048; the third 1KB chunk pushes the total to 3072 and trips the cap,
-    // so the remaining body is never buffered into memory.
-    const result = await readBodyWithLimit(streamResponse([chunk, chunk, chunk], onCancel), 2048)
+    const onCancel = vi.fn()
+    const result = await readBodyWithLimit(
+      streamResponse([chunk, chunk, chunk, chunk], onCancel),
+      2048
+    )
     expect(result).toBeNull()
-    expect(onCancel).toHaveBeenCalled()
+    expect(onCancel).toHaveBeenCalledOnce()
   })
 
-  it('enforces the cap on bodyless responses via the arrayBuffer fallback', async () => {
+  it('does not materialize a bodyless response whose size is unknown', async () => {
+    const arrayBuffer = vi.fn(async () => new Uint8Array(5000).buffer)
     // double-cast-allowed: minimal response stub exercising the no-stream branch
-    const oversized = {
+    const unknownSize = {
       body: null,
-      arrayBuffer: async () => new Uint8Array(5000).buffer,
+      arrayBuffer,
     } as unknown as Response
-    expect(await readBodyWithLimit(oversized, 4096)).toBeNull()
+    expect(await readBodyWithLimit(unknownSize, 4096)).toBeNull()
+    expect(arrayBuffer).not.toHaveBeenCalled()
+  })
 
+  it('uses a trusted content length to bound a bodyless response fallback', async () => {
     // double-cast-allowed: minimal response stub exercising the no-stream branch
     const within = {
       body: null,
+      headers: new Headers({ 'content-length': '100' }),
       arrayBuffer: async () => new Uint8Array(100).buffer,
     } as unknown as Response
     expect((await readBodyWithLimit(within, 4096))?.byteLength).toBe(100)
   })
 })
 
+describe('Microsoft Graph traversal cursors', () => {
+  it('round-trips canonical cursors and accepts the former OneDrive JSON shape', () => {
+    const state = {
+      folderStack: ['folder-a', 'folder-b'],
+      currentFolder: 'folder-current',
+      nextLink: 'https://graph.microsoft.com/v1.0/me/drive/root/children?$skiptoken=abc',
+    }
+
+    expect(
+      decodeMicrosoftGraphTraversalCursor(
+        encodeMicrosoftGraphTraversalCursor(state, 'OneDrive'),
+        'OneDrive'
+      )
+    ).toEqual(state)
+    expect(decodeMicrosoftGraphTraversalCursor(JSON.stringify(state), 'OneDrive')).toEqual(state)
+  })
+
+  it('rejects off-origin continuation URLs before they can receive a bearer token', () => {
+    expect(() => assertMicrosoftGraphNextLink('https://evil.example/steal')).toThrow(
+      /non-Microsoft Graph/
+    )
+    expect(() =>
+      decodeMicrosoftGraphTraversalCursor(
+        Buffer.from(
+          JSON.stringify({
+            folderStack: [],
+            nextLink: 'https://evil.example/steal',
+          })
+        ).toString('base64'),
+        'SharePoint'
+      )
+    ).toThrow(/non-Microsoft Graph/)
+  })
+
+  it('rejects invalid and oversized cursor members', () => {
+    const encode = (state: unknown) => Buffer.from(JSON.stringify(state)).toString('base64')
+
+    expect(() =>
+      decodeMicrosoftGraphTraversalCursor(encode({ folderStack: [42] }), 'OneDrive')
+    ).toThrow(/must be a string/)
+    expect(() =>
+      decodeMicrosoftGraphTraversalCursor(
+        encode({ folderStack: ['x'.repeat(MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES + 1)] }),
+        'OneDrive'
+      )
+    ).toThrow(/size limit/)
+    expect(() =>
+      decodeMicrosoftGraphTraversalCursor(
+        'x'.repeat(MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES + 1),
+        'OneDrive'
+      )
+    ).toThrow(/encoded state exceeds/)
+  })
+
+  it('enforces the pending-folder cap before mutating traversal state', () => {
+    const pending = Array.from(
+      { length: MICROSOFT_GRAPH_MAX_PENDING_FOLDERS },
+      (_, index) => `folder-${index}`
+    )
+
+    expect(() => appendPendingMicrosoftGraphFolders(pending, ['overflow'], 'OneDrive')).toThrow(
+      /Narrow the connector/
+    )
+    expect(pending).toHaveLength(MICROSOFT_GRAPH_MAX_PENDING_FOLDERS)
+    expect(pending).not.toContain('overflow')
+  })
+})
+
 describe('markSkipped', () => {
   const stub: ExternalDocument = {
     externalId: 'file-1',

apps/sim/connectors/utils.ts

@@ -1,4 +1,9 @@
+import { isPlainRecord } from '@sim/utils/object'
 import type { SecureFetchResponse } from '@/lib/core/security/input-validation.server'
+import {
+  isPayloadSizeLimitError,
+  readResponseToBufferWithLimit,
+} from '@/lib/core/utils/stream-limits'
 import { MAX_FILE_SIZE as KB_DOCUMENT_MAX_BYTES } from '@/lib/uploads/utils/validation'
 import type { ExternalDocument } from '@/connectors/types'
 
@@ -14,6 +19,223 @@ import type { ExternalDocument } from '@/connectors/types'
  */
 export const CONNECTOR_MAX_FILE_BYTES = KB_DOCUMENT_MAX_BYTES
 
+/** Maximum number of Microsoft Graph folders retained between connector pages. */
+export const MICROSOFT_GRAPH_MAX_PENDING_FOLDERS = 10_000
+
+/**
+ * Graph IDs are opaque strings with no documented maximum. This ceiling is far
+ * above ordinary identifiers while making the traversal's string working set
+ * provably bounded even if a provider response or stored cursor is malformed.
+ */
+export const MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES = 512
+
+/** Maximum server-issued continuation URL accepted into a traversal cursor. */
+export const MICROSOFT_GRAPH_MAX_NEXT_LINK_BYTES = 16 * 1024
+
+/** Maximum serialized traversal state, before and after base64 encoding. */
+export const MICROSOFT_GRAPH_MAX_CURSOR_JSON_BYTES = 8 * 1024 * 1024
+export const MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES = 12 * 1024 * 1024
+
+/** Parses an optional connector limit where zero or omission means unlimited. */
+export function parseOptionalUnlimitedSafeInteger(value: unknown, errorMessage: string): number {
+  if (value === undefined || value === null) return 0
+  if (typeof value !== 'string' && typeof value !== 'number') {
+    throw new Error(errorMessage)
+  }
+
+  const normalized = typeof value === 'string' ? value.trim() : value
+  if (normalized === '') return 0
+  if (typeof normalized === 'string' && !/^\d+$/.test(normalized)) {
+    throw new Error(errorMessage)
+  }
+
+  const parsed = Number(normalized)
+  if (!Number.isSafeInteger(parsed) || parsed < 0) {
+    throw new Error(errorMessage)
+  }
+  return parsed
+}
+
+const MICROSOFT_GRAPH_ORIGIN = 'https://graph.microsoft.com'
+
+export interface MicrosoftGraphTraversalState {
+  nextLink?: string
+  currentFolder?: string
+  folderStack: string[]
+}
+
+export interface MicrosoftGraphDriveItemShape {
+  id: string
+  name: string
+  file?: Record<string, unknown>
+  folder?: Record<string, unknown>
+  package?: Record<string, unknown>
+  remoteItem?: Record<string, unknown>
+}
+
+export function isMicrosoftGraphDriveItem(value: unknown): value is MicrosoftGraphDriveItemShape {
+  return (
+    isPlainRecord(value) &&
+    typeof value.id === 'string' &&
+    value.id.length > 0 &&
+    typeof value.name === 'string' &&
+    value.name.length > 0 &&
+    (isPlainRecord(value.file) ||
+      isPlainRecord(value.folder) ||
+      isPlainRecord(value.package) ||
+      isPlainRecord(value.remoteItem))
+  )
+}
+
+export function parseMicrosoftGraphDriveItemList(
+  value: unknown,
+  label: string
+): { value: MicrosoftGraphDriveItemShape[]; nextLink?: string } {
+  if (!isPlainRecord(value) || !Array.isArray(value.value)) {
+    throw new Error(`Microsoft Graph returned malformed ${label} list metadata`)
+  }
+  if (!value.value.every(isMicrosoftGraphDriveItem)) {
+    throw new Error(`Microsoft Graph returned malformed ${label} item metadata`)
+  }
+  const nextLink =
+    value['@odata.nextLink'] === undefined
+      ? undefined
+      : assertMicrosoftGraphNextLink(value['@odata.nextLink'])
+  return { value: value.value, nextLink }
+}
+
+function assertBoundedGraphString(
+  value: unknown,
+  maxBytes: number,
+  field: string,
+  allowEmpty = false
+): asserts value is string {
+  if (typeof value !== 'string' || (!allowEmpty && value.length === 0)) {
+    throw new Error(`Invalid Microsoft Graph traversal cursor: ${field} must be a string`)
+  }
+  if (Buffer.byteLength(value, 'utf8') > maxBytes) {
+    throw new Error(`Invalid Microsoft Graph traversal cursor: ${field} exceeds its size limit`)
+  }
+}
+
+/** Validates a Graph continuation URL before a bearer token can be sent to it. */
+export function assertMicrosoftGraphNextLink(value: unknown): string {
+  assertBoundedGraphString(value, MICROSOFT_GRAPH_MAX_NEXT_LINK_BYTES, 'nextLink')
+  const url = new URL(value.trim())
+  if (url.origin !== MICROSOFT_GRAPH_ORIGIN) {
+    throw new Error('Refusing to follow a non-Microsoft Graph @odata.nextLink')
+  }
+  return url.toString()
+}
+
+function validateMicrosoftGraphTraversalState(
+  value: unknown,
+  label: string
+): MicrosoftGraphTraversalState {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(`Invalid ${label} pagination cursor`)
+  }
+
+  const candidate = value as Record<string, unknown>
+  if (!Array.isArray(candidate.folderStack)) {
+    throw new Error(`Invalid ${label} pagination cursor: folderStack must be an array`)
+  }
+  if (candidate.folderStack.length > MICROSOFT_GRAPH_MAX_PENDING_FOLDERS) {
+    throw new Error(
+      `${label} folder traversal exceeds the safe limit of ${MICROSOFT_GRAPH_MAX_PENDING_FOLDERS.toLocaleString()} pending folders. Narrow the connector to a document library or subfolder and retry.`
+    )
+  }
+
+  const folderStack = candidate.folderStack.map((folderId, index) => {
+    assertBoundedGraphString(folderId, MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES, `folderStack[${index}]`)
+    return folderId
+  })
+
+  let currentFolder: string | undefined
+  if (candidate.currentFolder !== undefined) {
+    assertBoundedGraphString(
+      candidate.currentFolder,
+      MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES,
+      'currentFolder'
+    )
+    currentFolder = candidate.currentFolder
+  }
+
+  const nextLink =
+    candidate.nextLink === undefined ? undefined : assertMicrosoftGraphNextLink(candidate.nextLink)
+
+  return { folderStack, currentFolder, nextLink }
+}
+
+/**
+ * Encodes bounded Graph traversal state. Base64 is canonical; decoding also
+ * accepts the former OneDrive raw-JSON cursor so in-flight syncs survive rollout.
+ */
+export function encodeMicrosoftGraphTraversalCursor(
+  state: MicrosoftGraphTraversalState,
+  label: string
+): string {
+  const validated = validateMicrosoftGraphTraversalState(state, label)
+  const json = JSON.stringify(validated)
+  if (Buffer.byteLength(json, 'utf8') > MICROSOFT_GRAPH_MAX_CURSOR_JSON_BYTES) {
+    throw new Error(`Invalid ${label} pagination cursor: serialized state exceeds its size limit`)
+  }
+  const encoded = Buffer.from(json).toString('base64')
+  if (Buffer.byteLength(encoded, 'utf8') > MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES) {
+    throw new Error(`Invalid ${label} pagination cursor: encoded state exceeds its size limit`)
+  }
+  return encoded
+}
+
+export function decodeMicrosoftGraphTraversalCursor(
+  cursor: string,
+  label: string
+): MicrosoftGraphTraversalState {
+  if (Buffer.byteLength(cursor, 'utf8') > MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES) {
+    throw new Error(`Invalid ${label} pagination cursor: encoded state exceeds its size limit`)
+  }
+
+  let json: string
+  try {
+    json = cursor.trimStart().startsWith('{')
+      ? cursor
+      : Buffer.from(cursor, 'base64').toString('utf8')
+  } catch {
+    throw new Error(`Invalid ${label} pagination cursor`)
+  }
+  if (Buffer.byteLength(json, 'utf8') > MICROSOFT_GRAPH_MAX_CURSOR_JSON_BYTES) {
+    throw new Error(`Invalid ${label} pagination cursor: serialized state exceeds its size limit`)
+  }
+
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(json)
+  } catch {
+    throw new Error(`Invalid ${label} pagination cursor`)
+  }
+  return validateMicrosoftGraphTraversalState(parsed, label)
+}
+
+export function appendPendingMicrosoftGraphFolders(
+  pendingFolders: string[],
+  discoveredFolders: string[],
+  label: string
+): void {
+  if (pendingFolders.length + discoveredFolders.length > MICROSOFT_GRAPH_MAX_PENDING_FOLDERS) {
+    throw new Error(
+      `${label} folder traversal exceeds the safe limit of ${MICROSOFT_GRAPH_MAX_PENDING_FOLDERS.toLocaleString()} pending folders. Narrow the connector to a document library or subfolder and retry.`
+    )
+  }
+  for (let index = 0; index < discoveredFolders.length; index++) {
+    assertBoundedGraphString(
+      discoveredFolders[index],
+      MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES,
+      `discoveredFolders[${index}]`
+    )
+  }
+  pendingFolders.push(...discoveredFolders)
+}
+
 /** The named entities connector markup actually carries, decoded to their character. */
 const NAMED_ENTITIES: Record<string, string> = {
   amp: '&',
@@ -369,25 +591,15 @@ export async function readBodyWithLimit(
   response: Response | SecureFetchResponse,
   maxBytes: number
 ): Promise<Buffer | null> {
-  if (!response.body) {
-    const buffer = Buffer.from(await response.arrayBuffer())
-    return buffer.byteLength > maxBytes ? null : buffer
+  try {
+    return await readResponseToBufferWithLimit(response, {
+      maxBytes,
+      label: 'Connector file download',
+    })
+  } catch (error) {
+    if (isPayloadSizeLimitError(error)) return null
+    throw error
   }
-
-  const reader = response.body.getReader()
-  const chunks: Uint8Array[] = []
-  let total = 0
-  while (true) {
-    const { done, value } = await reader.read()
-    if (done) break
-    total += value.byteLength
-    if (total > maxBytes) {
-      await reader.cancel().catch(() => {})
-      return null
-    }
-    chunks.push(value)
-  }
-  return Buffer.concat(chunks)
 }
 
 /**

apps/sim/lib/api/contracts/knowledge/connectors.ts

@@ -94,6 +94,7 @@ export const syncLogDataSchema = z
     docsUpdated: z.number(),
     docsDeleted: z.number(),
     docsUnchanged: z.number(),
+    docsSkipped: z.number().int().nonnegative().default(0),
     docsFailed: z.number(),
     errorMessage: z.string().nullable(),
   })

apps/sim/lib/api/contracts/v2/knowledge.ts

@@ -1595,6 +1595,12 @@ export const v2KnowledgeConnectorSyncLogSchema = z
     docsUpdated: z.number().int().nonnegative().describe('Documents updated.'),
     docsDeleted: z.number().int().nonnegative().describe('Documents deleted.'),
     docsUnchanged: z.number().int().nonnegative().describe('Documents unchanged.'),
+    docsSkipped: z
+      .number()
+      .int()
+      .nonnegative()
+      .default(0)
+      .describe('Documents intentionally skipped because they could not be indexed safely.'),
     docsFailed: z.number().int().nonnegative().describe('Documents that failed to synchronize.'),
     errorMessage: z.string().nullable().describe('Synchronization error, or null.'),
   })

apps/sim/lib/atlassian/discovery.test.ts

@@ -19,8 +19,8 @@ function options(over: Record<string, unknown> = {}) {
   >[0]
 }
 
-/** Tiny delays so retry cases do not spend real seconds sleeping. */
-const FAST = { initialDelayMs: 1, maxDelayMs: 1 }
+/** Tiny waits keep retry cases fast; the explicit budget absorbs test-runner scheduling latency. */
+const FAST = { initialDelayMs: 1, maxDelayMs: 1, retryBudgetMs: 1_000 }
 
 function sites(entries: Array<{ id: string; url: string }>) {
   return createMockResponse({ json: entries })
@@ -108,8 +108,6 @@ describe('resolveAtlassianCloudId', () => {
   it('gives up on a persistent fault within a bounded attempt budget', async () => {
     fetchMock.mockImplementation(async () => failure(500))
 
-    // Delays only. `maxRetries` still comes from the discovery budget, so this
-    // fails if the shared default of 5 ever leaks back in.
     await expect(resolveAtlassianCloudId(options({ retryOptions: FAST }))).rejects.toThrow(
       /Failed to fetch Jira accessible resources: 500/
     )

apps/sim/lib/chunkers/chunk-budget.ts

@@ -0,0 +1,31 @@
+/** Raised before a chunker would produce more than its configured output ceiling. */
+export class ChunkLimitExceededError extends Error {
+  readonly maxChunks: number
+
+  constructor(maxChunks: number) {
+    super(`Chunk production exceeded the configured limit of ${maxChunks.toLocaleString()}`)
+    this.name = 'ChunkLimitExceededError'
+    this.maxChunks = maxChunks
+  }
+}
+
+/** Shared output budget that lets recursive chunkers enforce one aggregate ceiling. */
+export class ChunkBudget {
+  private produced = 0
+  private readonly maxChunks?: number
+
+  constructor(maxChunks?: number) {
+    if (maxChunks !== undefined && (!Number.isSafeInteger(maxChunks) || maxChunks < 0)) {
+      throw new RangeError('maxChunks must be a non-negative safe integer')
+    }
+    this.maxChunks = maxChunks
+  }
+
+  add<T>(target: T[], value: T): void {
+    if (this.maxChunks !== undefined && this.produced >= this.maxChunks) {
+      throw new ChunkLimitExceededError(this.maxChunks)
+    }
+    target.push(value)
+    this.produced++
+  }
+}

apps/sim/lib/chunkers/chunk-limit.test.ts

@@ -0,0 +1,114 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import { ChunkLimitExceededError } from '@/lib/chunkers/chunk-budget'
+import { JsonYamlChunker } from '@/lib/chunkers/json-yaml-chunker'
+import { RecursiveChunker } from '@/lib/chunkers/recursive-chunker'
+import { RegexChunker } from '@/lib/chunkers/regex-chunker'
+import { SentenceChunker } from '@/lib/chunkers/sentence-chunker'
+import { StructuredDataChunker } from '@/lib/chunkers/structured-data-chunker'
+import { TextChunker } from '@/lib/chunkers/text-chunker'
+import { TokenChunker } from '@/lib/chunkers/token-chunker'
+
+const LIMIT_CASES = [
+  {
+    name: 'text',
+    run: () =>
+      new TextChunker({ chunkSize: 1, maxChunks: 2 }).chunk(
+        'alpha bravo charlie delta echo foxtrot'
+      ),
+  },
+  {
+    name: 'token',
+    run: () =>
+      new TokenChunker({ chunkSize: 1, maxChunks: 2 }).chunk(
+        'alpha bravo charlie delta echo foxtrot'
+      ),
+  },
+  {
+    name: 'sentence',
+    run: () =>
+      new SentenceChunker({ chunkSize: 2, maxChunks: 2 }).chunk(
+        'Alpha bravo. Charlie delta. Echo foxtrot. Golf hotel.'
+      ),
+  },
+  {
+    name: 'recursive',
+    run: () =>
+      new RecursiveChunker({ chunkSize: 1, maxChunks: 2, separators: ['|'] }).chunk(
+        'alpha|bravo|charlie|delta'
+      ),
+  },
+  {
+    name: 'regex',
+    run: () =>
+      new RegexChunker({
+        pattern: '---',
+        strictBoundaries: true,
+        chunkSize: 100,
+        maxChunks: 2,
+      }).chunk('alpha---bravo---charlie---delta'),
+  },
+  {
+    name: 'structured data',
+    run: () =>
+      StructuredDataChunker.chunkStructuredData(
+        [
+          'name,value',
+          ...Array.from({ length: 16 }, (_, index) => `row-${index},value-${index}`),
+        ].join('\n'),
+        { chunkSize: 1, maxChunks: 2 }
+      ),
+  },
+  {
+    name: 'JSON/YAML',
+    run: () =>
+      new JsonYamlChunker({ chunkSize: 1, maxChunks: 2 }).chunk(
+        JSON.stringify(['alpha', 'bravo', 'charlie', 'delta'])
+      ),
+  },
+] as const
+
+describe('chunk production ceiling', () => {
+  it.each(LIMIT_CASES)('$name chunker stops before producing a third chunk', async ({ run }) => {
+    await expect(run()).rejects.toBeInstanceOf(ChunkLimitExceededError)
+  })
+
+  it('does not retain budget usage across calls on one chunker instance', async () => {
+    const chunker = new TokenChunker({ chunkSize: 100, maxChunks: 1 })
+
+    await expect(chunker.chunk('first document')).resolves.toHaveLength(1)
+    await expect(chunker.chunk('second document')).resolves.toHaveLength(1)
+  })
+
+  it('admits the exact limit when a short trailing token chunk is filtered out', async () => {
+    const chunker = new TokenChunker({
+      chunkSize: 1,
+      minCharactersPerChunk: 4,
+      maxChunks: 2,
+    })
+
+    await expect(chunker.chunk('aaaa aaaa x')).resolves.toHaveLength(2)
+  })
+
+  it('stops separator-heavy text production at the configured ceiling', async () => {
+    const content = Array.from({ length: 50_000 }, () => 'x').join(' ')
+    const chunker = new TextChunker({ chunkSize: 1, maxChunks: 2 })
+
+    await expect(chunker.chunk(content)).rejects.toBeInstanceOf(ChunkLimitExceededError)
+  })
+
+  it('stops strict-regex splitting without materializing every remaining segment', async () => {
+    const content = Array.from({ length: 50_000 }, () => 'x').join('---')
+    const chunker = new RegexChunker({
+      pattern: '---',
+      strictBoundaries: true,
+      chunkSize: 100,
+      maxChunks: 2,
+    })
+
+    await expect(chunker.chunk(content)).rejects.toBeInstanceOf(ChunkLimitExceededError)
+  })
+})

apps/sim/lib/chunkers/docs-chunker.test.ts

@@ -8,6 +8,7 @@ vi.mock('@/lib/knowledge/embeddings', () => ({
   getConfiguredEmbeddingModel: vi.fn(() => 'test-model'),
 }))
 
+import { ChunkLimitExceededError } from '@/lib/chunkers/chunk-budget'
 import { DocsChunker } from '@/lib/chunkers/docs-chunker'
 
 function cleanContent(content: string): string {
@@ -129,3 +130,42 @@ describe('cleanContent scaffolding strips', () => {
     expect(cleaned).toContain('Inside text stays')
   })
 })
+
+describe('DocsChunker output budget', () => {
+  function splitContent(
+    chunker: DocsChunker,
+    content: string
+  ): Promise<{ chunks: string[]; cleanedContent: string }> {
+    return (
+      chunker as unknown as {
+        splitContent(content: string): Promise<{ chunks: string[]; cleanedContent: string }>
+      }
+    ).splitContent.call(chunker, content)
+  }
+
+  it('applies maxChunks after short intermediate chunks are filtered out', async () => {
+    const chunker = new DocsChunker({ chunkSize: 1, chunkOverlap: 0, maxChunks: 1 })
+
+    await expect(splitContent(chunker, 'a b c d')).resolves.toEqual({
+      chunks: [],
+      cleanedContent: 'a b c d',
+    })
+  })
+
+  it('rejects when the final transformed output exceeds maxChunks', async () => {
+    const chunker = new DocsChunker({ chunkSize: 30, chunkOverlap: 0, maxChunks: 1 })
+    const content = `${'a'.repeat(110)}\n\n${'b'.repeat(110)}`
+
+    await expect(splitContent(chunker, content)).rejects.toThrow(ChunkLimitExceededError)
+  })
+
+  it('enforces maxChunks after oversized chunks are split into final chunks', () => {
+    const chunker = new DocsChunker({ chunkSize: 30, maxChunks: 1 })
+    const enforceSizeLimit = (
+      chunker as unknown as { enforceSizeLimit(chunks: string[]): string[] }
+    ).enforceSizeLimit.bind(chunker)
+    const longLine = 'a'.repeat(120)
+
+    expect(() => enforceSizeLimit([`${longLine}\n${longLine}`])).toThrow(ChunkLimitExceededError)
+  })
+})

apps/sim/lib/chunkers/docs-chunker.ts

@@ -1,6 +1,7 @@
 import fs from 'fs/promises'
 import path from 'path'
 import { createLogger } from '@sim/logger'
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import { TextChunker } from '@/lib/chunkers/text-chunker'
 import type { DocChunk, DocsChunkerOptions } from '@/lib/chunkers/types'
 import { estimateTokens } from '@/lib/chunkers/utils'
@@ -64,9 +65,11 @@ export class DocsChunker {
   private readonly textChunker: TextChunker
   private readonly baseUrl: string
   private readonly chunkSize: number
+  private readonly maxChunks?: number
 
   constructor(options: DocsChunkerOptions = {}) {
     this.chunkSize = options.chunkSize ?? 300
+    this.maxChunks = options.maxChunks
     this.textChunker = new TextChunker({
       chunkSize: this.chunkSize,
       minCharactersPerChunk: options.minCharactersPerChunk ?? 1,
@@ -395,12 +398,17 @@ export class DocsChunker {
 
   private enforceSizeLimit(chunks: string[]): string[] {
     const finalChunks: string[] = []
+    const budget = new ChunkBudget(this.maxChunks)
+    const addFinalChunk = (chunk: string): void => {
+      const normalized = chunk.trim()
+      if (normalized.length > 100) budget.add(finalChunks, normalized)
+    }
 
     for (const chunk of chunks) {
       const tokens = estimateTokens(chunk)
 
       if (tokens <= this.chunkSize) {
-        finalChunks.push(chunk)
+        addFinalChunk(chunk)
       } else {
         const lines = chunk.split('\n')
         let currentChunk = ''
@@ -412,18 +420,18 @@ export class DocsChunker {
             currentChunk = testChunk
           } else {
             if (currentChunk.trim()) {
-              finalChunks.push(currentChunk.trim())
+              addFinalChunk(currentChunk)
             }
             currentChunk = line
           }
         }
 
         if (currentChunk.trim()) {
-          finalChunks.push(currentChunk.trim())
+          addFinalChunk(currentChunk)
         }
       }
     }
 
-    return finalChunks.filter((chunk) => chunk.trim().length > 100)
+    return finalChunks
   }
 }

apps/sim/lib/chunkers/index.ts

@@ -1,3 +1,4 @@
+export { ChunkBudget, ChunkLimitExceededError } from './chunk-budget'
 export { DocsChunker } from './docs-chunker'
 export { JsonYamlChunker } from './json-yaml-chunker'
 export { RecursiveChunker } from './recursive-chunker'

apps/sim/lib/chunkers/json-yaml-chunker.test.ts

@@ -14,6 +14,29 @@ vi.mock('@/lib/tokenization/estimators', () => ({
 }))
 
 describe('JsonYamlChunker', () => {
+  it.each([
+    0,
+    -1,
+    0.5,
+    Number.NaN,
+    Number.NEGATIVE_INFINITY,
+    Number.POSITIVE_INFINITY,
+    Number.MAX_SAFE_INTEGER,
+  ])('rejects an invalid chunk size at construction: %s', (chunkSize) => {
+    expect(() => new JsonYamlChunker({ chunkSize })).toThrow(
+      'JSON/YAML chunk size must be a finite number between 1'
+    )
+  })
+
+  it('normalizes a fractional legacy chunk size to its integer token ceiling', async () => {
+    const chunks = await new JsonYamlChunker({ chunkSize: 100.5, minCharactersPerChunk: 1 }).chunk(
+      JSON.stringify({ value: 'x'.repeat(1_000) })
+    )
+
+    expect(chunks.length).toBeGreaterThan(1)
+    expect(chunks.every((chunk) => chunk.tokenCount <= 100)).toBe(true)
+  })
+
   describe('isStructuredData', () => {
     it('should detect valid JSON', () => {
       expect(JsonYamlChunker.isStructuredData('{"key": "value"}')).toBe(true)
@@ -269,6 +292,17 @@ server:
 
       expect(chunks.length).toBeGreaterThan(0)
     })
+
+    it('should fall back to bounded text chunking when YAML traversal fails', async () => {
+      const chunker = new JsonYamlChunker({ chunkSize: 1000, minCharactersPerChunk: 1 })
+      const cyclicYaml = ['root: &root', '  value: readable', '  self: *root'].join('\n')
+
+      const chunks = await chunker.chunk(cyclicYaml)
+
+      expect(chunks).toHaveLength(1)
+      expect(chunks[0].text).toContain('&root')
+      expect(chunks[0].text).toContain('readable')
+    })
   })
 
   describe('large inputs', () => {
@@ -293,6 +327,32 @@ server:
       const chunks = await chunker.chunk(json)
 
       expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 100)).toBe(true)
+    })
+
+    it('splits a long single-line scalar to the configured chunk size', async () => {
+      const chunker = new JsonYamlChunker({ chunkSize: 1024, minCharactersPerChunk: 1 })
+      const json = JSON.stringify({ value: `START-${'x'.repeat(32_755)}-END` })
+
+      const chunks = await chunker.chunk(json)
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 1024)).toBe(true)
+      expect(chunks.some((chunk) => chunk.text.includes('START-'))).toBe(true)
+      expect(chunks.some((chunk) => chunk.text.includes('-END'))).toBe(true)
+    })
+
+    it('splits a long scalar nested beyond the structured traversal depth', async () => {
+      const chunker = new JsonYamlChunker({ chunkSize: 1024, minCharactersPerChunk: 1 })
+      let nested: unknown = `START-${'x'.repeat(40_000)}-END`
+      for (let depth = 0; depth < 6; depth++) nested = [nested]
+
+      const chunks = await chunker.chunk(JSON.stringify(nested))
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 1024)).toBe(true)
+      expect(chunks.some((chunk) => chunk.text.includes('START-'))).toBe(true)
+      expect(chunks.some((chunk) => chunk.text.includes('-END'))).toBe(true)
     })
 
     it.concurrent('should handle deeply nested structure up to depth limit', async () => {
@@ -331,6 +391,82 @@ server:
   })
 
   describe('chunk metadata', () => {
+    it('preserves every source character and offset when bounding oversized chunks', async () => {
+      const key = 'p'.repeat(80)
+      const value = { value: 'alpha beta gamma' }
+      const expectedText = `// ${key}\n${JSON.stringify(value, null, 2)}`
+      const chunker = new JsonYamlChunker({ chunkSize: 10, minCharactersPerChunk: 1 })
+
+      const chunks = await chunker.chunk(JSON.stringify({ [key]: value }))
+
+      expect(chunks.length).toBeGreaterThan(1)
+      for (const chunk of chunks) {
+        expect(expectedText.slice(chunk.metadata.startIndex, chunk.metadata.endIndex)).toBe(
+          chunk.text
+        )
+      }
+      expect(chunks.map((chunk) => chunk.text).join('')).toBe(expectedText)
+    })
+
+    it('preserves spaces inside an oversized scalar line', async () => {
+      const content = JSON.stringify('AAAAAA   BBBBBB')
+      const chunker = new JsonYamlChunker({ chunkSize: 2, minCharactersPerChunk: 1 })
+
+      const chunks = await chunker.chunk(content)
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 2)).toBe(true)
+      expect(chunks.map((chunk) => chunk.text).join('')).toBe(content)
+      for (const chunk of chunks) {
+        expect(content.slice(chunk.metadata.startIndex, chunk.metadata.endIndex)).toBe(chunk.text)
+      }
+    })
+
+    it('preserves array item ranges when batch formatting requires bounded splits', async () => {
+      const values = ['x0', 'x1', 'x2', 'x3']
+      const chunker = new JsonYamlChunker({ chunkSize: 2, minCharactersPerChunk: 1 })
+
+      const chunks = await chunker.chunk(JSON.stringify(values))
+
+      expect(new Set(chunks.map((chunk) => JSON.stringify(chunk.metadata)))).toEqual(
+        new Set([
+          JSON.stringify({ startIndex: 0, endIndex: 1 }),
+          JSON.stringify({ startIndex: 2, endIndex: 3 }),
+        ])
+      )
+      for (const chunk of chunks) {
+        expect(chunk.tokenCount).toBeLessThanOrEqual(2)
+      }
+      expect(
+        chunks
+          .filter((chunk) => chunk.metadata.startIndex === 0)
+          .map((chunk) => chunk.text)
+          .join(' ')
+      ).toContain('x0')
+      expect(
+        chunks
+          .filter((chunk) => chunk.metadata.startIndex === 2)
+          .map((chunk) => chunk.text)
+          .join(' ')
+      ).toContain('x2')
+    })
+
+    it('preserves the item range when an oversized array item is split', async () => {
+      const oversizedItem = 'alpha beta gamma delta epsilon zeta eta theta'
+      const chunker = new JsonYamlChunker({ chunkSize: 5, minCharactersPerChunk: 1 })
+
+      const chunks = await chunker.chunk(JSON.stringify([oversizedItem, 'short']))
+
+      expect(chunks.filter((chunk) => chunk.metadata.startIndex === 0).length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 5)).toBe(true)
+      expect(new Set(chunks.map((chunk) => JSON.stringify(chunk.metadata)))).toEqual(
+        new Set([
+          JSON.stringify({ startIndex: 0, endIndex: 0 }),
+          JSON.stringify({ startIndex: 1, endIndex: 1 }),
+        ])
+      )
+    })
+
     it.concurrent('should include startIndex and endIndex in metadata', async () => {
       const chunker = new JsonYamlChunker({ chunkSize: 100 })
       const json = JSON.stringify({ key: 'value' })

apps/sim/lib/chunkers/json-yaml-chunker.ts

@@ -1,7 +1,14 @@
 import { createLogger } from '@sim/logger'
 import * as yaml from 'js-yaml'
+import { ChunkBudget, ChunkLimitExceededError } from '@/lib/chunkers/chunk-budget'
 import type { Chunk, ChunkerOptions } from '@/lib/chunkers/types'
-import { estimateTokens } from '@/lib/chunkers/utils'
+import {
+  estimateTokens,
+  iterateLines,
+  iterateLosslessWordBoundaryChunkSpans,
+  normalizeTokenChunkSize,
+  tokensToChars,
+} from '@/lib/chunkers/utils'
 
 const logger = createLogger('JsonYamlChunker')
 
@@ -9,16 +16,19 @@ type JsonPrimitive = string | number | boolean | null
 type JsonValue = JsonPrimitive | JsonObject | JsonArray
 type JsonObject = { [key: string]: JsonValue }
 type JsonArray = JsonValue[]
+type BoundedChunkMetadataMode = 'text-offsets' | 'preserve-range'
 
 const MAX_DEPTH = 5
 
 export class JsonYamlChunker {
   private chunkSize: number
   private minCharactersPerChunk: number
+  private maxChunks?: number
 
   constructor(options: ChunkerOptions = {}) {
-    this.chunkSize = options.chunkSize ?? 1024
+    this.chunkSize = normalizeTokenChunkSize(options.chunkSize ?? 1024, 'JSON/YAML chunk size')
     this.minCharactersPerChunk = options.minCharactersPerChunk ?? 100
+    this.maxChunks = options.maxChunks
   }
 
   static isStructuredData(content: string): boolean {
@@ -43,25 +53,36 @@ export class JsonYamlChunker {
       } catch {
         data = yaml.load(content) as JsonValue
       }
-      const chunks = this.chunkStructuredData(data, [], 0)
+
+      const chunks: Chunk[] = []
+      this.chunkStructuredData(data, [], 0, chunks, new ChunkBudget(this.maxChunks))
 
       const totalTokens = chunks.reduce((sum, c) => sum + c.tokenCount, 0)
       logger.info(`JSON chunking complete: ${chunks.length} chunks, ${totalTokens} total tokens`)
 
       return chunks
     } catch (error) {
-      logger.info('JSON parsing failed, falling back to text chunking')
-      return this.chunkAsText(content)
+      if (error instanceof ChunkLimitExceededError) throw error
+      logger.info('Structured data chunking failed, falling back to text chunking')
+      return this.chunkAsText(content, new ChunkBudget(this.maxChunks))
     }
   }
 
-  private chunkStructuredData(data: JsonValue, path: string[], depth: number): Chunk[] {
+  private chunkStructuredData(
+    data: JsonValue,
+    path: string[],
+    depth: number,
+    chunks: Chunk[],
+    budget: ChunkBudget
+  ): void {
     if (Array.isArray(data)) {
-      return this.chunkArray(data, path, depth)
+      this.chunkArray(data, path, depth, chunks, budget)
+      return
     }
 
     if (typeof data === 'object' && data !== null) {
-      return this.chunkObject(data as JsonObject, path, depth)
+      this.chunkObject(data as JsonObject, path, depth, chunks, budget)
+      return
     }
 
     const content = JSON.stringify(data, null, 2)
@@ -69,25 +90,29 @@ export class JsonYamlChunker {
     const contentTokens = estimateTokens(content)
 
     if (contentTokens > this.chunkSize) {
-      return this.chunkAsText(contextHeader + content)
+      this.chunkAsText(contextHeader + content, budget, chunks)
+      return
     }
 
     if (content.length < this.minCharactersPerChunk) {
-      return []
+      return
     }
 
     const text = contextHeader + content
-    return [
-      {
-        text,
-        tokenCount: estimateTokens(text),
-        metadata: { startIndex: 0, endIndex: text.length },
-      },
-    ]
+    this.addBoundedChunk(chunks, budget, {
+      text,
+      tokenCount: estimateTokens(text),
+      metadata: { startIndex: 0, endIndex: text.length },
+    })
   }
 
-  private chunkArray(arr: JsonArray, path: string[], depth: number): Chunk[] {
-    const chunks: Chunk[] = []
+  private chunkArray(
+    arr: JsonArray,
+    path: string[],
+    depth: number,
+    chunks: Chunk[],
+    budget: ChunkBudget
+  ): void {
     let currentBatch: JsonValue[] = []
     let currentTokens = 0
 
@@ -100,25 +125,37 @@ export class JsonYamlChunker {
 
       if (itemTokens > this.chunkSize) {
         if (currentBatch.length > 0) {
-          chunks.push(
-            this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1)
+          this.addBoundedChunk(
+            chunks,
+            budget,
+            this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1),
+            'preserve-range'
           )
           currentBatch = []
           currentTokens = 0
         }
 
         if (depth < MAX_DEPTH && typeof item === 'object' && item !== null) {
-          chunks.push(...this.chunkStructuredData(item, [...path, `[${i}]`], depth + 1))
+          this.chunkStructuredData(item, [...path, `[${i}]`], depth + 1, chunks, budget)
         } else {
-          chunks.push({
-            text: contextHeader + itemStr,
-            tokenCount: itemTokens,
-            metadata: { startIndex: i, endIndex: i },
-          })
+          const text = contextHeader + itemStr
+          this.addBoundedChunk(
+            chunks,
+            budget,
+            {
+              text,
+              tokenCount: estimateTokens(text),
+              metadata: { startIndex: i, endIndex: i },
+            },
+            'preserve-range'
+          )
         }
       } else if (currentTokens + itemTokens > this.chunkSize && currentBatch.length > 0) {
-        chunks.push(
-          this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1)
+        this.addBoundedChunk(
+          chunks,
+          budget,
+          this.buildBatchChunk(contextHeader, currentBatch, i - currentBatch.length, i - 1),
+          'preserve-range'
         )
         currentBatch = [item]
         currentTokens = itemTokens
@@ -129,21 +166,27 @@ export class JsonYamlChunker {
     }
 
     if (currentBatch.length > 0) {
-      chunks.push(
+      this.addBoundedChunk(
+        chunks,
+        budget,
         this.buildBatchChunk(
           contextHeader,
           currentBatch,
           arr.length - currentBatch.length,
           arr.length - 1
-        )
+        ),
+        'preserve-range'
       )
     }
-
-    return chunks
   }
 
-  private chunkObject(obj: JsonObject, path: string[], depth: number): Chunk[] {
-    const chunks: Chunk[] = []
+  private chunkObject(
+    obj: JsonObject,
+    path: string[],
+    depth: number,
+    chunks: Chunk[],
+    budget: ChunkBudget
+  ): void {
     const entries = Object.entries(obj)
 
     const fullContent = JSON.stringify(obj, null, 2)
@@ -152,13 +195,12 @@ export class JsonYamlChunker {
     if (fullTokens <= this.chunkSize) {
       const contextHeader = path.length > 0 ? `// ${path.join('.')}\n` : ''
       const text = contextHeader + fullContent
-      return [
-        {
-          text,
-          tokenCount: estimateTokens(text),
-          metadata: { startIndex: 0, endIndex: text.length },
-        },
-      ]
+      this.addBoundedChunk(chunks, budget, {
+        text,
+        tokenCount: estimateTokens(text),
+        metadata: { startIndex: 0, endIndex: text.length },
+      })
+      return
     }
 
     const contextHeader = path.length > 0 ? `// ${path.join('.')}\n` : ''
@@ -172,7 +214,7 @@ export class JsonYamlChunker {
       if (valueTokens > this.chunkSize) {
         if (Object.keys(currentObj).length > 0) {
           const objContent = contextHeader + JSON.stringify(currentObj, null, 2)
-          chunks.push({
+          this.addBoundedChunk(chunks, budget, {
             text: objContent,
             tokenCount: estimateTokens(objContent),
             metadata: { startIndex: 0, endIndex: objContent.length },
@@ -182,20 +224,16 @@ export class JsonYamlChunker {
         }
 
         if (depth < MAX_DEPTH && typeof value === 'object' && value !== null) {
-          chunks.push(...this.chunkStructuredData(value, [...path, key], depth + 1))
+          this.chunkStructuredData(value, [...path, key], depth + 1, chunks, budget)
         } else {
-          chunks.push({
-            text: contextHeader + valueStr,
-            tokenCount: valueTokens,
-            metadata: { startIndex: 0, endIndex: valueStr.length },
-          })
+          this.chunkAsText(contextHeader + valueStr, budget, chunks)
         }
       } else if (
         currentTokens + valueTokens > this.chunkSize &&
         Object.keys(currentObj).length > 0
       ) {
         const objContent = contextHeader + JSON.stringify(currentObj, null, 2)
-        chunks.push({
+        this.addBoundedChunk(chunks, budget, {
           text: objContent,
           tokenCount: estimateTokens(objContent),
           metadata: { startIndex: 0, endIndex: objContent.length },
@@ -210,14 +248,12 @@ export class JsonYamlChunker {
 
     if (Object.keys(currentObj).length > 0) {
       const objContent = contextHeader + JSON.stringify(currentObj, null, 2)
-      chunks.push({
+      this.addBoundedChunk(chunks, budget, {
         text: objContent,
         tokenCount: estimateTokens(objContent),
         metadata: { startIndex: 0, endIndex: objContent.length },
       })
     }
-
-    return chunks
   }
 
   private buildBatchChunk(
@@ -234,18 +270,74 @@ export class JsonYamlChunker {
     }
   }
 
-  private chunkAsText(content: string): Chunk[] {
-    const chunks: Chunk[] = []
-    const lines = content.split('\n')
+  private addBoundedChunk(
+    chunks: Chunk[],
+    budget: ChunkBudget,
+    chunk: Chunk,
+    metadataMode: BoundedChunkMetadataMode = 'text-offsets'
+  ): void {
+    if (chunk.tokenCount <= this.chunkSize) {
+      budget.add(chunks, chunk)
+      return
+    }
+
+    for (const segment of iterateLosslessWordBoundaryChunkSpans(
+      chunk.text,
+      tokensToChars(this.chunkSize)
+    )) {
+      budget.add(chunks, {
+        text: segment.text,
+        tokenCount: estimateTokens(segment.text),
+        metadata:
+          metadataMode === 'preserve-range'
+            ? chunk.metadata
+            : {
+                startIndex: chunk.metadata.startIndex + segment.startIndex,
+                endIndex: chunk.metadata.startIndex + segment.endIndex,
+              },
+      })
+    }
+  }
+
+  private chunkAsText(content: string, budget: ChunkBudget, chunks: Chunk[] = []): Chunk[] {
     let currentChunk = ''
     let currentTokens = 0
     let startIndex = 0
 
-    for (const line of lines) {
+    for (const line of iterateLines(content)) {
       const lineTokens = estimateTokens(line)
 
+      if (lineTokens > this.chunkSize) {
+        if (currentChunk) {
+          budget.add(chunks, {
+            text: currentChunk,
+            tokenCount: currentTokens,
+            metadata: { startIndex, endIndex: startIndex + currentChunk.length },
+          })
+          startIndex += currentChunk.length + 1
+          currentChunk = ''
+          currentTokens = 0
+        }
+        const lineStartIndex = startIndex
+        for (const segment of iterateLosslessWordBoundaryChunkSpans(
+          line,
+          tokensToChars(this.chunkSize)
+        )) {
+          budget.add(chunks, {
+            text: segment.text,
+            tokenCount: estimateTokens(segment.text),
+            metadata: {
+              startIndex: lineStartIndex + segment.startIndex,
+              endIndex: lineStartIndex + segment.endIndex,
+            },
+          })
+        }
+        startIndex += line.length + 1
+        continue
+      }
+
       if (currentTokens + lineTokens > this.chunkSize && currentChunk) {
-        chunks.push({
+        budget.add(chunks, {
           text: currentChunk,
           tokenCount: currentTokens,
           metadata: { startIndex, endIndex: startIndex + currentChunk.length },
@@ -261,7 +353,7 @@ export class JsonYamlChunker {
     }
 
     if (currentChunk && currentChunk.length >= this.minCharactersPerChunk) {
-      chunks.push({
+      budget.add(chunks, {
         text: currentChunk,
         tokenCount: currentTokens,
         metadata: { startIndex, endIndex: startIndex + currentChunk.length },

apps/sim/lib/chunkers/recursive-chunker.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@sim/logger'
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import { MAX_CHUNKING_SEPARATOR_LENGTH, MAX_CHUNKING_SEPARATORS } from '@/lib/chunkers/constants'
 import type { Chunk, RecursiveChunkerOptions } from '@/lib/chunkers/types'
 import {
@@ -6,8 +7,10 @@ import {
   buildChunks,
   cleanText,
   estimateTokens,
+  hasMultipleNonEmptyLiteralParts,
+  iterateLiteralParts,
+  iterateWordBoundaryChunks,
   resolveChunkerOptions,
-  splitAtWordBoundaries,
   tokensToChars,
 } from '@/lib/chunkers/utils'
 
@@ -57,11 +60,13 @@ export class RecursiveChunker {
   private readonly chunkSize: number
   private readonly chunkOverlap: number
   private readonly separators: string[]
+  private readonly maxChunks?: number
 
   constructor(options: RecursiveChunkerOptions = {}) {
     const resolved = resolveChunkerOptions(options)
     this.chunkSize = resolved.chunkSize
     this.chunkOverlap = resolved.chunkOverlap
+    this.maxChunks = options.maxChunks
 
     /**
      * Bounded here as well as at the API boundary: a config persisted before the
@@ -92,11 +97,17 @@ export class RecursiveChunker {
     }
   }
 
-  private splitRecursively(text: string, separatorIndex = 0): string[] {
+  private splitRecursively(
+    text: string,
+    chunks: string[],
+    budget: ChunkBudget,
+    separatorIndex = 0
+  ): void {
     const tokenCount = estimateTokens(text)
 
     if (tokenCount <= this.chunkSize) {
-      return text.trim() ? [text] : []
+      if (text.trim()) budget.add(chunks, text)
+      return
     }
 
     /**
@@ -105,7 +116,6 @@ export class RecursiveChunker {
      */
     let index = separatorIndex
     let separator = ''
-    let parts: string[] = []
 
     while (index < this.separators.length) {
       separator = this.separators[index]
@@ -115,9 +125,7 @@ export class RecursiveChunker {
         break
       }
 
-      parts = text.split(separator).filter((part) => part.trim())
-
-      if (parts.length > 1) {
+      if (hasMultipleNonEmptyLiteralParts(text, separator)) {
         break
       }
 
@@ -126,27 +134,27 @@ export class RecursiveChunker {
 
     if (index >= this.separators.length) {
       const chunkSizeChars = tokensToChars(this.chunkSize)
-      return splitAtWordBoundaries(text, chunkSizeChars)
+      for (const part of iterateWordBoundaryChunks(text, chunkSizeChars)) {
+        budget.add(chunks, part)
+      }
+      return
     }
 
-    const chunks: string[] = []
     let currentChunk = ''
 
-    for (const part of parts) {
+    for (const part of iterateLiteralParts(text, separator)) {
+      if (!part.trim()) continue
       const testChunk = currentChunk + (currentChunk ? separator : '') + part
 
       if (estimateTokens(testChunk) <= this.chunkSize) {
         currentChunk = testChunk
       } else {
         if (currentChunk.trim()) {
-          chunks.push(currentChunk.trim())
+          budget.add(chunks, currentChunk.trim())
         }
 
         if (estimateTokens(part) > this.chunkSize) {
-          const subChunks = this.splitRecursively(part, index + 1)
-          for (const subChunk of subChunks) {
-            chunks.push(subChunk)
-          }
+          this.splitRecursively(part, chunks, budget, index + 1)
           currentChunk = ''
         } else {
           currentChunk = part
@@ -155,10 +163,8 @@ export class RecursiveChunker {
     }
 
     if (currentChunk.trim()) {
-      chunks.push(currentChunk.trim())
+      budget.add(chunks, currentChunk.trim())
     }
-
-    return chunks
   }
 
   async chunk(content: string): Promise<Chunk[]> {
@@ -167,7 +173,8 @@ export class RecursiveChunker {
     }
 
     const cleaned = cleanText(content)
-    let chunks = this.splitRecursively(cleaned)
+    let chunks: string[] = []
+    this.splitRecursively(cleaned, chunks, new ChunkBudget(this.maxChunks))
 
     if (this.chunkOverlap > 0) {
       const overlapChars = tokensToChars(this.chunkOverlap)

apps/sim/lib/chunkers/regex-chunker.test.ts

@@ -256,6 +256,35 @@ describe('RegexChunker', () => {
   })
 
   describe('strictBoundaries mode', () => {
+    it.concurrent('preserves delimiter-only content as one chunk', async () => {
+      const chunker = new RegexChunker({
+        pattern: '---',
+        chunkSize: 1024,
+        strictBoundaries: true,
+      })
+
+      const chunks = await chunker.chunk('------')
+
+      expect(chunks).toHaveLength(1)
+      expect(chunks[0].text).toBe('------')
+    })
+
+    it.concurrent(
+      'preserves the original text when only one meaningful segment remains',
+      async () => {
+        const chunker = new RegexChunker({
+          pattern: '---',
+          chunkSize: 1024,
+          strictBoundaries: true,
+        })
+
+        const chunks = await chunker.chunk('---alpha---')
+
+        expect(chunks).toHaveLength(1)
+        expect(chunks[0].text).toBe('---alpha---')
+      }
+    )
+
     it.concurrent(
       'should produce one chunk per match without merging small adjacent segments',
       async () => {

apps/sim/lib/chunkers/regex-chunker.ts

@@ -1,13 +1,14 @@
 import { createLogger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import type { Chunk, RegexChunkerOptions } from '@/lib/chunkers/types'
 import {
   addOverlap,
   buildChunks,
   cleanText,
   estimateTokens,
+  iterateWordBoundaryChunks,
   resolveChunkerOptions,
-  splitAtWordBoundaries,
   tokensToChars,
 } from '@/lib/chunkers/utils'
 import {
@@ -63,6 +64,7 @@ export class RegexChunker {
   private readonly chunkOverlap: number
   private readonly regex: LinearRegex
   private readonly strictBoundaries: boolean
+  private readonly maxChunks?: number
 
   constructor(options: RegexChunkerOptions) {
     const resolved = resolveChunkerOptions(options)
@@ -70,6 +72,7 @@ export class RegexChunker {
     this.chunkOverlap = resolved.chunkOverlap
     this.regex = this.compilePattern(options.pattern)
     this.strictBoundaries = options.strictBoundaries ?? false
+    this.maxChunks = options.maxChunks
   }
 
   /**
@@ -122,21 +125,41 @@ export class RegexChunker {
 
     if (!this.strictBoundaries && estimateTokens(cleaned) <= this.chunkSize) {
       logger.info('Content fits in single chunk')
-      return buildChunks([cleaned], 0)
+      const texts: string[] = []
+      new ChunkBudget(this.maxChunks).add(texts, cleaned)
+      return buildChunks(texts, 0)
     }
 
-    const segments = this.regex.split(cleaned).filter((s) => s.trim().length > 0)
-
-    if (segments.length <= 1) {
-      if (this.strictBoundaries) {
+    const segments = this.nonEmptySegments(cleaned)
+    if (this.strictBoundaries) {
+      const first = segments.next()
+      const second = segments.next()
+      if (first.done || second.done) {
+        const chunks: string[] = []
+        new ChunkBudget(this.maxChunks).add(chunks, cleaned.trim())
         logger.info('Regex pattern produced no splits in strict mode, returning single chunk')
-        return buildChunks([cleaned.trim()], 0)
+        return buildChunks(chunks, 0)
       }
+
+      const allSegments = this.prependSegments(first.value, second.value, segments)
+      const chunks = this.expandOversizedSegments(allSegments, new ChunkBudget(this.maxChunks))
+      logger.info(`Chunked into ${chunks.length} strict-boundary regex chunks`)
+      return buildChunks(chunks, 0)
+    }
+
+    const first = segments.next()
+    const second = segments.next()
+
+    if (first.done || second.done) {
       logger.warn(
         'Regex pattern did not produce any splits, falling back to word-boundary splitting'
       )
       const chunkSizeChars = tokensToChars(this.chunkSize)
-      let chunks = splitAtWordBoundaries(cleaned, chunkSizeChars)
+      const budget = new ChunkBudget(this.maxChunks)
+      let chunks: string[] = []
+      for (const chunk of iterateWordBoundaryChunks(cleaned, chunkSizeChars)) {
+        budget.add(chunks, chunk)
+      }
       if (this.chunkOverlap > 0) {
         const overlapChars = tokensToChars(this.chunkOverlap)
         chunks = addOverlap(chunks, overlapChars)
@@ -144,13 +167,9 @@ export class RegexChunker {
       return buildChunks(chunks, this.chunkOverlap)
     }
 
-    if (this.strictBoundaries) {
-      const chunks = this.expandOversizedSegments(segments)
-      logger.info(`Chunked into ${chunks.length} strict-boundary regex chunks`)
-      return buildChunks(chunks, 0)
-    }
-
-    const merged = this.mergeSegments(segments)
+    const allSegments = this.prependSegments(first.value, second.value, segments)
+    const budget = new ChunkBudget(this.maxChunks)
+    const merged = this.mergeSegments(allSegments, budget)
 
     let chunks = merged
     if (this.chunkOverlap > 0) {
@@ -162,12 +181,28 @@ export class RegexChunker {
     return buildChunks(chunks, this.chunkOverlap)
   }
 
+  private *nonEmptySegments(content: string): Generator<string> {
+    for (const segment of this.regex.iterateSplits(content)) {
+      if (segment.trim()) yield segment
+    }
+  }
+
+  private *prependSegments(
+    first: string,
+    second: string,
+    rest: Iterable<string>
+  ): Generator<string> {
+    yield first
+    yield second
+    yield* rest
+  }
+
   /**
    * In strict-boundary mode each segment becomes its own chunk. Segments that
    * exceed chunkSize are still split at word boundaries to preserve the token
    * limit invariant; this is a safety floor, not a merge.
    */
-  private expandOversizedSegments(segments: string[]): string[] {
+  private expandOversizedSegments(segments: Iterable<string>, budget: ChunkBudget): string[] {
     const result: string[] = []
     const chunkSizeChars = tokensToChars(this.chunkSize)
 
@@ -176,11 +211,10 @@ export class RegexChunker {
       if (!trimmed) continue
 
       if (estimateTokens(trimmed) <= this.chunkSize) {
-        result.push(trimmed)
+        budget.add(result, trimmed)
       } else {
-        const subChunks = splitAtWordBoundaries(trimmed, chunkSizeChars)
-        for (const sub of subChunks) {
-          if (sub.trim()) result.push(sub)
+        for (const sub of iterateWordBoundaryChunks(trimmed, chunkSizeChars)) {
+          if (sub.trim()) budget.add(result, sub)
         }
       }
     }
@@ -188,7 +222,7 @@ export class RegexChunker {
     return result
   }
 
-  private mergeSegments(segments: string[]): string[] {
+  private mergeSegments(segments: Iterable<string>, budget: ChunkBudget): string[] {
     const chunks: string[] = []
     let current = ''
 
@@ -199,14 +233,13 @@ export class RegexChunker {
         current = test
       } else {
         if (current.trim()) {
-          chunks.push(current.trim())
+          budget.add(chunks, current.trim())
         }
 
         if (estimateTokens(segment) > this.chunkSize) {
           const chunkSizeChars = tokensToChars(this.chunkSize)
-          const subChunks = splitAtWordBoundaries(segment, chunkSizeChars)
-          for (const sub of subChunks) {
-            chunks.push(sub)
+          for (const sub of iterateWordBoundaryChunks(segment, chunkSizeChars)) {
+            budget.add(chunks, sub)
           }
           current = ''
         } else {
@@ -216,7 +249,7 @@ export class RegexChunker {
     }
 
     if (current.trim()) {
-      chunks.push(current.trim())
+      budget.add(chunks, current.trim())
     }
 
     return chunks

apps/sim/lib/chunkers/sentence-chunker.ts

@@ -1,36 +1,45 @@
 import { createLogger } from '@sim/logger'
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import type { Chunk, SentenceChunkerOptions } from '@/lib/chunkers/types'
 import {
   buildChunks,
   cleanText,
   estimateTokens,
+  iterateWordBoundaryChunks,
   resolveChunkerOptions,
-  splitAtWordBoundaries,
   tokensToChars,
 } from '@/lib/chunkers/utils'
 
 const logger = createLogger('SentenceChunker')
 
+const SENTENCE_BOUNDARY_PATTERN =
+  /(?<!\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|St|Rev|Gen|Sgt|Capt|Lt|Col|Maj|No|Fig|Vol|Ch|vs|etc|Inc|Ltd|Corp|Co|approx|dept|est|govt|Ave|Blvd|Rd|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|i\.e|e\.g)\.)(?<![A-Z]\.)(?<!\.\.)(?<!\d\.)(?<=[.!?])\s+/g
+
 /** Never splits mid-sentence unless a single sentence exceeds the limit. */
 export class SentenceChunker {
   private readonly chunkSize: number
   private readonly chunkOverlap: number
   private readonly minSentencesPerChunk: number
+  private readonly maxChunks?: number
 
   constructor(options: SentenceChunkerOptions = {}) {
     const resolved = resolveChunkerOptions(options)
     this.chunkSize = resolved.chunkSize
     this.chunkOverlap = resolved.chunkOverlap
     this.minSentencesPerChunk = options.minSentencesPerChunk ?? 1
+    this.maxChunks = options.maxChunks
   }
 
   /** Splits on sentence boundaries while avoiding abbreviations, decimals, and ellipses. */
-  private splitSentences(text: string): string[] {
-    return text
-      .split(
-        /(?<!\b(?:Mr|Mrs|Ms|Dr|Prof|Sr|Jr|St|Rev|Gen|Sgt|Capt|Lt|Col|Maj|No|Fig|Vol|Ch|vs|etc|Inc|Ltd|Corp|Co|approx|dept|est|govt|Ave|Blvd|Rd|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|i\.e|e\.g)\.)(?<![A-Z]\.)(?<!\.\.)(?<!\d\.)(?<=[.!?])\s+/
-      )
-      .filter((s) => s.trim().length > 0)
+  private *splitSentences(text: string): Generator<string> {
+    let cursor = 0
+    for (const match of text.matchAll(SENTENCE_BOUNDARY_PATTERN)) {
+      const sentence = text.slice(cursor, match.index)
+      if (sentence.trim()) yield sentence
+      cursor = match.index + match[0].length
+    }
+    const tail = text.slice(cursor)
+    if (tail.trim()) yield tail
   }
 
   async chunk(content: string): Promise<Chunk[]> {
@@ -39,34 +48,31 @@ export class SentenceChunker {
     }
 
     const cleaned = cleanText(content)
-    const sentences = this.splitSentences(cleaned)
-
-    if (sentences.length === 0) {
-      return []
-    }
 
     if (estimateTokens(cleaned) <= this.chunkSize) {
       logger.info('Content fits in single chunk')
-      return buildChunks([cleaned], 0)
+      const texts: string[] = []
+      new ChunkBudget(this.maxChunks).add(texts, cleaned)
+      return buildChunks(texts, 0)
     }
 
+    const budget = new ChunkBudget(this.maxChunks)
     const chunkSentenceGroups: string[][] = []
     let currentGroup: string[] = []
     let currentTokens = 0
     const chunkSizeChars = tokensToChars(this.chunkSize)
 
-    for (const sentence of sentences) {
+    for (const sentence of this.splitSentences(cleaned)) {
       const sentenceTokens = estimateTokens(sentence)
 
       if (sentenceTokens > this.chunkSize) {
         if (currentGroup.length > 0) {
-          chunkSentenceGroups.push(currentGroup)
+          budget.add(chunkSentenceGroups, currentGroup)
           currentGroup = []
           currentTokens = 0
         }
-        const parts = splitAtWordBoundaries(sentence, chunkSizeChars)
-        for (const part of parts) {
-          chunkSentenceGroups.push([part])
+        for (const part of iterateWordBoundaryChunks(sentence, chunkSizeChars)) {
+          budget.add(chunkSentenceGroups, [part])
         }
         continue
       }
@@ -75,7 +81,7 @@ export class SentenceChunker {
       const hasMinSentences = currentGroup.length >= this.minSentencesPerChunk
 
       if (wouldExceed && hasMinSentences) {
-        chunkSentenceGroups.push(currentGroup)
+        budget.add(chunkSentenceGroups, currentGroup)
         currentGroup = [sentence]
         currentTokens = sentenceTokens
       } else {
@@ -85,7 +91,7 @@ export class SentenceChunker {
     }
 
     if (currentGroup.length > 0) {
-      chunkSentenceGroups.push(currentGroup)
+      budget.add(chunkSentenceGroups, currentGroup)
     }
 
     const rawChunks = this.applyOverlapFromGroups(chunkSentenceGroups)

apps/sim/lib/chunkers/structured-data-chunker.test.ts

@@ -74,6 +74,30 @@ describe('StructuredDataChunker', () => {
       expect(chunks).toEqual([])
     })
 
+    it.each([
+      0,
+      -1,
+      0.5,
+      Number.NaN,
+      Number.NEGATIVE_INFINITY,
+      Number.POSITIVE_INFINITY,
+      Number.MAX_SAFE_INTEGER,
+    ])('rejects an invalid chunk size before inspecting content: %s', async (chunkSize) => {
+      await expect(StructuredDataChunker.chunkStructuredData('', { chunkSize })).rejects.toThrow(
+        'Structured data chunk size must be a finite number between 1'
+      )
+    })
+
+    it('normalizes a fractional legacy chunk size to its integer token ceiling', async () => {
+      const chunks = await StructuredDataChunker.chunkStructuredData(
+        `value\n${'x'.repeat(1_000)}`,
+        { chunkSize: 100.5 }
+      )
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 100)).toBe(true)
+    })
+
     it.concurrent('should chunk basic CSV data', async () => {
       const csv = `name,age,city
 Alice,30,New York
@@ -128,6 +152,81 @@ Bob,25`
       expect(chunks.length).toBeGreaterThan(1)
     })
 
+    it('splits a maximum-length spreadsheet cell without dropping its content', async () => {
+      const value = `START-${'x'.repeat(32_755)}-END`
+      const chunks = await StructuredDataChunker.chunkStructuredData(`value\n${value}`, {
+        chunkSize: 1024,
+      })
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 1024)).toBe(true)
+      expect(chunks[0].text).toContain('START-')
+      expect(chunks.at(-1)?.text).toContain('-END')
+    })
+
+    it('splits an oversized header without repeating it into every row segment', async () => {
+      const header = `HEADER-${'h '.repeat(2_500)}-END`
+      const row = `ROW-${'r '.repeat(5_000)}-END`
+
+      const chunks = await StructuredDataChunker.chunkStructuredData(`${header}\n${row}`, {
+        chunkSize: 1024,
+      })
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.length).toBeLessThan(20)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 1024)).toBe(true)
+      expect(
+        chunks
+          .filter((chunk) => chunk.metadata.startIndex === 0)
+          .map((chunk) => chunk.text)
+          .join('')
+      ).toBe(header)
+      expect(
+        chunks
+          .filter((chunk) => chunk.metadata.startIndex === 1)
+          .map((chunk) => chunk.text)
+          .join('')
+      ).toBe(row)
+      expect(chunks.every((chunk) => !chunk.text.includes('Headers:'))).toBe(true)
+      expect(chunks.every((chunk) => !chunk.text.includes('rows of data'))).toBe(true)
+    })
+
+    it('preserves spaces when splitting an oversized structured row', async () => {
+      const row = 'AAAAAA BBBBBB'
+      const chunks = await StructuredDataChunker.chunkStructuredData(`value\n${row}`, {
+        chunkSize: 15,
+      })
+      const prefix = 'Headers: value\n-----\n'
+      const suffix = '\n\n[1 rows of data]'
+      const rowSegments = chunks
+        .filter((chunk) => chunk.metadata.startIndex === 1)
+        .map((chunk) => chunk.text.slice(prefix.length, -suffix.length))
+
+      expect(rowSegments.join('')).toBe(row)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 15)).toBe(true)
+    })
+
+    it('does not let the minimum row target exceed the token target', async () => {
+      const row = 'x'.repeat(2_900)
+      const content = ['value', row, row, row, row, row].join('\n')
+
+      const chunks = await StructuredDataChunker.chunkStructuredData(content, { chunkSize: 1024 })
+
+      expect(chunks.length).toBe(5)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 1024)).toBe(true)
+    })
+
+    it('accounts for labels, separators, sheet names, and footers before batching rows', async () => {
+      const content = ['h', ...Array.from({ length: 30 }, () => 'x')].join('\n')
+      const chunks = await StructuredDataChunker.chunkStructuredData(content, {
+        chunkSize: 20,
+        sheetName: 'S',
+      })
+
+      expect(chunks.length).toBeGreaterThan(1)
+      expect(chunks.every((chunk) => chunk.tokenCount <= 20)).toBe(true)
+    })
+
     it.concurrent('should include token count in chunk metadata', async () => {
       const csv = `name,age
 Alice,30

apps/sim/lib/chunkers/structured-data-chunker.ts

@@ -1,5 +1,11 @@
 import { createLogger } from '@sim/logger'
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import type { Chunk, StructuredDataOptions } from '@/lib/chunkers/types'
+import {
+  iterateLines,
+  iterateLosslessWordBoundaryChunkSpans,
+  normalizeTokenChunkSize,
+} from '@/lib/chunkers/utils'
 
 /** Structured data is denser in tokens (~3 chars/token vs ~4 for prose) */
 function estimateStructuredTokens(text: string): number {
@@ -7,6 +13,22 @@ function estimateStructuredTokens(text: string): number {
   return Math.ceil(text.length / 3)
 }
 
+function estimateFormattedChunkTokens(
+  headerLine: string,
+  rowCharacters: number,
+  rowCount: number,
+  sheetName?: string
+): number {
+  let characters = rowCharacters + Math.max(0, rowCount - 1)
+  if (sheetName) characters += 4 + sheetName.length + 6
+  if (DEFAULT_CONFIG.INCLUDE_HEADERS_IN_EACH_CHUNK) {
+    characters += 9 + headerLine.length + 1
+    characters += Math.min(80, headerLine.length) + 1
+  }
+  characters += 3 + String(rowCount).length + 14
+  return Math.ceil(characters / 3)
+}
+
 const logger = createLogger('StructuredDataChunker')
 
 const DEFAULT_CONFIG = {
@@ -21,47 +43,107 @@ export class StructuredDataChunker {
     content: string,
     options: StructuredDataOptions = {}
   ): Promise<Chunk[]> {
-    const chunks: Chunk[] = []
-    const lines = content.split('\n').filter((line) => line.trim())
+    const targetChunkSize = normalizeTokenChunkSize(
+      options.chunkSize ?? DEFAULT_CONFIG.TARGET_CHUNK_SIZE,
+      'Structured data chunk size'
+    )
 
-    if (lines.length === 0) {
+    const chunks: Chunk[] = []
+    const sampleLines: string[] = []
+    for (const line of iterateLines(content)) {
+      if (!line.trim()) continue
+      sampleLines.push(line)
+      if (sampleLines.length === 10) break
+    }
+
+    if (sampleLines.length === 0) {
       return chunks
     }
 
-    const targetChunkSize = options.chunkSize ?? DEFAULT_CONFIG.TARGET_CHUNK_SIZE
-
-    const headerLine = options.headers?.join('\t') || lines[0]
+    const budget = new ChunkBudget(options.maxChunks)
+    const headerLine = options.headers?.join('\t') || sampleLines[0]
     const dataStartIndex = options.headers ? 0 : 1
 
     const estimatedTokensPerRow = StructuredDataChunker.estimateStructuredTokensPerRow(
-      lines.slice(dataStartIndex, Math.min(10, lines.length))
+      sampleLines.slice(dataStartIndex)
     )
     const optimalRowsPerChunk = StructuredDataChunker.calculateOptimalRowsPerChunk(
       estimatedTokensPerRow,
       targetChunkSize
     )
 
-    logger.info(
-      `Structured data chunking: ${lines.length} rows, ~${estimatedTokensPerRow} tokens/row, ${optimalRowsPerChunk} rows/chunk, target: ${targetChunkSize} tokens`
-    )
-
     let currentChunkRows: string[] = []
-    let currentTokenEstimate = 0
-    const headerTokens = estimateStructuredTokens(headerLine)
+    let currentRowsCharacters = 0
     let chunkStartRow = dataStartIndex
+    let oversizedHeaderEmitted = false
 
-    for (let i = dataStartIndex; i < lines.length; i++) {
-      const row = lines[i]
-      const rowTokens = estimateStructuredTokens(row)
+    let lineIndex = 0
+    for (const row of iterateLines(content)) {
+      if (!row.trim()) continue
+      const i = lineIndex
+      lineIndex++
+      if (i < dataStartIndex) continue
 
-      const projectedTokens =
-        currentTokenEstimate +
-        rowTokens +
-        (DEFAULT_CONFIG.INCLUDE_HEADERS_IN_EACH_CHUNK ? headerTokens : 0)
+      const standaloneRow = StructuredDataChunker.formatChunk(headerLine, [row], options.sheetName)
+      if (estimateStructuredTokens(standaloneRow) > targetChunkSize) {
+        if (currentChunkRows.length > 0) {
+          const chunkContent = StructuredDataChunker.formatChunk(
+            headerLine,
+            currentChunkRows,
+            options.sheetName
+          )
+          budget.add(chunks, StructuredDataChunker.createChunk(chunkContent, chunkStartRow, i - 1))
+          currentChunkRows = []
+          currentRowsCharacters = 0
+        }
+        const emptyRowOverhead = estimateStructuredTokens(
+          StructuredDataChunker.formatChunk(headerLine, [''], options.sheetName)
+        )
+        if (emptyRowOverhead >= targetChunkSize) {
+          if (!oversizedHeaderEmitted) {
+            const headerContent = options.sheetName
+              ? `${options.sheetName}\n${headerLine}`
+              : headerLine
+            const headerRow = Math.max(0, dataStartIndex - 1)
+            for (const segment of iterateLosslessWordBoundaryChunkSpans(
+              headerContent,
+              targetChunkSize * 3
+            )) {
+              budget.add(
+                chunks,
+                StructuredDataChunker.createChunk(segment.text, headerRow, headerRow)
+              )
+            }
+            oversizedHeaderEmitted = true
+          }
+          for (const segment of iterateLosslessWordBoundaryChunkSpans(row, targetChunkSize * 3)) {
+            budget.add(chunks, StructuredDataChunker.createChunk(segment.text, i, i))
+          }
+          chunkStartRow = i + 1
+          continue
+        }
+        const rowSegmentChars = Math.max(1, (targetChunkSize - emptyRowOverhead) * 3)
+        for (const segment of iterateLosslessWordBoundaryChunkSpans(row, rowSegmentChars)) {
+          const segmentContent = StructuredDataChunker.formatChunk(
+            headerLine,
+            [segment.text],
+            options.sheetName
+          )
+          budget.add(chunks, StructuredDataChunker.createChunk(segmentContent, i, i))
+        }
+        chunkStartRow = i + 1
+        continue
+      }
+
+      const projectedTokens = estimateFormattedChunkTokens(
+        headerLine,
+        currentRowsCharacters + row.length,
+        currentChunkRows.length + 1,
+        options.sheetName
+      )
 
       const shouldCreateChunk =
-        (projectedTokens > targetChunkSize &&
-          currentChunkRows.length >= DEFAULT_CONFIG.MIN_ROWS_PER_CHUNK) ||
+        (projectedTokens > targetChunkSize && currentChunkRows.length > 0) ||
         currentChunkRows.length >= optimalRowsPerChunk
 
       if (shouldCreateChunk && currentChunkRows.length > 0) {
@@ -70,15 +152,15 @@ export class StructuredDataChunker {
           currentChunkRows,
           options.sheetName
         )
-        chunks.push(StructuredDataChunker.createChunk(chunkContent, chunkStartRow, i - 1))
+        budget.add(chunks, StructuredDataChunker.createChunk(chunkContent, chunkStartRow, i - 1))
 
         currentChunkRows = []
-        currentTokenEstimate = 0
+        currentRowsCharacters = 0
         chunkStartRow = i
       }
 
       currentChunkRows.push(row)
-      currentTokenEstimate += rowTokens
+      currentRowsCharacters += row.length
     }
 
     if (currentChunkRows.length > 0) {
@@ -87,10 +169,15 @@ export class StructuredDataChunker {
         currentChunkRows,
         options.sheetName
       )
-      chunks.push(StructuredDataChunker.createChunk(chunkContent, chunkStartRow, lines.length - 1))
+      budget.add(
+        chunks,
+        StructuredDataChunker.createChunk(chunkContent, chunkStartRow, lineIndex - 1)
+      )
     }
 
-    logger.info(`Created ${chunks.length} chunks from ${lines.length} rows of structured data`)
+    logger.info(
+      `Created ${chunks.length} chunks from ${lineIndex} rows of structured data at ~${estimatedTokensPerRow} tokens/row and ${optimalRowsPerChunk} rows/chunk (target: ${targetChunkSize} tokens)`
+    )
 
     return chunks
   }
@@ -156,7 +243,11 @@ export class StructuredDataChunker {
       }
     }
 
-    const lines = content.split('\n').slice(0, 10)
+    const lines: string[] = []
+    for (const line of iterateLines(content)) {
+      lines.push(line)
+      if (lines.length === 10) break
+    }
     if (lines.length < 2) return false
 
     const delimiters = [',', '\t', '|']

apps/sim/lib/chunkers/text-chunker.ts

@@ -1,17 +1,21 @@
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import type { Chunk, ChunkerOptions } from '@/lib/chunkers/types'
 import {
   addOverlap,
   buildChunks,
   cleanText,
   estimateTokens,
+  hasMultipleNonEmptyLiteralParts,
+  iterateLiteralParts,
+  iterateWordBoundaryChunks,
   resolveChunkerOptions,
-  splitAtWordBoundaries,
   tokensToChars,
 } from '@/lib/chunkers/utils'
 
 export class TextChunker {
   private readonly chunkSize: number
   private readonly chunkOverlap: number
+  private readonly maxChunks?: number
 
   private readonly separators = [
     '\n---\n',
@@ -37,45 +41,51 @@ export class TextChunker {
     const resolved = resolveChunkerOptions(options)
     this.chunkSize = resolved.chunkSize
     this.chunkOverlap = resolved.chunkOverlap
+    this.maxChunks = options.maxChunks
   }
 
-  private splitRecursively(text: string, separatorIndex = 0): string[] {
+  private splitRecursively(
+    text: string,
+    chunks: string[],
+    budget: ChunkBudget,
+    separatorIndex = 0
+  ): void {
     const tokenCount = estimateTokens(text)
 
     if (tokenCount <= this.chunkSize) {
-      return text.trim() ? [text] : []
+      if (text.trim()) budget.add(chunks, text)
+      return
     }
 
     if (separatorIndex >= this.separators.length) {
       const chunkSizeChars = tokensToChars(this.chunkSize)
-      return splitAtWordBoundaries(text, chunkSizeChars)
+      for (const part of iterateWordBoundaryChunks(text, chunkSizeChars)) {
+        budget.add(chunks, part)
+      }
+      return
     }
 
     const separator = this.separators[separatorIndex]
-    const parts = text.split(separator).filter((part) => part.trim())
-
-    if (parts.length <= 1) {
-      return this.splitRecursively(text, separatorIndex + 1)
+    if (!hasMultipleNonEmptyLiteralParts(text, separator)) {
+      this.splitRecursively(text, chunks, budget, separatorIndex + 1)
+      return
     }
 
-    const chunks: string[] = []
     let currentChunk = ''
 
-    for (const part of parts) {
+    for (const part of iterateLiteralParts(text, separator)) {
+      if (!part.trim()) continue
       const testChunk = currentChunk + (currentChunk ? separator : '') + part
 
       if (estimateTokens(testChunk) <= this.chunkSize) {
         currentChunk = testChunk
       } else {
         if (currentChunk.trim()) {
-          chunks.push(currentChunk.trim())
+          budget.add(chunks, currentChunk.trim())
         }
 
         if (estimateTokens(part) > this.chunkSize) {
-          const subChunks = this.splitRecursively(part, separatorIndex + 1)
-          for (const subChunk of subChunks) {
-            chunks.push(subChunk)
-          }
+          this.splitRecursively(part, chunks, budget, separatorIndex + 1)
           currentChunk = ''
         } else {
           currentChunk = part
@@ -84,10 +94,8 @@ export class TextChunker {
     }
 
     if (currentChunk.trim()) {
-      chunks.push(currentChunk.trim())
+      budget.add(chunks, currentChunk.trim())
     }
-
-    return chunks
   }
 
   async chunk(text: string): Promise<Chunk[]> {
@@ -96,7 +104,8 @@ export class TextChunker {
     }
 
     const cleaned = cleanText(text)
-    let chunks = this.splitRecursively(cleaned)
+    let chunks: string[] = []
+    this.splitRecursively(cleaned, chunks, new ChunkBudget(this.maxChunks))
 
     if (this.chunkOverlap > 0) {
       const overlapChars = tokensToChars(this.chunkOverlap)

apps/sim/lib/chunkers/token-chunker.ts

@@ -1,11 +1,12 @@
 import { createLogger } from '@sim/logger'
+import { ChunkBudget } from '@/lib/chunkers/chunk-budget'
 import type { Chunk, ChunkerOptions } from '@/lib/chunkers/types'
 import {
   buildChunks,
   cleanText,
   estimateTokens,
+  iterateWordBoundaryChunks,
   resolveChunkerOptions,
-  splitAtWordBoundaries,
   tokensToChars,
 } from '@/lib/chunkers/utils'
 
@@ -15,12 +16,14 @@ export class TokenChunker {
   private readonly chunkSize: number
   private readonly chunkOverlap: number
   private readonly minCharactersPerChunk: number
+  private readonly maxChunks?: number
 
   constructor(options: ChunkerOptions = {}) {
     const resolved = resolveChunkerOptions(options)
     this.chunkSize = resolved.chunkSize
     this.chunkOverlap = resolved.chunkOverlap
     this.minCharactersPerChunk = resolved.minCharactersPerChunk
+    this.maxChunks = options.maxChunks
   }
 
   async chunk(content: string): Promise<Chunk[]> {
@@ -29,24 +32,38 @@ export class TokenChunker {
     }
 
     const cleaned = cleanText(content)
+    const budget = new ChunkBudget(this.maxChunks)
 
     if (estimateTokens(cleaned) <= this.chunkSize) {
       logger.info('Content fits in single chunk')
-      return buildChunks([cleaned], 0)
+      const texts: string[] = []
+      budget.add(texts, cleaned)
+      return buildChunks(texts, 0)
     }
 
     const chunkSizeChars = tokensToChars(this.chunkSize)
     const overlapChars = tokensToChars(this.chunkOverlap)
     const stepChars = this.chunkOverlap > 0 ? chunkSizeChars - overlapChars : undefined
 
-    const rawChunks = splitAtWordBoundaries(cleaned, chunkSizeChars, stepChars)
+    const rawChunks: string[] = []
+    const filteredChunks: string[] = []
+    const filteredBudget = new ChunkBudget(this.maxChunks)
+    let rawChunkCount = 0
+    for (const chunk of iterateWordBoundaryChunks(cleaned, chunkSizeChars, stepChars)) {
+      rawChunkCount++
+      if (this.maxChunks === undefined || rawChunks.length <= this.maxChunks) {
+        rawChunks.push(chunk)
+      }
+      if (chunk.length >= this.minCharactersPerChunk) {
+        filteredBudget.add(filteredChunks, chunk)
+      }
+    }
 
-    const filtered =
-      rawChunks.length > 1
-        ? rawChunks.filter((c) => c.length >= this.minCharactersPerChunk)
-        : rawChunks
-
-    const chunks = filtered.length > 0 ? filtered : rawChunks
+    let chunks = filteredChunks
+    if (rawChunkCount <= 1 || filteredChunks.length === 0) {
+      chunks = []
+      for (const chunk of rawChunks) budget.add(chunks, chunk)
+    }
 
     logger.info(`Chunked into ${chunks.length} token-based chunks`)
     return buildChunks(chunks, this.chunkOverlap)

apps/sim/lib/chunkers/types.ts

@@ -7,6 +7,7 @@ export interface ChunkerOptions {
   chunkSize?: number
   chunkOverlap?: number
   minCharactersPerChunk?: number
+  maxChunks?: number
 }
 
 export interface Chunk {

apps/sim/lib/chunkers/utils.test.ts

@@ -8,6 +8,7 @@ import {
   buildChunks,
   cleanText,
   estimateTokens,
+  iterateLosslessWordBoundaryChunkSpans,
   resolveChunkerOptions,
   splitAtWordBoundaries,
   tokensToChars,
@@ -108,6 +109,15 @@ describe('addOverlap', () => {
 })
 
 describe('splitAtWordBoundaries', () => {
+  it.each([0, -1, 0.5, Number.NaN, Number.POSITIVE_INFINITY])(
+    'rejects a non-positive, fractional, or non-finite chunk size: %s',
+    (chunkSize) => {
+      expect(() => splitAtWordBoundaries('bounded text', chunkSize)).toThrow(
+        'Word-boundary chunk size must be a positive safe integer'
+      )
+    }
+  )
+
   it('returns single element for short text', () => {
     const result = splitAtWordBoundaries('short text', 100)
     expect(result).toHaveLength(1)
@@ -158,6 +168,28 @@ describe('splitAtWordBoundaries', () => {
   })
 })
 
+describe('iterateLosslessWordBoundaryChunkSpans', () => {
+  it('preserves every source character while preferring word boundaries', () => {
+    const text = '  alpha   beta gamma  '
+    const spans = Array.from(iterateLosslessWordBoundaryChunkSpans(text, 8))
+
+    expect(spans.map((span) => span.text).join('')).toBe(text)
+    expect(spans.every((span) => span.text.length <= 8)).toBe(true)
+    for (const span of spans) {
+      expect(text.slice(span.startIndex, span.endIndex)).toBe(span.text)
+    }
+  })
+
+  it.each([0, -1, 0.5, Number.NaN, Number.POSITIVE_INFINITY])(
+    'rejects an invalid chunk size: %s',
+    (chunkSize) => {
+      expect(() =>
+        Array.from(iterateLosslessWordBoundaryChunkSpans('bounded text', chunkSize))
+      ).toThrow('Lossless word-boundary chunk size must be a positive safe integer')
+    }
+  )
+})
+
 describe('buildChunks', () => {
   it('creates Chunk objects with text, tokenCount, and metadata', () => {
     const texts = ['hello world', 'foo bar']
@@ -204,6 +236,16 @@ describe('resolveChunkerOptions', () => {
     expect(result.chunkOverlap).toBe(50)
   })
 
+  it('normalizes a fractional legacy token size before deriving character windows', () => {
+    expect(resolveChunkerOptions({ chunkSize: 100.5 }).chunkSize).toBe(100)
+  })
+
+  it('rejects a token size whose character window would exceed safe integer bounds', () => {
+    expect(() => resolveChunkerOptions({ chunkSize: Number.MAX_SAFE_INTEGER })).toThrow(
+      'Chunk size must be a finite number between 1'
+    )
+  })
+
   it('respects provided values when within limits', () => {
     const result = resolveChunkerOptions({
       chunkSize: 500,

apps/sim/lib/chunkers/utils.ts

@@ -1,5 +1,7 @@
 import type { Chunk } from '@/lib/chunkers/types'
 
+const MAX_TOKEN_CHUNK_SIZE = Math.floor(Number.MAX_SAFE_INTEGER / 4)
+
 /** 1 token ≈ 4 characters for English text */
 export function estimateTokens(text: string): number {
   if (!text?.trim()) return 0
@@ -10,6 +12,19 @@ export function tokensToChars(tokens: number): number {
   return tokens * 4
 }
 
+export function assertPositiveSafeInteger(value: number, name: string): void {
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`${name} must be a positive safe integer`)
+  }
+}
+
+export function normalizeTokenChunkSize(value: number, name: string): number {
+  if (!Number.isFinite(value) || value < 1 || value > MAX_TOKEN_CHUNK_SIZE) {
+    throw new Error(`${name} must be a finite number between 1 and ${MAX_TOKEN_CHUNK_SIZE}`)
+  }
+  return Math.floor(value)
+}
+
 export function cleanText(text: string): string {
   return text
     .replace(/\r\n/g, '\n')
@@ -60,7 +75,42 @@ export function splitAtWordBoundaries(
   chunkSizeChars: number,
   stepChars?: number
 ): string[] {
-  const parts: string[] = []
+  return Array.from(iterateWordBoundaryChunks(text, chunkSizeChars, stepChars))
+}
+
+export interface WordBoundaryChunkSpan {
+  text: string
+  startIndex: number
+  endIndex: number
+}
+
+/** Iterates bounded word-aware source slices without trimming or skipping characters. */
+export function* iterateLosslessWordBoundaryChunkSpans(
+  text: string,
+  chunkSizeChars: number
+): Generator<WordBoundaryChunkSpan> {
+  assertPositiveSafeInteger(chunkSizeChars, 'Lossless word-boundary chunk size')
+
+  let startIndex = 0
+  while (startIndex < text.length) {
+    let endIndex = Math.min(startIndex + chunkSizeChars, text.length)
+    if (endIndex < text.length) {
+      const lastSpace = text.lastIndexOf(' ', endIndex - 1)
+      if (lastSpace > startIndex) endIndex = lastSpace + 1
+    }
+    yield { text: text.slice(startIndex, endIndex), startIndex, endIndex }
+    startIndex = endIndex
+  }
+}
+
+/** Iterates trimmed word-boundary chunks while preserving their source offsets. */
+export function* iterateWordBoundaryChunkSpans(
+  text: string,
+  chunkSizeChars: number,
+  stepChars?: number
+): Generator<WordBoundaryChunkSpan> {
+  assertPositiveSafeInteger(chunkSizeChars, 'Word-boundary chunk size')
+
   let pos = 0
 
   while (pos < text.length) {
@@ -68,30 +118,79 @@ export function splitAtWordBoundaries(
 
     if (end < text.length) {
       const lastSpace = text.lastIndexOf(' ', end)
-      if (lastSpace > pos) {
-        end = lastSpace
-      }
+      if (lastSpace > pos) end = lastSpace
     }
 
-    const part = text.slice(pos, end).trim()
-    if (part) {
-      parts.push(part)
+    const rawPart = text.slice(pos, end)
+    const startIndex = pos + (rawPart.length - rawPart.trimStart().length)
+    const endIndex = end - (rawPart.length - rawPart.trimEnd().length)
+    if (endIndex > startIndex) {
+      yield { text: text.slice(startIndex, endIndex), startIndex, endIndex }
     }
 
     if (stepChars !== undefined) {
-      // Sliding window: advance by step for predictable overlap
       const nextPos = pos + Math.max(1, stepChars)
       if (nextPos >= text.length) break
       pos = nextPos
     } else {
-      // Non-overlapping: advance from end of extracted content
       if (end >= text.length) break
       pos = end
     }
     while (pos < text.length && text[pos] === ' ') pos++
   }
+}
 
-  return parts
+export function* iterateWordBoundaryChunks(
+  text: string,
+  chunkSizeChars: number,
+  stepChars?: number
+): Generator<string> {
+  for (const span of iterateWordBoundaryChunkSpans(text, chunkSizeChars, stepChars)) {
+    yield span.text
+  }
+}
+
+/** Iterates literal-separated parts while preserving String.split's raw part values. */
+export function* iterateLiteralParts(text: string, separator: string): Generator<string> {
+  if (!separator) {
+    yield text
+    return
+  }
+
+  let cursor = 0
+  while (cursor <= text.length) {
+    const next = text.indexOf(separator, cursor)
+    if (next === -1) {
+      yield text.slice(cursor)
+      return
+    }
+    yield text.slice(cursor, next)
+    cursor = next + separator.length
+  }
+}
+
+export function hasMultipleNonEmptyLiteralParts(text: string, separator: string): boolean {
+  let nonEmptyParts = 0
+  for (const part of iterateLiteralParts(text, separator)) {
+    if (!part.trim()) continue
+    nonEmptyParts++
+    if (nonEmptyParts === 2) return true
+  }
+  return false
+}
+
+/** Iterates lines without allocating an array proportional to line count. */
+export function* iterateLines(text: string): Generator<string> {
+  let cursor = 0
+  while (cursor <= text.length) {
+    const next = text.indexOf('\n', cursor)
+    if (next === -1) {
+      yield text.slice(cursor)
+      return
+    }
+    yield text.slice(cursor, next)
+    cursor = next + 1
+  }
 }
 
 export function buildChunks(texts: string[], overlapTokens: number): Chunk[] {
@@ -133,7 +232,7 @@ export function resolveChunkerOptions(options: {
   chunkOverlap?: number
   minCharactersPerChunk?: number
 }): { chunkSize: number; chunkOverlap: number; minCharactersPerChunk: number } {
-  const chunkSize = options.chunkSize ?? 1024
+  const chunkSize = normalizeTokenChunkSize(options.chunkSize ?? 1024, 'Chunk size')
   const maxOverlap = Math.floor(chunkSize * 0.5)
   return {
     chunkSize,

apps/sim/lib/core/security/linear-regex.differential.test.ts

@@ -157,6 +157,23 @@ describe('differential: split parity with the built-in engine', () => {
   )
 })
 
+describe('differential: lazy split parity with eager split', () => {
+  it.each(SPLIT_PATTERNS.filter((pattern) => !isKnownDivergence(pattern)))(
+    'iterates %s identically to the eager linear split across every document',
+    (pattern) => {
+      const compiled = compile(pattern)
+      if (!compiled) return
+
+      for (const doc of DOCUMENTS) {
+        expect(
+          normalize(Array.from(compiled.iterateSplits(doc))),
+          `pattern ${pattern} on ${JSON.stringify(doc)}`
+        ).toEqual(normalize(compiled.split(doc)))
+      }
+    }
+  )
+})
+
 describe('differential: test/find parity with the built-in engine', () => {
   /** Grep-style patterns, where `test` and the match index are what matter. */
   const MATCH_PATTERNS = [

apps/sim/lib/core/security/linear-regex.test.ts

@@ -131,6 +131,17 @@ describe('compileLookaroundSplit', () => {
     )
   })
 
+  it('keeps split independent of its object receiver', () => {
+    const doc = '# One\nalpha\n# Two\nbeta'
+    const { split } = compileLookaroundSplit('(?=#\\s)')!
+
+    expect(split(doc)).toEqual(doc.split(/(?=#\s)/g).filter(Boolean))
+    expect([doc, doc].map(split)).toEqual([
+      doc.split(/(?=#\s)/g).filter(Boolean),
+      doc.split(/(?=#\s)/g).filter(Boolean),
+    ])
+  })
+
   it('splits after each delimiter for (?<=X)', () => {
     const doc = '<s>one</s><s>two</s><s>three</s>'
     expect(compileLookaroundSplit('(?<=</s>)')?.split(doc)).toEqual([

apps/sim/lib/core/security/linear-regex.ts

@@ -40,6 +40,7 @@ export interface LinearRegex {
    * omitted — RE2 drops it, and every caller here discards empties anyway.
    */
   split(text: string): string[]
+  iterateSplits(text: string): IterableIterator<string>
 }
 
 const METACHARACTERS = /[.*+?^${}()|[\]\\]/
@@ -284,28 +285,26 @@ export function compileLookaroundSplit(
     return { start: matcher.start('mid'), end: matcher.end('mid') }
   }
 
+  const iterateSplits = function* (text: string): Generator<string> {
+    let cursor = 0
+    let searchFrom = 0
+    while (searchFrom <= text.length) {
+      const span = delimiterAt(text, searchFrom)
+      if (!span) break
+      searchFrom = span.end > span.start ? span.end : span.start + 1
+      if (span.start < cursor || span.start >= text.length) continue
+      if (span.start === cursor && span.end === cursor) continue
+      yield text.slice(cursor, span.start)
+      cursor = span.end
+    }
+    if (cursor < text.length || cursor === 0) yield text.slice(cursor)
+  }
+
   return {
     test: (text) => compiled.matcher(text).find(),
     find: (text) => delimiterAt(text, 0)?.start ?? -1,
-    split: (text) => {
-      const segments: string[] = []
-      let cursor = 0
-      let searchFrom = 0
-      while (searchFrom <= text.length) {
-        const span = delimiterAt(text, searchFrom)
-        if (!span) break
-        // Always advance, so a zero-width delimiter cannot spin.
-        searchFrom = span.end > span.start ? span.end : span.start + 1
-        // Skip a boundary behind the cursor, or one that would only emit an
-        // empty leading or trailing segment.
-        if (span.start < cursor || span.start >= text.length) continue
-        if (span.start === cursor && span.end === cursor) continue
-        segments.push(text.slice(cursor, span.start))
-        cursor = span.end
-      }
-      segments.push(text.slice(cursor))
-      return segments
-    },
+    iterateSplits,
+    split: (text) => Array.from(iterateSplits(text)),
   }
 }
 
@@ -329,8 +328,15 @@ export function literalRegex(pattern: string, options: LinearRegexOptions = {}):
       const match = scanner.exec(text)
       return match ? match.index : -1
     },
-    // Built lazily: no caller splits on a literal, so the second compile is
-    // only paid if one ever does.
+    iterateSplits: function* (text) {
+      const splitter = new RegExp(source, `g${caseFlag}`)
+      let cursor = 0
+      for (const match of text.matchAll(splitter)) {
+        yield text.slice(cursor, match.index)
+        cursor = match.index + match[0].length
+      }
+      if (cursor < text.length || cursor === 0) yield text.slice(cursor)
+    },
     split: (text) => text.split(new RegExp(source, `g${caseFlag}`)),
   }
 }
@@ -353,12 +359,25 @@ export function compileLinearRegex(
       translateToRe2(pattern),
       options.ignoreCase ? RE2JS.CASE_INSENSITIVE : 0
     )
+    const iterateSplits = function* (text: string): Generator<string> {
+      const matcher = compiled.matcher(text)
+      let cursor = 0
+      while (matcher.find()) {
+        const start = matcher.start()
+        const end = matcher.end()
+        yield text.slice(cursor, start)
+        cursor = end
+      }
+      if (cursor < text.length || cursor === 0) yield text.slice(cursor)
+    }
+
     return {
       test: (text) => compiled.matcher(text).find(),
       find: (text) => {
         const matcher = compiled.matcher(text)
         return matcher.find() ? matcher.start() : -1
       },
+      iterateSplits,
       split: (text) => compiled.split(text),
     }
   } catch {

apps/sim/lib/core/security/redaction.test.ts

@@ -4,6 +4,7 @@ import {
   isSensitiveKey,
   REDACTED_MARKER,
   redactApiKeys,
+  redactExactSensitiveValues,
   redactSensitiveValues,
   sanitizeEventData,
   sanitizeForLogging,
@@ -178,6 +179,298 @@ describe('redactSensitiveValues', () => {
     expect(result).not.toContain('key123456')
   })
 
+  it.concurrent('should redact form and percent-encoded OAuth credentials', () => {
+    const input =
+      'refresh_token=secret-one&client_secret=secret-two&oauth_token=secret-three&client_password=secret-four oauth_token%3Dsecret-five%26client_password%3Dsecret-six%26scope%3Dx'
+    const result = redactSensitiveValues(input)
+
+    expect(result).not.toContain('secret-one')
+    expect(result).not.toContain('secret-two')
+    expect(result).not.toContain('secret-three')
+    expect(result).not.toContain('secret-four')
+    expect(result).not.toContain('secret-five')
+    expect(result).not.toContain('secret-six')
+    expect(result).toContain('refresh_token=[REDACTED]')
+    expect(result).toContain('oauth_token%3D[REDACTED]')
+    expect(result).toContain('scope%3Dx')
+  })
+
+  it.concurrent('redacts authorization schemes without exposing the credential suffix', () => {
+    expect(redactSensitiveValues('authorization=Bearer <token> scope=openid')).toBe(
+      'authorization=[REDACTED] scope=openid'
+    )
+    expect(redactSensitiveValues('authorization=Basic dXNlcjpwYXNz')).toBe(
+      'authorization=[REDACTED]'
+    )
+    expect(redactSensitiveValues('authorization=Basic YQ== scope=openid')).toBe(
+      'authorization=[REDACTED] scope=openid'
+    )
+    expect(redactSensitiveValues('authorization=Bearer <padded-token>= scope=openid')).toBe(
+      'authorization=[REDACTED] scope=openid'
+    )
+  })
+
+  it.concurrent('redacts parameterized authorization schemes through the field delimiter', () => {
+    expect(
+      redactSensitiveValues(
+        'authorization=Digest username="user", realm="realm", response="digest-secret"&scope=openid'
+      )
+    ).toBe('authorization=[REDACTED]&scope=openid')
+    expect(
+      redactSensitiveValues(
+        'proxyAuthorization=OAuth realm="Example", oauth_signature="oauth-secret"&scope=openid'
+      )
+    ).toBe('proxyAuthorization=[REDACTED]&scope=openid')
+  })
+
+  it.concurrent('redacts parameterized authorization header values through the line ending', () => {
+    expect(
+      redactSensitiveValues(
+        'Authorization: Digest username="user", realm="realm", response="digest-secret"\nstatus: 401'
+      )
+    ).toBe('Authorization: [REDACTED]\nstatus: 401')
+    expect(
+      redactSensitiveValues(
+        'Proxy-Authorization: AWS4-HMAC-SHA256 Credential=key/path, SignedHeaders=host, Signature=signature-secret'
+      )
+    ).toBe('Proxy-Authorization: [REDACTED]')
+  })
+
+  it.concurrent(
+    'redacts malformed single-token authorization headers through the line ending',
+    () => {
+      expect(redactSensitiveValues('Authorization: Bearer prefix:secret-tail')).toBe(
+        'Authorization: Bearer [REDACTED]'
+      )
+      expect(redactSensitiveValues('Proxy-Authorization: Basic prefix:secret-tail')).toBe(
+        'Proxy-Authorization: Basic [REDACTED]'
+      )
+    }
+  )
+
+  it.concurrent('does not consume the next line when an authorization header is empty', () => {
+    expect(redactSensitiveValues('Authorization: Bearer\nstatus: 401')).toBe(
+      'Authorization: [REDACTED]\nstatus: 401'
+    )
+    expect(redactSensitiveValues('Authorization:\r\nstatus: 401')).toBe(
+      'Authorization: [REDACTED]\r\nstatus: 401'
+    )
+  })
+
+  it.concurrent('redacts encoded form credentials using delimiters at the matching depth', () => {
+    let encoded = new URLSearchParams({
+      access_token: 'prefix&secret-tail',
+      scope: 'openid',
+    }).toString()
+
+    for (let layer = 0; layer <= 3; layer++) {
+      const result = redactSensitiveValues(encoded)
+      expect(result).not.toContain('prefix')
+      expect(result).not.toContain('secret-tail')
+      expect(result).toContain('scope')
+      encoded = encodeURIComponent(encoded)
+    }
+  })
+
+  it.concurrent('uses the canonical sensitive-key policy for form fields', () => {
+    const keys = ['authorization', 'auth', 'bearer', 'private_key', 'passphrase']
+    const input = keys
+      .flatMap((key, index) => [`${key}=plain-secret-${index}`, `${key}%3Dencoded-secret-${index}`])
+      .join('&')
+    const result = redactSensitiveValues(input)
+
+    for (let index = 0; index < keys.length; index++) {
+      expect(result).not.toContain(`plain-secret-${index}`)
+      expect(result).not.toContain(`encoded-secret-${index}`)
+    }
+  })
+
+  it.concurrent('classifies percent-encoded form keys after bounded decoding', () => {
+    const input =
+      'api%5Fkey=secret-one&private%2Dkey=secret-two&cl%69ent%5Fsecret=secret-three&display%5Fname=Alice&next%50age%54oken=cursor'
+
+    expect(redactSensitiveValues(input)).toBe(
+      'api%5Fkey=[REDACTED]&private%2Dkey=[REDACTED]&cl%69ent%5Fsecret=[REDACTED]&display%5Fname=Alice&next%50age%54oken=cursor'
+    )
+  })
+
+  it.concurrent('redacts repeatedly encoded authorization keys through their delimiter', () => {
+    const input =
+      'proxy%252Dauthorization%253DDigest username="user", response="secret"%2526scope%253Dopenid'
+
+    expect(redactSensitiveValues(input)).toBe(
+      'proxy%252Dauthorization%253D[REDACTED]%2526scope%253Dopenid'
+    )
+  })
+
+  it.concurrent('fails closed for malformed or excessively encoded form keys', () => {
+    for (const malformedKey of [
+      'api%ZZkey',
+      'api%G_key',
+      'private%2/key',
+      'authorization%?%?',
+      'authorization%?%3Fscope',
+      'api%?%26scope',
+      'authorization%FF%3Fscope',
+      'api%C0%26scope',
+      'api%',
+    ]) {
+      expect(redactSensitiveValues(`${malformedKey}=secret-one&scope=openid`)).toBe(
+        `${malformedKey}=[REDACTED]&scope=openid`
+      )
+    }
+    expect(redactSensitiveValues('api%2525255Fkey=secret-two&scope=openid')).toBe(
+      'api%2525255Fkey=[REDACTED]&scope=openid'
+    )
+    expect(redactSensitiveValues('authorization%E2%82%AC%3Fscope=public&next=ok')).toBe(
+      'authorization%E2%82%AC%3Fscope=public&next=ok'
+    )
+  })
+
+  it.concurrent('preserves safe form keys containing an encoded literal percent', () => {
+    expect(redactSensitiveValues('discount%25value=10&scope=openid')).toBe(
+      'discount%25value=10&scope=openid'
+    )
+  })
+
+  it.concurrent('redacts sensitive raw fields nested inside a non-sensitive URL value', () => {
+    const input = 'redirect_uri=https://example.com/callback?access_token=raw-secret&scope=openid'
+
+    expect(redactSensitiveValues(input)).toBe(
+      'redirect_uri=https://example.com/callback?access_token=[REDACTED]&scope=openid'
+    )
+  })
+
+  it.concurrent('redacts a raw sensitive value containing an encoded ampersand in full', () => {
+    const input = 'access_token=prefix%26secret-tail&scope=openid'
+
+    expect(redactSensitiveValues(input)).toBe('access_token=[REDACTED]&scope=openid')
+  })
+
+  it.concurrent(
+    'redacts sensitive percent-encoded fields nested inside a non-sensitive URL value',
+    () => {
+      const input =
+        'redirect_uri%3Dhttps%3A%2F%2Fexample.com%2Fcallback%3Faccess_token%3Dencoded-secret%26scope%3Dopenid'
+
+      expect(redactSensitiveValues(input)).toBe(
+        'redirect_uri%3Dhttps%3A%2F%2Fexample.com%2Fcallback%3Faccess_token%3D[REDACTED]%26scope%3Dopenid'
+      )
+    }
+  )
+
+  it.concurrent('redacts authorization fields after repeatedly encoded query separators', () => {
+    const cases = [
+      [
+        '%3Fauthorization%3DBearer%20nested-secret%26scope%3Dopenid',
+        '%3Fauthorization%3D[REDACTED]%26scope%3Dopenid',
+      ],
+      [
+        '%253Fauthorization%253DBearer%2520nested-secret%2526scope%253Dopenid',
+        '%253Fauthorization%253D[REDACTED]%2526scope%253Dopenid',
+      ],
+      [
+        '%25253Fauthorization%25253DBearer%252520nested-secret%252526scope%25253Dopenid',
+        '%25253Fauthorization%25253D[REDACTED]%252526scope%25253Dopenid',
+      ],
+    ]
+
+    for (const [input, expected] of cases) {
+      expect(redactSensitiveValues(input)).toBe(expected)
+    }
+  })
+
+  it.concurrent('redacts authorization fields inside repeatedly encoded URL values', () => {
+    let input =
+      'redirect_uri=https%3A%2F%2Fexample.com%2Fcallback%3Fauthorization%3DBearer%20nested-secret%26scope%3Dopenid'
+
+    for (let depth = 0; depth < 2; depth++) {
+      const result = redactSensitiveValues(input)
+      expect(result).not.toContain('nested-secret')
+      expect(result).toContain('authorization')
+      expect(result).toContain('scope')
+      input = encodeURIComponent(input)
+    }
+  })
+
+  it.concurrent('redacts an encoded nested authorization field after a long safe prefix', () => {
+    const prefix = 'x'.repeat(400_000)
+    const field = '%3Fauthorization%3DBearer%20nested-secret%26scope%3Dopenid'
+
+    expect(redactSensitiveValues(prefix + field)).toBe(
+      `${prefix}%3Fauthorization%3D[REDACTED]%26scope%3Dopenid`
+    )
+  })
+
+  it.concurrent('does not end a sensitive value at an encoded query separator', () => {
+    expect(redactSensitiveValues('access_token=prefix%3Fscope%3Dsecret-tail&next=ok')).toBe(
+      'access_token=[REDACTED]&next=ok'
+    )
+  })
+
+  it.concurrent('preserves safe fields after an encoded query separator', () => {
+    const input = 'redirect_uri=https%3A%2F%2Fexample.com%2F%3Fpage%3Dhome%26scope%3Dopenid'
+    expect(redactSensitiveValues(input)).toBe(input)
+  })
+
+  it.concurrent('preserves non-secret pagination tokens in form-encoded strings', () => {
+    const input =
+      'nextPageToken=page-one nextPageToken%3Dpage-two nextpagetoken=page-three NEXTPAGETOKEN%3Dpage-four'
+
+    expect(redactSensitiveValues(input)).toBe(input)
+  })
+
+  it.concurrent('should redact exact secrets echoed in free-form text', () => {
+    const result = redactExactSensitiveValues(
+      'provider echoed s3cr%2Ft, s3cr%2ft, space+secret, and plain s3cr/t',
+      ['s3cr/t', 'space secret']
+    )
+
+    expect(result).not.toContain('s3cr/t')
+    expect(result).not.toContain('s3cr%2Ft')
+    expect(result).not.toContain('s3cr%2ft')
+    expect(result).not.toContain('space+secret')
+  })
+
+  it.concurrent('redacts overlapping secrets longest-first', () => {
+    const result = redactExactSensitiveValues('provider echoed abcSECRET', ['abc', 'abcSECRET'])
+
+    expect(result).toBe('provider echoed [REDACTED]')
+    expect(result).not.toContain('SECRET')
+  })
+
+  it.concurrent('redacts mixed-case percent escapes', () => {
+    const result = redactExactSensitiveValues('provider echoed %2f%3A', ['/:'])
+
+    expect(result).toBe('provider echoed [REDACTED]')
+  })
+
+  it.concurrent('redacts exact secrets through bounded repeated encoding', () => {
+    const secret = 'path/secret'
+    const encoded = encodeURIComponent(encodeURIComponent(encodeURIComponent(secret)))
+
+    expect(redactExactSensitiveValues(`provider echoed ${encoded}`, [secret])).toBe(
+      'provider echoed [REDACTED]'
+    )
+  })
+
+  it.concurrent(
+    'redacts exact secrets containing raw form delimiters before parsing fields',
+    () => {
+      const secret = 'prefix&secret-tail'
+
+      expect(redactExactSensitiveValues(`echo access_token=${secret}`, [secret])).toBe(
+        'echo access_token=[REDACTED]'
+      )
+    }
+  )
+
+  it.concurrent('redacts exact secrets containing whitespace before generic auth parsing', () => {
+    const secret = 'password with spaces'
+
+    expect(redactExactSensitiveValues(`Basic ${secret}`, [secret])).toBe('Basic [REDACTED]')
+  })
+
   it.concurrent('should not modify safe strings', () => {
     const input = 'This is a normal string with no secrets'
     const result = redactSensitiveValues(input)

apps/sim/lib/core/security/redaction.ts

@@ -7,7 +7,7 @@ import { filterUserFileForDisplay, isUserFile } from '@/lib/core/utils/user-file
 export const REDACTED_MARKER = '[REDACTED]'
 export const TRUNCATED_MARKER = '[TRUNCATED]'
 
-const BYPASS_REDACTION_KEYS = new Set(['nextPageToken'])
+const BYPASS_REDACTION_KEYS = new Set(['nextpagetoken'])
 
 /** Keys that contain large binary/encoded data that should be truncated in logs */
 const LARGE_DATA_KEYS = new Set(['base64'])
@@ -28,6 +28,7 @@ const SENSITIVE_KEY_PATTERNS: RegExp[] = [
   /^.*api[_-]?key$/i,
   /^passphrase$/i,
   /^authorization$/i,
+  /^proxy[_-]?authorization$/i,
   /^bearer$/i,
   /^private$/i,
   /^auth$/i,
@@ -41,6 +42,17 @@ const SENSITIVE_VALUE_PATTERNS: Array<{
   pattern: RegExp
   replacement: string
 }> = [
+  // Single-token authorization headers retain their scheme for diagnostics.
+  {
+    pattern: /\b((?:proxy[_-]?)?authorization[ \t]*:[ \t]*(?:Bearer|Basic)[ \t]+)[^\r\n]*/gi,
+    replacement: `$1${REDACTED_MARKER}`,
+  },
+  // Parameterized and unknown authorization headers fail closed through the line ending.
+  {
+    pattern:
+      /\b((?:proxy[_-]?)?authorization[ \t]*:)(?![ \t]*(?:Bearer|Basic)[ \t])[ \t]*[^\r\n]*/gi,
+    replacement: `$1 ${REDACTED_MARKER}`,
+  },
   // Bearer tokens
   {
     pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi,
@@ -73,14 +85,313 @@ const SENSITIVE_VALUE_PATTERNS: Array<{
   },
 ]
 
+const FORM_FIELD_MARKER_PATTERN = /=|%(?:25)*3D/i
+const ENCODED_FORM_KEY_COMPONENT_PATTERN = /%[0-9A-F]{2}/i
+const FORM_WHITESPACE_PATTERN = /\s/u
+const AUTHORIZATION_FORM_KEYS = new Set(['authorization', 'proxyauthorization'])
+const SINGLE_TOKEN_AUTHORIZATION_PATTERN = /^(?:Bearer|Basic)\s+\S+/i
+const MAX_EXACT_SECRET_ENCODING_LAYERS = 3
+
+interface ActiveSensitiveFormField {
+  start: number
+  encodingDepth: number
+  whitespaceBoundaryAfter?: number
+}
+
+interface NormalizedFormKey {
+  value: string
+  complete: boolean
+}
+
+interface FormKeySpan {
+  endIndex: number
+  malformed: boolean
+}
+
+type EncodedFormMarkerKind = 'delimiter' | 'field' | 'query'
+
+interface EncodedFormMarker {
+  kind: EncodedFormMarkerKind
+  text: string
+}
+
+type FormFieldPrefixKind =
+  | 'start'
+  | 'delimiter'
+  | 'encoded-delimiter'
+  | 'encoded-query'
+  | 'whitespace'
+  | 'other'
+
+interface FormFieldToken {
+  kind: 'field'
+  index: number
+  endIndex: number
+  key: string
+  fieldMarker: string
+  prefixKind: FormFieldPrefixKind
+  prefixMarker?: string
+}
+
+interface FormDelimiterToken {
+  kind: 'delimiter'
+  index: number
+  delimiter: string
+}
+
+type FormToken = FormFieldToken | FormDelimiterToken
+
 export function isSensitiveKey(key: string): boolean {
-  if (BYPASS_REDACTION_KEYS.has(key)) {
-    return false
-  }
   const lowerKey = key.toLowerCase()
+  if (BYPASS_REDACTION_KEYS.has(lowerKey)) return false
   return SENSITIVE_KEY_PATTERNS.some((pattern) => pattern.test(lowerKey))
 }
 
+function getFormEncodingDepth(marker: string): number {
+  return marker.length === 1 ? 0 : (marker.length - 1) / 2
+}
+
+function normalizeFormKey(key: string): string {
+  return key.toLowerCase().replaceAll('_', '').replaceAll('-', '')
+}
+
+function decodeFormKey(key: string): NormalizedFormKey {
+  let value = key
+  let decodedLayer = false
+  for (let layer = 0; layer < MAX_EXACT_SECRET_ENCODING_LAYERS; layer++) {
+    if (!ENCODED_FORM_KEY_COMPONENT_PATTERN.test(value)) {
+      return { value, complete: decodedLayer || !value.includes('%') }
+    }
+    try {
+      value = decodeURIComponent(value)
+      decodedLayer = true
+    } catch {
+      return { value, complete: false }
+    }
+  }
+  return { value, complete: !ENCODED_FORM_KEY_COMPONENT_PATTERN.test(value) }
+}
+
+function isAuthorizationFormKey(key: string): boolean {
+  return AUTHORIZATION_FORM_KEYS.has(normalizeFormKey(key))
+}
+
+function isRawFormKeyCharacter(character: string): boolean {
+  const code = character.charCodeAt(0)
+  return (
+    (code >= 48 && code <= 57) ||
+    (code >= 65 && code <= 90) ||
+    (code >= 97 && code <= 122) ||
+    character === '_' ||
+    character === '-'
+  )
+}
+
+function readEncodedFormMarker(value: string, index: number): EncodedFormMarker | undefined {
+  if (value[index] !== '%') return undefined
+
+  let cursor = index + 1
+  while (value.slice(cursor, cursor + 2).toLowerCase() === '25') cursor += 2
+
+  const code = value.slice(cursor, cursor + 2).toLowerCase()
+  const kind: EncodedFormMarkerKind | undefined =
+    code === '26' ? 'delimiter' : code === '3d' ? 'field' : code === '3f' ? 'query' : undefined
+  if (!kind) return undefined
+  return { kind, text: value.slice(index, cursor + 2) }
+}
+
+function readFormKeySpan(value: string, start: number): FormKeySpan {
+  let cursor = start
+  let malformed = false
+  while (cursor < value.length) {
+    if (isRawFormKeyCharacter(value[cursor])) {
+      cursor++
+      continue
+    }
+    if (value[cursor] !== '%') break
+
+    const encodedMarker = readEncodedFormMarker(value, cursor)
+    if (encodedMarker) {
+      if (encodedMarker.kind === 'field') break
+      if (!malformed && !decodeFormKey(value.slice(start, cursor)).complete) malformed = true
+      if (!malformed) break
+      cursor += encodedMarker.text.length
+      continue
+    }
+
+    const componentStart = cursor
+    cursor++
+    for (let offset = 0; offset < 2 && cursor < value.length; offset++) {
+      const character = value[cursor]
+      if (
+        character === '%' ||
+        character === '&' ||
+        character === '=' ||
+        FORM_WHITESPACE_PATTERN.test(character)
+      ) {
+        break
+      }
+      cursor++
+    }
+    if (!ENCODED_FORM_KEY_COMPONENT_PATTERN.test(value.slice(componentStart, cursor))) {
+      malformed = true
+    }
+  }
+  return { endIndex: cursor, malformed }
+}
+
+/** Iterates form fields and delimiters without regex backtracking over provider-controlled input. */
+function* iterateFormTokens(value: string): Generator<FormToken> {
+  let index = 0
+
+  while (index < value.length) {
+    const encodedPrefix = readEncodedFormMarker(value, index)
+    let prefixKind: FormFieldPrefixKind | undefined
+    let keyStart = index
+    let prefixMarker: string | undefined
+
+    if (encodedPrefix?.kind === 'delimiter' || encodedPrefix?.kind === 'query') {
+      prefixKind = encodedPrefix.kind === 'delimiter' ? 'encoded-delimiter' : 'encoded-query'
+      prefixMarker = encodedPrefix.text
+      keyStart += encodedPrefix.text.length
+    } else if (value[index] === '&') {
+      prefixKind = 'delimiter'
+      keyStart++
+    } else if (FORM_WHITESPACE_PATTERN.test(value[index])) {
+      prefixKind = 'whitespace'
+      while (keyStart < value.length && FORM_WHITESPACE_PATTERN.test(value[keyStart])) keyStart++
+    } else if (!isRawFormKeyCharacter(value[index]) && value[index] !== '%') {
+      prefixKind = 'other'
+      keyStart++
+    } else if (index === 0) {
+      prefixKind = 'start'
+    }
+
+    if (prefixKind) {
+      const { endIndex: keyEnd } = readFormKeySpan(value, keyStart)
+      const encodedFieldMarker = readEncodedFormMarker(value, keyEnd)
+      const fieldMarker =
+        value[keyEnd] === '='
+          ? '='
+          : encodedFieldMarker?.kind === 'field'
+            ? encodedFieldMarker.text
+            : undefined
+
+      if (keyEnd > keyStart && fieldMarker) {
+        const endIndex = keyEnd + fieldMarker.length
+        yield {
+          kind: 'field',
+          index,
+          endIndex,
+          key: value.slice(keyStart, keyEnd),
+          fieldMarker,
+          prefixKind,
+          prefixMarker,
+        }
+        index = endIndex
+        continue
+      }
+
+      if (encodedPrefix?.kind === 'delimiter') {
+        yield { kind: 'delimiter', index, delimiter: encodedPrefix.text }
+        index += encodedPrefix.text.length
+        continue
+      }
+      if (value[index] === '&') {
+        yield { kind: 'delimiter', index, delimiter: '&' }
+        index++
+        continue
+      }
+
+      index = Math.max(index + 1, keyEnd)
+      continue
+    }
+
+    if (isRawFormKeyCharacter(value[index])) {
+      do index++
+      while (index < value.length && isRawFormKeyCharacter(value[index]))
+      continue
+    }
+    index++
+  }
+}
+
+function findWhitespaceRunStart(value: string, end: number): number {
+  let start = end
+  while (start > 0 && /\s/u.test(value[start - 1])) start--
+  return start
+}
+
+function redactSensitiveFormFields(value: string): string {
+  if (!FORM_FIELD_MARKER_PATTERN.test(value)) return value
+
+  let result = ''
+  let cursor = 0
+  let activeField: ActiveSensitiveFormField | undefined
+
+  const closeActiveField = (end: number) => {
+    if (!activeField) return
+    if (end > activeField.start) {
+      result += `${value.slice(cursor, activeField.start)}${REDACTED_MARKER}`
+      cursor = end
+    }
+    activeField = undefined
+  }
+
+  for (const token of iterateFormTokens(value)) {
+    if (token.kind === 'field') {
+      if (activeField && token.prefixKind !== 'start') {
+        const boundaryIndex =
+          token.prefixKind === 'whitespace'
+            ? findWhitespaceRunStart(value, token.index)
+            : token.index
+        const closesActiveField =
+          token.prefixKind === 'delimiter' ||
+          (token.prefixKind === 'encoded-delimiter' &&
+            token.prefixMarker !== undefined &&
+            getFormEncodingDepth(token.prefixMarker) === activeField.encodingDepth) ||
+          (token.prefixKind === 'whitespace' &&
+            activeField.whitespaceBoundaryAfter !== undefined &&
+            boundaryIndex >= activeField.whitespaceBoundaryAfter)
+        if (closesActiveField) closeActiveField(boundaryIndex)
+      }
+
+      const normalizedKey = decodeFormKey(token.key)
+      if (!activeField && (!normalizedKey.complete || isSensitiveKey(normalizedKey.value))) {
+        const start = token.endIndex
+        const authorization = normalizedKey.complete && isAuthorizationFormKey(normalizedKey.value)
+        const singleTokenAuthorization = authorization
+          ? value.slice(start).match(SINGLE_TOKEN_AUTHORIZATION_PATTERN)?.[0]
+          : undefined
+        activeField = {
+          start,
+          encodingDepth: getFormEncodingDepth(token.fieldMarker),
+          whitespaceBoundaryAfter: authorization
+            ? singleTokenAuthorization === undefined
+              ? undefined
+              : start + singleTokenAuthorization.length
+            : normalizedKey.complete
+              ? start
+              : undefined,
+        }
+      }
+      continue
+    }
+
+    if (!activeField) continue
+
+    if (
+      token.delimiter === '&' ||
+      getFormEncodingDepth(token.delimiter) === activeField.encodingDepth
+    ) {
+      closeActiveField(token.index)
+    }
+  }
+
+  closeActiveField(value.length)
+  return cursor === 0 ? value : result + value.slice(cursor)
+}
+
 /**
  * Redacts sensitive patterns from a string value
  * @param value - The string to redact
@@ -91,13 +402,49 @@ export function redactSensitiveValues(value: string): string {
     return value
   }
 
-  let result = value
+  let result = redactSensitiveFormFields(value)
   for (const { pattern, replacement } of SENSITIVE_VALUE_PATTERNS) {
     result = result.replace(pattern, replacement)
   }
   return result
 }
 
+/**
+ * Redacts known secret values in all literal and URL-encoded forms before the
+ * generic pattern pass. Exact replacement must run first because a credential
+ * can itself contain form delimiters that would otherwise split it and leave a
+ * suffix visible before the exact matcher sees the original value.
+ */
+export function redactKnownSensitiveValues(value: string, secrets: string[]): string {
+  let result = value
+  const orderedSecrets = [...new Set(secrets.filter(Boolean))].sort(
+    (left, right) => right.length - left.length
+  )
+  for (const secret of orderedSecrets) {
+    result = result.replaceAll(secret, REDACTED_MARKER)
+    const encodedVariants = new Set<string>()
+    let uriEncoded = secret
+    let formEncoded = secret
+    for (let layer = 0; layer < MAX_EXACT_SECRET_ENCODING_LAYERS; layer++) {
+      uriEncoded = encodeURIComponent(uriEncoded)
+      formEncoded = new URLSearchParams({ value: formEncoded }).toString().slice('value='.length)
+      encodedVariants.add(uriEncoded)
+      encodedVariants.add(formEncoded)
+    }
+    for (const encoded of encodedVariants) {
+      if (encoded !== secret) {
+        const escaped = encoded.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+        result = result.replace(new RegExp(escaped, 'gi'), REDACTED_MARKER)
+      }
+    }
+  }
+  return result
+}
+
+export function redactExactSensitiveValues(value: string, secrets: string[]): string {
+  return redactSensitiveValues(redactKnownSensitiveValues(value, secrets))
+}
+
 export function isLargeDataKey(key: string): boolean {
   return LARGE_DATA_KEYS.has(key)
 }

apps/sim/lib/core/utils/stream-limits.test.ts

@@ -185,6 +185,65 @@ describe('stream limits', () => {
     ).rejects.toBeInstanceOf(PayloadSizeLimitError)
   })
 
+  it('does not let the text convenience reader bypass the bodyless fail-closed rule', async () => {
+    const text = vi.fn(async () => 'small but not independently bounded')
+
+    await expect(
+      readResponseTextWithLimit(
+        { body: null, text },
+        { maxBytes: 100, label: 'unknown text response' }
+      )
+    ).rejects.toBeInstanceOf(PayloadSizeLimitError)
+    expect(text).not.toHaveBeenCalled()
+  })
+
+  it('rejects an undeclared bodyless response even without a fallback materializer', async () => {
+    await expect(
+      readResponseToBufferWithLimit(
+        { body: null },
+        { maxBytes: 100, label: 'unknown empty response' }
+      )
+    ).rejects.toBeInstanceOf(PayloadSizeLimitError)
+  })
+
+  it('allows a bodyless fallback only with a trusted length or explicit declaration', async () => {
+    const declared = await readResponseTextWithLimit(
+      { headers: headers('5'), body: null, text: async () => 'hello' },
+      { maxBytes: 5, label: 'declared response' }
+    )
+    const explicitlyBounded = await readResponseTextWithLimit(
+      { body: null, text: async () => 'hello' },
+      { maxBytes: 5, label: 'trusted fallback', allowNoBodyFallback: true }
+    )
+
+    expect(declared).toBe('hello')
+    expect(explicitlyBounded).toBe('hello')
+  })
+
+  it.each([204, 205, 304])('accepts a semantically bodyless HTTP %i response', async (status) => {
+    const text = vi.fn(async () => 'must not be materialized')
+
+    const result = await readResponseTextWithLimit(
+      { status, headers: headers('1000'), body: null, text },
+      { maxBytes: 100, label: 'bodyless response' }
+    )
+
+    expect(result).toBe('')
+    expect(text).not.toHaveBeenCalled()
+  })
+
+  it('accepts a semantically bodyless HEAD response', async () => {
+    const text = vi.fn(async () => 'must not be materialized')
+
+    const result = await readResponseTextWithLimit(
+      { status: 200, headers: headers('1000'), body: null, text },
+      { maxBytes: 100, label: 'HEAD response', requestMethod: 'HEAD' }
+    )
+
+    expect(result).toBe('')
+    expect(text).not.toHaveBeenCalled()
+  })
+
   it('cancels when the abort signal is already aborted', async () => {
     const controller = new AbortController()
     controller.abort(new Error('stop'))

apps/sim/lib/core/utils/stream-limits.ts

@@ -295,10 +295,12 @@ export interface ReadResponseWithLimitOptions extends ReadStreamWithLimitOptions
   headers?: { get(name: string): string | null }
   preferTextFallback?: boolean
   allowNoBodyFallback?: boolean
+  requestMethod?: string
 }
 
 export async function readResponseToBufferWithLimit(
   response: {
+    status?: number
     headers?: { get(name: string): string | null }
     body?: ReadableStream<Uint8Array> | null
     arrayBuffer?: () => Promise<ArrayBuffer>
@@ -306,6 +308,16 @@ export async function readResponseToBufferWithLimit(
   },
   options: ReadResponseWithLimitOptions
 ): Promise<Buffer> {
+  const isSemanticallyBodyless =
+    response.status === 204 ||
+    response.status === 205 ||
+    response.status === 304 ||
+    options.requestMethod?.toUpperCase() === 'HEAD'
+  if (isSemanticallyBodyless) {
+    await response.body?.cancel().catch(() => {})
+    return Buffer.alloc(0)
+  }
+
   const contentLength = getContentLength(response.headers ?? options.headers)
   try {
     if (contentLength !== null) {
@@ -317,13 +329,7 @@ export async function readResponseToBufferWithLimit(
     }
     throw error
   }
-  if (
-    !options.allowNoBodyFallback &&
-    !options.preferTextFallback &&
-    !response.body &&
-    contentLength === null &&
-    (response.arrayBuffer || response.text)
-  ) {
+  if (!options.allowNoBodyFallback && !response.body && contentLength === null) {
     throw new PayloadSizeLimitError({
       label: options.label,
       maxBytes: options.maxBytes,
@@ -357,6 +363,7 @@ export async function readResponseToBufferWithLimit(
 
 export async function readResponseTextWithLimit(
   response: {
+    status?: number
     headers?: { get(name: string): string | null }
     body?: ReadableStream<Uint8Array> | null
     arrayBuffer?: () => Promise<ArrayBuffer>

apps/sim/lib/embeddings/client.test.ts

@@ -4,13 +4,19 @@
 import { resetEnvMock, setEnv } from '@sim/testing'
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import {
+  clampEmbeddingConcurrency,
   EMBEDDING_MAX_RETRIES,
   EmbeddingAPIError,
+  EmbeddingOutputLimitError,
+  EmbeddingQuotaExhaustedError,
   embed,
   embedKnowledgeForDeployment,
   embedOpenRouter,
+  isEmbeddingQuotaExhaustion,
   isTransientEmbeddingError,
+  MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES,
 } from '@/lib/embeddings/client'
+import { resetEmbeddingQuotaCircuitsForTesting } from '@/lib/embeddings/quota-circuit'
 
 const { mockGetBYOKKey } = vi.hoisted(() => ({
   mockGetBYOKKey: vi.fn(),
@@ -30,24 +36,55 @@ vi.mock('@/lib/api-key/byok', () => ({
 const originalFetch = global.fetch
 
 function jsonResponse(body: unknown, status = 200, responseHeaders?: HeadersInit): Response {
-  return {
-    ok: status >= 200 && status < 300,
+  return new Response(JSON.stringify(body), {
     status,
     statusText: String(status),
-    // A real Response always carries these; the failure path reads them for rate-limit signals.
     headers: new Headers(responseHeaders),
-    json: async () => body,
-    text: async () => JSON.stringify(body),
-  } as Response
+  })
 }
 
-function openAIBody(vectors: number[][], totalTokens = 5) {
+function rawJsonResponse(body: string, status = 200): Response {
+  return new Response(body, {
+    status,
+    statusText: String(status),
+    headers: new Headers({ 'content-type': 'application/json' }),
+  })
+}
+
+function sizedVector(values: number[], dimensions: number): number[] {
+  return [...values, ...Array(Math.max(0, dimensions - values.length)).fill(0)].slice(0, dimensions)
+}
+
+function openAIBody(vectors: number[][], totalTokens = 5, dimensions: number | null = 1536) {
   return {
-    data: vectors.map((embedding) => ({ embedding })),
+    data: vectors.map((embedding) => ({
+      embedding: dimensions === null ? embedding : sizedVector(embedding, dimensions),
+    })),
     usage: { total_tokens: totalTokens },
   }
 }
 
+function oversizedChunkedSuccessResponse(): Response {
+  const chunkBytes = 1024 * 1024
+  const chunk = new Uint8Array(chunkBytes).fill(0x20)
+  const chunkCount = Math.floor(MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES / chunkBytes) + 1
+  let emitted = 0
+
+  return new Response(
+    new ReadableStream<Uint8Array>({
+      pull(controller) {
+        if (emitted >= chunkCount) {
+          controller.close()
+          return
+        }
+        controller.enqueue(chunk)
+        emitted++
+      },
+    }),
+    { status: 200 }
+  )
+}
+
 let fetchMock: ReturnType<typeof vi.fn>
 
 beforeEach(() => {
@@ -68,6 +105,7 @@ beforeEach(() => {
 })
 
 afterEach(() => {
+  resetEmbeddingQuotaCircuitsForTesting()
   global.fetch = originalFetch
   vi.useRealTimers()
   vi.restoreAllMocks()
@@ -90,7 +128,8 @@ describe('embed', () => {
       input: ['hello'],
       model: 'text-embedding-3-small',
     })
-    expect(result.embeddings).toEqual([[1, 2, 3]])
+    expect(result.embeddings[0].slice(0, 3)).toEqual([1, 2, 3])
+    expect(result.embeddings[0]).toHaveLength(1536)
     expect(result.totalTokens).toBe(4)
     expect(result.dimensions).toBe(1536)
     expect(result.pricingId).toBe('text-embedding-3-small')
@@ -104,7 +143,9 @@ describe('embed', () => {
       const body = JSON.parse((init as RequestInit).body as string)
       const count = body.requests.length
       // Each vector encodes its global input index so ordering is verifiable.
-      const embeddings = Array.from({ length: count }, (_, i) => ({ values: [cursor + i] }))
+      const embeddings = Array.from({ length: count }, (_, i) => ({
+        values: sizedVector([cursor + i], 3072),
+      }))
       cursor += count
       return jsonResponse({ embeddings })
     })
@@ -126,7 +167,9 @@ describe('embed', () => {
   })
 
   it('estimates tokens when the provider omits usage', async () => {
-    fetchMock.mockResolvedValue(jsonResponse({ embeddings: [{ values: [1, 2] }] }))
+    fetchMock.mockResolvedValue(
+      jsonResponse({ embeddings: [{ values: sizedVector([1, 2], 3072) }] })
+    )
 
     const result = await embed(['some text to embed'], {
       model: 'gemini-embedding-001',
@@ -139,7 +182,7 @@ describe('embed', () => {
   it("bills Gemini on its reported token count rather than tiktoken's guess", async () => {
     fetchMock.mockResolvedValue(
       jsonResponse({
-        embeddings: [{ values: [1, 2] }],
+        embeddings: [{ values: sizedVector([1, 2], 3072) }],
         usageMetadata: { promptTokenCount: 4321 },
       })
     )
@@ -174,11 +217,40 @@ describe('embed', () => {
     expect(result.embeddings).toHaveLength(40)
   })
 
+  it('splits max-dimension batches to the successful-response byte budget and preserves order', async () => {
+    fetchMock.mockImplementation(async (_url, init) => {
+      const body = JSON.parse((init as RequestInit).body as string)
+      const inputs = body.input as string[]
+      return jsonResponse(
+        openAIBody(
+          inputs.map((input) => [Number(input.slice(1))]),
+          inputs.length,
+          3072
+        )
+      )
+    })
+    const inputs = Array.from({ length: 400 }, (_, index) => `i${index}`)
+
+    const result = await embed(inputs, {
+      model: 'text-embedding-3-large',
+      apiKey: 'sk-test',
+    })
+
+    expect(
+      fetchMock.mock.calls.map(
+        ([, init]) => JSON.parse((init as RequestInit).body as string).input.length
+      )
+    ).toEqual([169, 169, 62])
+    expect(result.embeddings.map(([value]) => value)).toEqual(
+      inputs.map((input) => Number(input.slice(1)))
+    )
+  })
+
   it('keeps a long Cohere input whole rather than cutting it to the batch budget', async () => {
     fetchMock.mockImplementation(async (_url, init) => {
       const body = JSON.parse((init as RequestInit).body as string)
       return jsonResponse({
-        embeddings: { float: body.texts.map(() => [1]) },
+        embeddings: { float: body.texts.map(() => sizedVector([1], 1536)) },
         meta: { billed_units: { input_tokens: 1 } },
       })
     })
@@ -196,7 +268,7 @@ describe('embed', () => {
   })
 
   it('forwards a supported dimension reduction and reports it back', async () => {
-    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]])))
+    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]], 5, 1024)))
 
     const result = await embed(['hello'], {
       model: 'text-embedding-3-large',
@@ -231,7 +303,7 @@ describe('embed', () => {
   it('omits the dimension field for a model without Matryoshka support', async () => {
     fetchMock.mockResolvedValue(
       jsonResponse({
-        data: [{ embedding: [1, 2], index: 0 }],
+        data: [{ embedding: sizedVector([1, 2], 1024), index: 0 }],
         usage: { total_tokens: 5 },
       })
     )
@@ -258,15 +330,132 @@ describe('embed', () => {
   })
 
   it('surfaces a non-retryable provider error with its status', async () => {
-    fetchMock.mockResolvedValue(jsonResponse({ error: 'bad key' }, 401))
+    const echoedSecret = 'sk-provider-echoed-secret'
+    fetchMock.mockResolvedValue(jsonResponse({ error: `bad key: ${echoedSecret}` }, 401))
 
-    await expect(
-      embed(['hello'], { model: 'text-embedding-3-small', apiKey: 'sk-bad' })
-    ).rejects.toThrow(/Embedding API failed: 401/)
+    const error = await embed(['hello'], {
+      model: 'text-embedding-3-small',
+      apiKey: 'sk-bad',
+    }).catch((caught) => caught)
+
+    expect(error).toBeInstanceOf(Error)
+    expect((error as Error).message).toMatch(/Embedding API failed: 401/)
+    expect((error as Error).message).not.toContain(echoedSecret)
     // 401 is not retryable, so exactly one attempt is made.
     expect(fetchMock).toHaveBeenCalledTimes(1)
   })
 
+  it('rejects an oversized chunked success response before JSON materialization', async () => {
+    fetchMock.mockResolvedValue(oversizedChunkedSuccessResponse())
+
+    await expect(
+      embed(['hello'], { model: 'text-embedding-3-small', apiKey: 'sk-test' })
+    ).rejects.toMatchObject({
+      name: 'PayloadSizeLimitError',
+      label: 'Embedding API success response',
+      maxBytes: MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES,
+    })
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it.each([
+    {
+      name: 'the wrong number of vectors',
+      inputs: ['alpha', 'beta'],
+      body: openAIBody([[1]], 2),
+      message: 'returned 1 embeddings for 2 inputs',
+    },
+    {
+      name: 'an empty vector',
+      inputs: ['alpha'],
+      body: openAIBody([[]], 1, null),
+      message: 'vector 0 is empty or not an array',
+    },
+    {
+      name: 'a vector with the wrong catalog dimension',
+      inputs: ['alpha'],
+      body: openAIBody([[1, 2]], 1, null),
+      message: 'vector 0 has 2 unexpected dimensions; expected 1536',
+    },
+    {
+      name: 'a vector with a nonnumeric coordinate',
+      inputs: ['alpha'],
+      body: { data: [{ embedding: [1, 'invalid'] }], usage: { total_tokens: 1 } },
+      message: 'vector 0 contains a non-numeric or non-finite coordinate',
+    },
+    {
+      name: 'an unparseable vector envelope',
+      inputs: ['alpha'],
+      body: { data: {}, usage: { total_tokens: 1 } },
+      message: 'the vector payload could not be parsed',
+    },
+  ])('rejects a valid-JSON success body containing $name', async ({ inputs, body, message }) => {
+    fetchMock.mockResolvedValue(jsonResponse(body))
+
+    await expect(
+      embed(inputs, { model: 'text-embedding-3-small', apiKey: 'sk-test' })
+    ).rejects.toMatchObject({
+      name: 'EmbeddingResponseValidationError',
+      message: expect.stringContaining(message),
+    })
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('rejects a valid-JSON success body containing a non-finite coordinate', async () => {
+    fetchMock.mockResolvedValue(
+      rawJsonResponse('{"data":[{"embedding":[1e999]}],"usage":{"total_tokens":1}}')
+    )
+
+    await expect(
+      embed(['alpha'], { model: 'text-embedding-3-small', apiKey: 'sk-test' })
+    ).rejects.toMatchObject({
+      name: 'EmbeddingResponseValidationError',
+      message: expect.stringContaining('non-numeric or non-finite coordinate'),
+    })
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('rejects one malformed concurrent batch after admitting all independent batches', async () => {
+    const inputs = Array.from({ length: 3 }, (_, index) => `i${index} ${'word '.repeat(5000)}`)
+    fetchMock.mockImplementation(async (_url, init) => {
+      const body = JSON.parse((init as RequestInit).body as string)
+      const input = (body.input as string[])[0]
+      return input.startsWith('i1')
+        ? jsonResponse({ data: [], usage: { total_tokens: 1 } })
+        : jsonResponse(openAIBody([[Number(input[1])]], 1))
+    })
+
+    await expect(
+      embed(inputs, { model: 'text-embedding-3-small', apiKey: 'sk-test' })
+    ).rejects.toMatchObject({
+      name: 'EmbeddingResponseValidationError',
+      message: expect.stringContaining('returned 0 embeddings for 1 inputs'),
+    })
+    expect(fetchMock).toHaveBeenCalledTimes(3)
+  })
+
+  it('preserves input order when concurrent batches complete out of order', async () => {
+    const inputs = Array.from({ length: 3 }, (_, index) => `i${index} ${'word '.repeat(5000)}`)
+    const responders = new Map<string, (response: Response) => void>()
+    fetchMock.mockImplementation(
+      async (_url, init) =>
+        new Promise<Response>((resolve) => {
+          const body = JSON.parse((init as RequestInit).body as string)
+          const input = (body.input as string[])[0]
+          responders.set(input.slice(0, 2), resolve)
+        })
+    )
+
+    const pending = embed(inputs, { model: 'text-embedding-3-small', apiKey: 'sk-test' })
+    await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(3))
+    for (const index of [2, 1, 0]) {
+      responders.get(`i${index}`)?.(jsonResponse(openAIBody([[index]], 1)))
+    }
+    const result = await pending
+
+    expect(result.embeddings.map(([value]) => value)).toEqual([0, 1, 2])
+  })
+
   it('retries a rate-limited request and succeeds on a later attempt', async () => {
     fetchMock
       .mockResolvedValueOnce(jsonResponse({ error: 'slow down' }, 429))
@@ -278,7 +467,7 @@ describe('embed', () => {
     })
 
     expect(fetchMock).toHaveBeenCalledTimes(2)
-    expect(result.embeddings).toEqual([[7, 8]])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([7, 8])
   })
 
   it('marks a caller-supplied key as BYOK so Sim does not bill for it', async () => {
@@ -294,7 +483,7 @@ describe('embed', () => {
   })
 
   it('uses OpenRouter as an explicit transport for an OpenAI catalog model', async () => {
-    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]])))
+    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]], 5, 1024)))
 
     await embed(['hello'], {
       model: 'text-embedding-3-large',
@@ -320,7 +509,9 @@ describe('embed', () => {
    */
   describe('per-model token limits', () => {
     it("truncates against Gemini's lower ceiling rather than a shared constant", async () => {
-      fetchMock.mockResolvedValue(jsonResponse({ embeddings: [{ values: [1] }] }))
+      fetchMock.mockResolvedValue(
+        jsonResponse({ embeddings: [{ values: sizedVector([1], 3072) }] })
+      )
       // ~10k tokens: over Gemini's 2048 ceiling, but under the old 8000 constant,
       // so this used to reach the provider whole and come back a 502.
       const long = 'word '.repeat(8000)
@@ -334,7 +525,10 @@ describe('embed', () => {
 
     it("keeps text intact up to Cohere's much higher ceiling", async () => {
       fetchMock.mockResolvedValue(
-        jsonResponse({ embeddings: { float: [[1]] }, meta: { billed_units: { input_tokens: 9 } } })
+        jsonResponse({
+          embeddings: { float: [sizedVector([1], 1536)] },
+          meta: { billed_units: { input_tokens: 9 } },
+        })
       )
       // Over the old 8000 constant, well under Cohere's 128k, so it must survive.
       const long = 'word '.repeat(8000)
@@ -382,7 +576,9 @@ describe('embed', () => {
 
     it('estimates tokens from the projected values, not the originals', async () => {
       // Gemini omits usage, so the token count is estimated from what was sent.
-      fetchMock.mockResolvedValue(jsonResponse({ embeddings: [{ values: [1, 2, 3] }] }))
+      fetchMock.mockResolvedValue(
+        jsonResponse({ embeddings: [{ values: sizedVector([1, 2, 3], 3072) }] })
+      )
 
       const result = await embed(['x'.repeat(400)], {
         model: 'gemini-embedding-001',
@@ -403,7 +599,9 @@ describe('embed', () => {
      * shortening one discarded content that would have fit.
      */
     it('batches the projected text, not the original', async () => {
-      fetchMock.mockResolvedValue(jsonResponse({ embeddings: [{ values: [1] }] }))
+      fetchMock.mockResolvedValue(
+        jsonResponse({ embeddings: [{ values: sizedVector([1], 3072) }] })
+      )
       // Under Gemini's 2048 ceiling before projection, far over it after.
       const short = 'secret'
 
@@ -439,17 +637,17 @@ describe('embed', () => {
 
 describe('embedOpenRouter', () => {
   it('uses a dynamic model and reports the returned native dimensions', async () => {
-    fetchMock.mockResolvedValue(
-      jsonResponse(
+    fetchMock.mockImplementation(async (_url, init) => {
+      const body = JSON.parse((init as RequestInit).body as string)
+      const inputs = body.input as string[]
+      return jsonResponse(
         openAIBody(
-          [
-            [1, 2, 3],
-            [4, 5, 6],
-          ],
-          7
+          inputs.map((input) => (input === 'alpha' ? [1, 2, 3] : [4, 5, 6])),
+          inputs[0] === 'alpha' ? 3 : 4,
+          null
         )
       )
-    )
+    })
 
     const result = await embedOpenRouter(['alpha', 'beta'], {
       model: 'openrouter/qwen/qwen3-embedding-8b',
@@ -461,9 +659,13 @@ describe('embedOpenRouter', () => {
     const [url, init] = fetchMock.mock.calls[0]
     expect(url).toBe('https://openrouter.ai/api/v1/embeddings')
     expect(JSON.parse((init as RequestInit).body as string)).toMatchObject({
-      input: ['alpha', 'beta'],
+      input: ['alpha'],
       model: 'qwen/qwen3-embedding-8b',
     })
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(
+      JSON.parse((fetchMock.mock.calls[1][1] as RequestInit).body as string)
+    ).not.toHaveProperty('dimensions')
     expect(result).toMatchObject({
       embeddings: [
         [1, 2, 3],
@@ -478,33 +680,96 @@ describe('embedOpenRouter', () => {
   })
 
   it('fails when OpenRouter returns the wrong number of vectors', async () => {
-    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]])))
+    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]], 5, null)))
 
     await expect(
       embedOpenRouter(['alpha', 'beta'], {
         model: 'openrouter/qwen/qwen3-embedding-8b',
         apiKey: 'or-test',
         maxInputTokens: 32768,
+        dimensions: 2,
         projectInputs: null,
       })
     ).rejects.toThrow('returned 1 embeddings for 2 inputs')
   })
 
   it('fails when OpenRouter returns inconsistent vector dimensions', async () => {
-    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2], [3]])))
+    fetchMock
+      .mockResolvedValueOnce(jsonResponse(openAIBody([[1, 2]], 1, null)))
+      .mockResolvedValueOnce(jsonResponse(openAIBody([[3, 4], [5]], 2, null)))
 
     await expect(
-      embedOpenRouter(['alpha', 'beta'], {
+      embedOpenRouter(['alpha', 'beta', 'gamma'], {
         model: 'openrouter/qwen/qwen3-embedding-8b',
         apiKey: 'or-test',
         maxInputTokens: 32768,
         projectInputs: null,
       })
-    ).rejects.toThrow('inconsistent dimensions')
+    ).rejects.toThrow('vector 1 has 1 unexpected dimensions; expected 2')
+  })
+
+  it('fails when OpenRouter violates an explicitly requested dimension', async () => {
+    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]], 5, null)))
+
+    await expect(
+      embedOpenRouter(['alpha'], {
+        model: 'openrouter/qwen/qwen3-embedding-8b',
+        apiKey: 'or-test',
+        maxInputTokens: 32768,
+        dimensions: 3,
+        projectInputs: null,
+      })
+    ).rejects.toThrow('vector 0 has 2 unexpected dimensions; expected 3')
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('rejects a dynamic-model batch that differs from the learned dimension', async () => {
+    const inputs = Array.from({ length: 2049 }, (_, index) => `i${index}`)
+    fetchMock.mockImplementation(async (_url, init) => {
+      const body = JSON.parse((init as RequestInit).body as string)
+      const batch = body.input as string[]
+      const embedding = batch.length === 1 ? [2, 3, 4] : [1, 3]
+      return jsonResponse(
+        openAIBody(
+          batch.map(() => embedding),
+          batch.length,
+          null
+        )
+      )
+    })
+
+    await expect(
+      embedOpenRouter(inputs, {
+        model: 'openrouter/qwen/qwen3-embedding-8b',
+        apiKey: 'or-test',
+        maxInputTokens: 32768,
+        projectInputs: null,
+      })
+    ).rejects.toThrow('vector 0 has 2 unexpected dimensions; expected 3')
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+  })
+
+  it('treats OpenRouter HTTP 402 as exhausted credit and opens the circuit', async () => {
+    fetchMock.mockResolvedValue(jsonResponse({ error: { message: 'Payment required' } }, 402))
+
+    const options = {
+      model: 'openrouter/qwen/qwen3-embedding-8b',
+      apiKey: 'or-exhausted',
+      maxInputTokens: 32768,
+      projectInputs: null,
+    } as const
+
+    await expect(embedOpenRouter(['alpha'], options)).rejects.toEqual(
+      expect.objectContaining({ name: 'EmbeddingQuotaExhaustedError', status: 402 })
+    )
+    await expect(embedOpenRouter(['beta'], options)).rejects.toBeInstanceOf(
+      EmbeddingQuotaExhaustedError
+    )
+    expect(fetchMock).toHaveBeenCalledTimes(1)
   })
 
   it('truncates inputs to the selected model context length', async () => {
-    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]])))
+    fetchMock.mockResolvedValue(jsonResponse(openAIBody([[1, 2]], 5, null)))
 
     await embedOpenRouter(['alpha beta gamma'], {
       model: 'openrouter/thenlper/gte-base',
@@ -526,7 +791,8 @@ describe('embedOpenRouter', () => {
       return jsonResponse(
         openAIBody(
           inputs.map((input) => [Number(input.slice(1))]),
-          inputs.length
+          inputs.length,
+          null
         )
       )
     })
@@ -549,6 +815,87 @@ describe('embedOpenRouter', () => {
     expect(result.embeddings[0]).toEqual([0])
     expect(result.embeddings[2048]).toEqual([2048])
   })
+
+  it('learns a dynamic model dimension before response-safe batching', async () => {
+    const dimensions = 32_768
+    const inputs = Array.from({ length: 17 }, (_, index) => `i${index}`)
+    fetchMock.mockImplementation(async (_url, init) => {
+      const body = JSON.parse((init as RequestInit).body as string)
+      const batch = body.input as string[]
+      return jsonResponse(
+        openAIBody(
+          batch.map((input) => sizedVector([Number(input.slice(1))], dimensions)),
+          batch.length,
+          null
+        )
+      )
+    })
+
+    const result = await embedOpenRouter(inputs, {
+      model: 'openrouter/example/high-dimensional-model',
+      apiKey: 'or-test',
+      maxInputTokens: 32768,
+      projectInputs: null,
+    })
+
+    expect(
+      fetchMock.mock.calls
+        .map(([, init]) => JSON.parse((init as RequestInit).body as string).input.length)
+        .sort((a, b) => a - b)
+    ).toEqual([1, 1, 15])
+    expect(result.dimensions).toBe(dimensions)
+    expect(result.embeddings.map(([first]) => first)).toEqual(
+      Array.from({ length: 17 }, (_, index) => index)
+    )
+  })
+
+  it('rejects an oversized dynamic aggregate after discovery and before fan-out', async () => {
+    const dimensions = 32_768
+    fetchMock.mockResolvedValue(jsonResponse(openAIBody([sizedVector([1], dimensions)], 1, null)))
+
+    await expect(
+      embedOpenRouter(
+        Array.from({ length: 100 }, (_, index) => `i${index}`),
+        {
+          model: 'openrouter/example/high-dimensional-model',
+          apiKey: 'or-test',
+          maxInputTokens: 32768,
+          projectInputs: null,
+        }
+      )
+    ).rejects.toThrow('Embedding output')
+
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('rejects an oversized explicit-dimension aggregate before making a request', async () => {
+    await expect(
+      embedOpenRouter(
+        Array.from({ length: 1000 }, (_, index) => `i${index}`),
+        {
+          model: 'openrouter/example/high-dimensional-model',
+          apiKey: 'or-test',
+          maxInputTokens: 32768,
+          dimensions: 4096,
+          projectInputs: null,
+        }
+      )
+    ).rejects.toThrow('Embedding output')
+
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+})
+
+describe('embedding concurrency admission', () => {
+  it.each([
+    [Number.NaN, 8],
+    [0, 1],
+    [-10, 1],
+    [4.9, 4],
+    [10_000, 16],
+  ])('clamps %s to %s', (configured, expected) => {
+    expect(clampEmbeddingConcurrency(configured)).toBe(expected)
+  })
 })
 
 describe('knowledge embedding transport fallback', () => {
@@ -573,12 +920,25 @@ describe('knowledge embedding transport fallback', () => {
       dimensions: 1536,
     })
     expect(result).toMatchObject({
-      embeddings: [[1, 2]],
       billableTokens: 3,
       isBYOK: false,
       modelName: 'text-embedding-3-small',
       dimensions: 1536,
     })
+    expect(result.embeddings[0].slice(0, 2)).toEqual([1, 2])
+    expect(result.embeddings[0]).toHaveLength(1536)
+  })
+
+  it('rejects aggregate output above the safe limit before selecting a transport', async () => {
+    setEnv({ OPENROUTER_API_KEY: 'or-test' })
+    const texts = Array.from({ length: 5000 }, (_, index) => `input-${index}`)
+    const projectInputs = vi.fn((inputs: string[]) => inputs)
+
+    await expect(
+      embedKnowledgeForDeployment(texts, { ...options, projectInputs }, false)
+    ).rejects.toBeInstanceOf(EmbeddingOutputLimitError)
+    expect(projectInputs).not.toHaveBeenCalled()
+    expect(fetchMock).not.toHaveBeenCalled()
   })
 
   it('keeps the original OpenAI path when OpenRouter is not configured', async () => {
@@ -634,7 +994,9 @@ describe('knowledge embedding transport fallback', () => {
 
   it('does not use OpenRouter for non-OpenAI knowledge models', async () => {
     setEnv({ GEMINI_API_KEY: 'gemini-test', OPENROUTER_API_KEY: 'or-test' })
-    fetchMock.mockResolvedValue(jsonResponse({ embeddings: [{ values: [1, 2] }] }))
+    fetchMock.mockResolvedValue(
+      jsonResponse({ embeddings: [{ values: sizedVector([1, 2], 1536) }] })
+    )
 
     await embedKnowledgeForDeployment(
       ['hello'],
@@ -666,6 +1028,41 @@ describe('knowledge embedding transport fallback', () => {
     expect(fetchMock).toHaveBeenCalledOnce()
   })
 
+  it('falls back once when a provider returns a malformed success body', async () => {
+    setEnv({ OPENAI_API_KEY: 'openai-test', OPENROUTER_API_KEY: 'or-test' })
+    fetchMock.mockImplementation(async (url) =>
+      url === 'https://api.openai.com/v1/embeddings'
+        ? jsonResponse({ data: [], usage: { total_tokens: 1 } })
+        : jsonResponse(openAIBody([[7, 8]], 2))
+    )
+
+    const result = await embedKnowledgeForDeployment(['hello'], options, false)
+
+    expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
+      'https://api.openai.com/v1/embeddings',
+      'https://openrouter.ai/api/v1/embeddings',
+    ])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([7, 8])
+  })
+
+  it('falls back immediately when the first provider credential has exhausted credit', async () => {
+    setEnv({ OPENAI_API_KEY: 'openai-test', OPENROUTER_API_KEY: 'or-test' })
+    fetchMock.mockImplementation(async (url) =>
+      url === 'https://api.openai.com/v1/embeddings'
+        ? jsonResponse({ error: { type: 'insufficient_quota', code: 'insufficient_quota' } }, 429)
+        : jsonResponse(openAIBody([[7, 8]], 2))
+    )
+
+    const result = await embedKnowledgeForDeployment(['hello'], options, false)
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
+      'https://api.openai.com/v1/embeddings',
+      'https://openrouter.ai/api/v1/embeddings',
+    ])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([7, 8])
+  })
+
   it('falls back after transient retries and projects inputs only once', async () => {
     vi.useFakeTimers()
     setEnv({ OPENAI_API_KEY: 'openai-test', OPENROUTER_API_KEY: 'or-test' })
@@ -687,7 +1084,7 @@ describe('knowledge embedding transport fallback', () => {
     ).toBe(true)
     expect(fetchMock.mock.calls[attempts][0]).toBe('https://openrouter.ai/api/v1/embeddings')
     expect(projectInputs).toHaveBeenCalledOnce()
-    expect(result.embeddings).toEqual([[7, 8]])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([7, 8])
   })
 
   it('falls back only the failed batch and retains successful provider work', async () => {
@@ -719,7 +1116,7 @@ describe('knowledge embedding transport fallback', () => {
     expect(openRouterInputs).toEqual([secondInput])
     // The succeeding batch, every attempt on the failing one, then its fallback.
     expect(fetchMock).toHaveBeenCalledTimes(1 + (EMBEDDING_MAX_RETRIES + 1) + 1)
-    expect(result.embeddings).toEqual([[1], [2]])
+    expect(result.embeddings.map(([value]) => value)).toEqual([1, 2])
     expect(result.totalTokens).toBe(6)
     expect(result.billableTokens).toBe(3)
     expect(result.isBYOK).toBe(false)
@@ -784,7 +1181,7 @@ describe('knowledge embedding transport fallback', () => {
 
     const openAICalls = fetchMock.mock.calls.filter(([url]) => url.includes('api.openai.com'))
     expect(openAICalls).toHaveLength(1)
-    expect(result.embeddings).toEqual([[9, 9]])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([9, 9])
   })
 
   /**
@@ -819,7 +1216,7 @@ describe('knowledge embedding transport fallback', () => {
     const result = await pending
 
     expect(fetchMock).toHaveBeenCalledTimes(2)
-    expect(result.embeddings).toEqual([[4, 4]])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([4, 4])
   })
 
   /**
@@ -827,33 +1224,65 @@ describe('knowledge embedding transport fallback', () => {
    * sync — so retrying one burns the budget per document, indefinitely.
    */
   it('does not retry a 429 that reports an exhausted balance', async () => {
-    vi.useFakeTimers()
     setEnv({ OPENAI_API_KEY: 'openai-test' })
-    const fetchMock = vi.fn().mockImplementation(
-      async () =>
-        ({
-          ok: false,
-          status: 429,
-          statusText: 'Too Many Requests',
-          headers: new Headers(),
-          json: async () => ({}),
-          text: async () =>
-            JSON.stringify({
-              error: {
-                message: 'You have no credits remaining.',
-                type: 'insufficient_quota',
-                code: 'credit_balance_exhausted',
-              },
-            }),
-        }) as Response
+    const fetchMock = vi.fn().mockResolvedValue(
+      jsonResponse(
+        {
+          error: {
+            message: 'You have no credits remaining.',
+            type: 'insufficient_quota',
+            code: 'credit_balance_exhausted',
+          },
+        },
+        429
+      )
     )
     vi.stubGlobal('fetch', fetchMock)
 
-    const pending = embed(['hello'], { ...options, apiKey: 'openai-test' }).catch((e) => e)
-    await vi.runAllTimersAsync()
-    const error = await pending
+    await expect(embed(['hello'], { ...options, apiKey: 'openai-test' })).rejects.toEqual(
+      expect.objectContaining({ name: 'EmbeddingQuotaExhaustedError', quotaExhausted: true })
+    )
+    expect(fetchMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('classifies quota JSON beyond the diagnostic truncation boundary', async () => {
+    fetchMock.mockResolvedValue(
+      jsonResponse(
+        {
+          diagnosticPadding: 'x'.repeat(10_000),
+          error: {
+            message: 'You have no credits remaining.',
+            type: 'insufficient_quota',
+            code: 'insufficient_quota',
+          },
+        },
+        429
+      )
+    )
+
+    await expect(embed(['hello'], { ...options, apiKey: 'large-quota-body-key' })).rejects.toEqual(
+      expect.objectContaining({
+        name: 'EmbeddingQuotaExhaustedError',
+        status: 429,
+      })
+    )
+    expect(fetchMock).toHaveBeenCalledOnce()
+  })
+
+  it('short-circuits later requests that use the exhausted credential', async () => {
+    const quotaResponse = jsonResponse(
+      { error: { type: 'insufficient_quota', code: 'insufficient_quota' } },
+      429
+    )
+    fetchMock.mockResolvedValue(quotaResponse)
+
+    await expect(embed(['first'], { ...options, apiKey: 'exhausted-key' })).rejects.toBeInstanceOf(
+      EmbeddingQuotaExhaustedError
+    )
+    await expect(embed(['second'], { ...options, apiKey: 'exhausted-key' })).rejects.toBeInstanceOf(
+      EmbeddingQuotaExhaustedError
+    )
 
-    expect(error).toBeInstanceOf(EmbeddingAPIError)
     expect(fetchMock).toHaveBeenCalledTimes(1)
   })
 
@@ -887,7 +1316,7 @@ describe('knowledge embedding transport fallback', () => {
     const result = await pending
 
     expect(fetchMock).toHaveBeenCalledTimes(2)
-    expect(result.embeddings).toEqual([[5, 5]])
+    expect(result.embeddings[0].slice(0, 2)).toEqual([5, 5])
   })
 
   /**
@@ -900,6 +1329,20 @@ describe('knowledge embedding transport fallback', () => {
     expect(isTransientEmbeddingError(error)).toBe(true)
   })
 
+  it('classifies aggregate quota exhaustion only when every fallback exhausted credit', () => {
+    const openAIQuota = new EmbeddingQuotaExhaustedError('openai')
+    const openRouterQuota = new EmbeddingQuotaExhaustedError('openrouter')
+
+    expect(isEmbeddingQuotaExhaustion(new AggregateError([openAIQuota, openRouterQuota]))).toBe(
+      true
+    )
+    expect(
+      isEmbeddingQuotaExhaustion(
+        new AggregateError([openAIQuota, new EmbeddingAPIError('temporarily unavailable', 503)])
+      )
+    ).toBe(false)
+  })
+
   it('classifies only transient embedding failures for failover', () => {
     expect(isTransientEmbeddingError(new EmbeddingAPIError('unavailable', 503))).toBe(true)
     expect(isTransientEmbeddingError(new EmbeddingAPIError('rate limited', 429))).toBe(true)

apps/sim/lib/embeddings/client.ts

@@ -10,6 +10,11 @@ import {
 } from '@/lib/core/config/env-capabilities'
 import { isHosted } from '@/lib/core/config/env-flags'
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import {
+  DEFAULT_MAX_ERROR_BODY_BYTES,
+  readResponseJsonWithLimit,
+  readResponseTextWithLimit,
+} from '@/lib/core/utils/stream-limits'
 import {
   DEFAULT_EMBEDDING_MODEL,
   type EmbeddingModelInfo,
@@ -20,9 +25,16 @@ import {
 import { resolveProviderKey } from '@/lib/embeddings/keys'
 import { DEFAULT_OPENROUTER_EMBEDDING_MODEL } from '@/lib/embeddings/openrouter-models'
 import { getAdapterFactory } from '@/lib/embeddings/providers'
+import {
+  createEmbeddingQuotaCircuitIdentity,
+  type EmbeddingQuotaCircuitIdentity,
+  isEmbeddingQuotaCircuitOpen,
+  openEmbeddingQuotaCircuit,
+} from '@/lib/embeddings/quota-circuit'
 import { resolveEmbeddingRetryDelayMs } from '@/lib/embeddings/rate-limit'
 import type {
   EmbeddingProviderAdapter,
+  EmbeddingProviderKind,
   EmbeddingTaskType,
   EmbedOptions,
   EmbedResult,
@@ -48,7 +60,27 @@ const logger = createLogger('EmbeddingClient')
  * product reached four figures of in-flight requests against one key — enough to
  * hold a provider at its limit indefinitely, which no retry policy can absorb.
  */
-const MAX_CONCURRENT_BATCHES = envNumber(env.KB_CONFIG_EMBEDDING_CONCURRENCY, 8)
+const DEFAULT_CONCURRENT_BATCHES = 8
+const MAX_ALLOWED_CONCURRENT_BATCHES = 16
+
+/** Keeps one worker's fan-out inside a tested local memory/concurrency ceiling. */
+export function clampEmbeddingConcurrency(value: number): number {
+  if (!Number.isFinite(value)) return DEFAULT_CONCURRENT_BATCHES
+  return Math.min(Math.max(Math.floor(value), 1), MAX_ALLOWED_CONCURRENT_BATCHES)
+}
+
+const configuredEmbeddingConcurrency = envNumber(
+  env.KB_CONFIG_EMBEDDING_CONCURRENCY,
+  DEFAULT_CONCURRENT_BATCHES
+)
+const MAX_CONCURRENT_BATCHES = clampEmbeddingConcurrency(configuredEmbeddingConcurrency)
+if (configuredEmbeddingConcurrency !== MAX_CONCURRENT_BATCHES) {
+  logger.warn('Clamped embedding batch concurrency to the worker safety range', {
+    configured: configuredEmbeddingConcurrency,
+    effective: MAX_CONCURRENT_BATCHES,
+    maximum: MAX_ALLOWED_CONCURRENT_BATCHES,
+  })
+}
 const EMBEDDING_REQUEST_TIMEOUT_MS = 60_000
 
 /**
@@ -63,24 +95,47 @@ const EMBEDDING_REQUEST_TIMEOUT_MS = 60_000
  */
 const BATCH_TOKEN_TARGET = 8192
 
+/**
+ * Hard ceiling on one successful embedding response.
+ *
+ * Gemini's documented 100-item request cap at the catalog's largest 3,072
+ * dimensions fits comfortably inside 16 MiB, including a conservative JSON
+ * representation allowance. Larger OpenAI-style batches are split below from
+ * their expected vector width, so the guard rejects malformed provider output
+ * rather than valid catalog traffic.
+ */
+export const MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES = 16 * 1024 * 1024
+
+/** Bounds vectors retained across batches, aligned with the app's 100 MiB response ceiling. */
+export const MAX_EMBEDDING_AGGREGATE_RESPONSE_BYTES = 100 * 1024 * 1024
+
+/** Leaves room for the provider envelope, usage metadata, indices, and delimiters. */
+const EMBEDDING_RESPONSE_ENVELOPE_RESERVE_BYTES = 64 * 1024
+
+/**
+ * JSON may render a finite double with more characters than its in-memory
+ * representation. Thirty-two bytes per coordinate is deliberately conservative
+ * for the number, comma, and surrounding array syntax.
+ */
+const EMBEDDING_RESPONSE_BYTES_PER_DIMENSION = 32
+const EMBEDDING_RESPONSE_BYTES_PER_ITEM = 128
+
 /** Retries after the initial attempt, per embedding request. */
 export const EMBEDDING_MAX_RETRIES = 5
 
-/** Ceiling on a single wait between embedding attempts, including a provider-stated one. */
+/** Ceiling on exponential backoff when the provider supplies no retry delay. */
 export const EMBEDDING_MAX_RETRY_DELAY_MS = 30_000
 
 /**
- * Longest a request can stay in the retry loop: every attempt waits at most the
- * ceiling, so the budget is what the attempts span in total. A provider window
- * that reopens inside this is still reachable even though each individual wait
- * is clamped below it.
+ * Longest a request can stay in the retry loop. An admitted provider-stated wait
+ * is honored in full when it fits inside this deadline.
  */
 const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_RETRY_DELAY_MS
 
 export class EmbeddingAPIError extends Error {
   public status: number
 
-  /** Rejected for an exhausted balance rather than a recoverable rate. Both are 429. */
+  /** Rejected for an exhausted balance rather than a recoverable rate limit. */
   public quotaExhausted?: boolean
 
   /**
@@ -96,6 +151,59 @@ export class EmbeddingAPIError extends Error {
   }
 }
 
+class EmbeddingResponseValidationError extends EmbeddingAPIError {
+  constructor(message: string) {
+    super(`Embedding API returned an invalid success response: ${message}`, 502)
+    this.name = 'EmbeddingResponseValidationError'
+  }
+}
+
+export class EmbeddingOutputLimitError extends Error {
+  constructor(itemCount: number, dimensions: number, estimatedBytes: number) {
+    super(
+      `Embedding output for ${itemCount} inputs at ${dimensions} dimensions is estimated at ${estimatedBytes} bytes, exceeding the safe aggregate limit of ${MAX_EMBEDDING_AGGREGATE_RESPONSE_BYTES} bytes`
+    )
+    this.name = 'EmbeddingOutputLimitError'
+  }
+}
+
+export const EMBEDDING_QUOTA_EXHAUSTED_MESSAGE =
+  'The embedding provider has exhausted its available quota. Add credit or replace the credential before retrying.'
+
+/**
+ * A provider credential has no remaining credit. This remains transient across
+ * providers so a configured fallback can run, but it is terminal for the
+ * credential and for a Trigger task after every fallback is exhausted.
+ */
+export class EmbeddingQuotaExhaustedError extends EmbeddingAPIError {
+  public readonly providerId: EmbeddingProviderKind
+
+  constructor(providerId: EmbeddingProviderKind, cause?: unknown) {
+    const status = cause instanceof EmbeddingAPIError ? cause.status : 429
+    super(
+      `The ${providerId} embedding credential has exhausted its available quota. Add credit or replace the credential before retrying.`,
+      status
+    )
+    this.name = 'EmbeddingQuotaExhaustedError'
+    this.providerId = providerId
+    this.quotaExhausted = true
+    this.cause = cause
+  }
+}
+
+/**
+ * True only when the overall embedding operation failed because every provider
+ * it attempted had exhausted credit. A mixed fallback failure must retain task
+ * retries because another provider may merely be temporarily unavailable.
+ */
+export function isEmbeddingQuotaExhaustion(error: unknown): boolean {
+  if (error instanceof EmbeddingAPIError) return error.quotaExhausted === true
+  if (error instanceof AggregateError) {
+    return error.errors.length > 0 && error.errors.every(isEmbeddingQuotaExhaustion)
+  }
+  return false
+}
+
 /**
  * True when a rejection body reports an exhausted balance rather than a rate
  * limit. OpenAI returns 429 for both, but only a rate limit reopens: a spent
@@ -116,15 +224,24 @@ function isQuotaExhaustionBody(errorText: string): boolean {
   }
 }
 
+/** Reads a bounded provider body only for internal quota classification. */
+async function readEmbeddingErrorBody(response: Response): Promise<string> {
+  try {
+    return await readResponseTextWithLimit(response, {
+      maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES,
+      label: 'Embedding API error response',
+    })
+  } catch {
+    return ''
+  }
+}
+
 /**
  * True when the provider's stated wait outlasts the entire retry budget.
  *
- * The comparison is against {@link EMBEDDING_RETRY_BUDGET_MS} rather than the
- * per-attempt ceiling, because a window longer than one wait is still reachable:
- * the attempts are clamped individually but accumulate, so a 35s window reopens
- * before the second one lands. Only a window outlasting every attempt is
- * genuinely unreachable, and retrying into it spends the budget waiting on
- * something that cannot happen.
+ * The retry layer honors a provider-stated wait in full only when it fits inside
+ * the remaining operation deadline. A wait longer than the entire embedding
+ * budget can therefore never be admitted.
  *
  * The error stays transient — it just is not worth retrying here — so refusing
  * the retry surfaces it immediately and the fallback chain, which classifies
@@ -148,12 +265,14 @@ function statedWaitOutlastsBudget(error: unknown): boolean {
  */
 function isWorthRetrying(error: unknown): boolean {
   if (!isTransientEmbeddingError(error)) return false
+  if (error instanceof EmbeddingResponseValidationError) return false
   if (error instanceof EmbeddingAPIError && error.quotaExhausted) return false
   return !statedWaitOutlastsBudget(error)
 }
 
 export function isTransientEmbeddingError(error: unknown): boolean {
   if (error instanceof EmbeddingAPIError) {
+    if (error.quotaExhausted) return true
     return error.status === 429 || error.status >= 500
   }
   if (error instanceof Error && error.name === 'AbortError') return true
@@ -163,6 +282,8 @@ export function isTransientEmbeddingError(error: unknown): boolean {
 interface ResolvedProvider {
   adapter: EmbeddingProviderAdapter
   info: EmbeddingModelInfo
+  providerId: EmbeddingProviderKind
+  quotaCircuitIdentity: EmbeddingQuotaCircuitIdentity
   /** Model name as sent to the provider (an Azure deployment name when Azure is active). */
   modelName: string
   /** Dimensionality the request will produce, for reporting and billing. */
@@ -207,6 +328,8 @@ async function resolveProvider(model: string, options: EmbedOptions): Promise<Re
         nativeDimensions: info.nativeDimensions,
       }),
       info,
+      providerId: 'openrouter',
+      quotaCircuitIdentity: createEmbeddingQuotaCircuitIdentity('openrouter', options.apiKey),
       modelName: model,
       dimensions,
       isBYOK: true,
@@ -225,6 +348,8 @@ async function resolveProvider(model: string, options: EmbedOptions): Promise<Re
           apiVersion: azure.apiVersion,
         }),
         info,
+        providerId: 'azure-openai',
+        quotaCircuitIdentity: createEmbeddingQuotaCircuitIdentity('azure-openai', azure.apiKey),
         modelName: azure.deployment,
         dimensions,
         isBYOK: false,
@@ -243,28 +368,80 @@ async function resolveProvider(model: string, options: EmbedOptions): Promise<Re
       nativeDimensions: info.nativeDimensions,
     }),
     info,
+    providerId: info.provider,
+    quotaCircuitIdentity: createEmbeddingQuotaCircuitIdentity(info.provider, apiKey),
     modelName: model,
     dimensions,
     isBYOK,
   }
 }
 
+function validateEmbeddingBatch(
+  value: unknown,
+  expectedCount: number,
+  expectedDimensions: number | undefined
+): { embeddings: number[][]; dimensions: number } {
+  if (!Array.isArray(value)) {
+    throw new EmbeddingResponseValidationError('the vector payload is not an array')
+  }
+  if (value.length !== expectedCount) {
+    throw new EmbeddingResponseValidationError(
+      `returned ${value.length} embeddings for ${expectedCount} inputs`
+    )
+  }
+
+  let resolvedDimensions = expectedDimensions
+  for (let index = 0; index < value.length; index++) {
+    const vector = value[index]
+    if (!Array.isArray(vector) || vector.length === 0) {
+      throw new EmbeddingResponseValidationError(`vector ${index} is empty or not an array`)
+    }
+    if (
+      vector.some((coordinate) => typeof coordinate !== 'number' || !Number.isFinite(coordinate))
+    ) {
+      throw new EmbeddingResponseValidationError(
+        `vector ${index} contains a non-numeric or non-finite coordinate`
+      )
+    }
+
+    resolvedDimensions ??= vector.length
+    if (vector.length !== resolvedDimensions) {
+      const qualifier = expectedDimensions === undefined ? 'inconsistent' : 'unexpected'
+      throw new EmbeddingResponseValidationError(
+        `vector ${index} has ${vector.length} ${qualifier} dimensions; expected ${resolvedDimensions}`
+      )
+    }
+  }
+
+  if (resolvedDimensions === undefined) {
+    throw new EmbeddingResponseValidationError('the response did not contain any vectors')
+  }
+  return { embeddings: value as number[][], dimensions: resolvedDimensions }
+}
+
 /** `inputs` are already projected and batched by the embedding orchestrator. */
 async function callEmbeddingAPI(
   inputs: string[],
   adapter: EmbeddingProviderAdapter,
   tokenizerProvider: string,
   taskType: EmbeddingTaskType,
+  providerId: EmbeddingProviderKind,
+  quotaCircuitIdentity: EmbeddingQuotaCircuitIdentity,
   /**
    * The caller's explicit reduction, or undefined when none was requested. Kept
    * distinct from `provider.dimensions` because a model without Matryoshka
    * support rejects the parameter outright — sending it populated with the
    * native size is a 400, not a no-op.
    */
-  requestedDimensions: number | undefined
-): Promise<{ embeddings: number[][]; totalTokens: number }> {
+  requestedDimensions: number | undefined,
+  expectedDimensions: number | undefined
+): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> {
   return retryWithExponentialBackoff(
     async () => {
+      if (await isEmbeddingQuotaCircuitOpen(quotaCircuitIdentity)) {
+        throw new EmbeddingQuotaExhaustedError(providerId)
+      }
+
       const request = adapter.buildRequest({
         inputs,
         taskType,
@@ -282,12 +459,19 @@ async function callEmbeddingAPI(
       }).finally(() => clearTimeout(timeout))
 
       if (!response.ok) {
-        const errorText = await response.text()
+        const classificationBody = await readEmbeddingErrorBody(response)
         const error = new EmbeddingAPIError(
-          `Embedding API failed: ${response.status} ${response.statusText} - ${errorText}`,
+          `Embedding API failed: ${response.status}`,
           response.status
         )
-        error.quotaExhausted = isQuotaExhaustionBody(errorText)
+        error.quotaExhausted =
+          isQuotaExhaustionBody(classificationBody) ||
+          (providerId === 'openrouter' && response.status === 402)
+
+        if (error.quotaExhausted) {
+          await openEmbeddingQuotaCircuit(quotaCircuitIdentity)
+          throw new EmbeddingQuotaExhaustedError(providerId, error)
+        }
 
         /**
          * Carry the provider's own answer to "when may I retry" onto the error,
@@ -307,8 +491,21 @@ async function callEmbeddingAPI(
         throw error
       }
 
-      const json = await response.json()
-      const embeddings = request.parse(json)
+      const json = await readResponseJsonWithLimit(response, {
+        maxBytes: MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES,
+        label: 'Embedding API success response',
+      })
+      let parsedEmbeddings: unknown
+      try {
+        parsedEmbeddings = request.parse(json)
+      } catch {
+        throw new EmbeddingResponseValidationError('the vector payload could not be parsed')
+      }
+      const { embeddings, dimensions } = validateEmbeddingBatch(
+        parsedEmbeddings,
+        inputs.length,
+        expectedDimensions
+      )
       /**
        * Fallback for a response that carries no usage block. Estimated with the
        * provider's own tokenizer, which is approximate for every non-OpenAI
@@ -318,15 +515,13 @@ async function callEmbeddingAPI(
         request.parseTokens?.(json) ??
         inputs.reduce((sum, text) => sum + estimateTokenCount(text, tokenizerProvider).count, 0)
 
-      return { embeddings, totalTokens }
+      return { embeddings, totalTokens, dimensions }
     },
     {
       /**
        * Sized against a rate-limit window rather than a transient blip. The
-       * provider states its reset in tens of seconds, and the loop clamps that
-       * stated wait to `maxDelayMs` — at the previous 10s ceiling every attempt
-       * fired before the window reopened, so the budget was spent without one
-       * retry landing in the reopened window.
+       * provider states its reset in tens of seconds, and the loop honors that
+       * wait when it fits inside the operation budget.
        *
        * Bounded so a fully saturated provider cannot outlive the task: five
        * attempts at the ceiling is well inside `KB_CONFIG_MAX_DURATION`, and
@@ -335,6 +530,7 @@ async function callEmbeddingAPI(
       maxRetries: EMBEDDING_MAX_RETRIES,
       initialDelayMs: 1000,
       maxDelayMs: EMBEDDING_MAX_RETRY_DELAY_MS,
+      retryBudgetMs: EMBEDDING_RETRY_BUDGET_MS,
       retryCondition: isWorthRetrying,
     }
   )
@@ -408,11 +604,13 @@ async function embedWithProvider(
   requestedDimensions: number | undefined,
   provider: ResolvedProvider
 ): Promise<EmbedResult> {
+  assertEmbeddingAggregateResponseWithinLimit(boundedInputs.length, provider.dimensions)
   const batches = createEmbeddingBatches(
     boundedInputs,
     model,
     getEmbeddingInputLimits(provider.info),
-    provider.adapter.maxItemsPerRequest
+    provider.adapter.maxItemsPerRequest,
+    provider.dimensions
   )
 
   const batchResults = await mapWithConcurrency(
@@ -425,10 +623,18 @@ async function embedWithProvider(
           provider.adapter,
           provider.info.tokenizerProvider,
           taskType,
-          requestedDimensions
+          provider.providerId,
+          provider.quotaCircuitIdentity,
+          requestedDimensions,
+          provider.dimensions
         )
       } catch (error) {
-        logger.error(`Failed to generate embeddings for batch ${i + 1}/${batches.length}:`, error)
+        const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:`
+        if (isEmbeddingQuotaExhaustion(error)) {
+          logger.warn(message, { providerId: provider.providerId, quotaExhausted: true })
+        } else {
+          logger.error(message, error)
+        }
         throw error
       }
     }
@@ -451,7 +657,8 @@ function createEmbeddingBatches(
   boundedInputs: string[],
   model: string,
   limits: Pick<EmbeddingInputLimits, 'maxInputTokens' | 'maxTokensPerRequest'>,
-  itemLimit: number | undefined
+  itemLimit: number | undefined,
+  dimensions: number | undefined
 ): string[][] {
   const ceiling = limits.maxInputTokens
 
@@ -475,21 +682,65 @@ function createEmbeddingBatches(
   )
 
   const tokenBatches = batchByTokenLimit(boundedInputs, requestBudget, model)
-  return itemLimit ? tokenBatches.flatMap((batch) => chunkArray(batch, itemLimit)) : tokenBatches
+  const responseItemLimit = dimensions
+    ? Math.max(
+        1,
+        Math.floor(
+          (MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES - EMBEDDING_RESPONSE_ENVELOPE_RESERVE_BYTES) /
+            (dimensions * EMBEDDING_RESPONSE_BYTES_PER_DIMENSION +
+              EMBEDDING_RESPONSE_BYTES_PER_ITEM)
+        )
+      )
+    : undefined
+  const effectiveItemLimit =
+    itemLimit && responseItemLimit
+      ? Math.min(itemLimit, responseItemLimit)
+      : (itemLimit ?? responseItemLimit)
+
+  return effectiveItemLimit
+    ? tokenBatches.flatMap((batch) => chunkArray(batch, effectiveItemLimit))
+    : tokenBatches
+}
+
+function assertEmbeddingAggregateResponseWithinLimit(itemCount: number, dimensions: number): void {
+  if (itemCount <= getEmbeddingAggregateItemLimit(dimensions)) return
+
+  const estimatedBytes =
+    EMBEDDING_RESPONSE_ENVELOPE_RESERVE_BYTES +
+    itemCount *
+      (dimensions * EMBEDDING_RESPONSE_BYTES_PER_DIMENSION + EMBEDDING_RESPONSE_BYTES_PER_ITEM)
+  throw new EmbeddingOutputLimitError(itemCount, dimensions, estimatedBytes)
+}
+
+export function getEmbeddingAggregateItemLimit(dimensions: number): number {
+  if (!Number.isInteger(dimensions) || dimensions <= 0) {
+    throw new Error('Embedding dimensions must be a positive integer')
+  }
+  return Math.floor(
+    (MAX_EMBEDDING_AGGREGATE_RESPONSE_BYTES - EMBEDDING_RESPONSE_ENVELOPE_RESERVE_BYTES) /
+      (dimensions * EMBEDDING_RESPONSE_BYTES_PER_DIMENSION + EMBEDDING_RESPONSE_BYTES_PER_ITEM)
+  )
 }
 
 function combineEmbeddingBatches(
-  batchResults: readonly { embeddings: number[][]; totalTokens: number }[]
-): { embeddings: number[][]; totalTokens: number } {
+  batchResults: readonly { embeddings: number[][]; totalTokens: number; dimensions: number }[]
+): { embeddings: number[][]; totalTokens: number; dimensions: number | undefined } {
   const embeddings: number[][] = []
   let totalTokens = 0
+  let dimensions: number | undefined
   for (const batch of batchResults) {
+    dimensions ??= batch.dimensions
+    if (batch.dimensions !== dimensions) {
+      throw new EmbeddingResponseValidationError(
+        `concurrent batches returned inconsistent dimensions (${dimensions} and ${batch.dimensions})`
+      )
+    }
     for (const vector of batch.embeddings) {
       embeddings.push(vector)
     }
     totalTokens += batch.totalTokens
   }
-  return { embeddings, totalTokens }
+  return { embeddings, totalTokens, dimensions }
 }
 
 /**
@@ -532,26 +783,59 @@ export async function embedOpenRouter(
     apiKey: options.apiKey,
     nativeDimensions: options.dimensions ?? 0,
   })
-  const batches = createEmbeddingBatches(boundedInputs, model, limits, adapter.maxItemsPerRequest)
-  const batchResults = await mapWithConcurrency(batches, MAX_CONCURRENT_BATCHES, async (batch) =>
-    callEmbeddingAPI(batch, adapter, limits.tokenizerProvider, 'document', options.dimensions)
-  )
+  const quotaCircuitIdentity = createEmbeddingQuotaCircuitIdentity('openrouter', options.apiKey)
+  const callOpenRouterBatch = (
+    batch: string[],
+    expectedDimensions: number | undefined
+  ): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> =>
+    callEmbeddingAPI(
+      batch,
+      adapter,
+      limits.tokenizerProvider,
+      'document',
+      'openrouter',
+      quotaCircuitIdentity,
+      options.dimensions,
+      expectedDimensions
+    )
+
+  let batchResults: { embeddings: number[][]; totalTokens: number; dimensions: number }[]
+  if (options.dimensions === undefined) {
+    const firstInput = boundedInputs[0]
+    if (firstInput === undefined) {
+      throw new EmbeddingResponseValidationError('the response did not contain any vectors')
+    }
+    const firstResult = await callOpenRouterBatch([firstInput], undefined)
+    assertEmbeddingAggregateResponseWithinLimit(boundedInputs.length, firstResult.dimensions)
+    const batches = createEmbeddingBatches(
+      boundedInputs.slice(1),
+      model,
+      limits,
+      adapter.maxItemsPerRequest,
+      firstResult.dimensions
+    )
+    const remainingResults = await mapWithConcurrency(batches, MAX_CONCURRENT_BATCHES, (batch) =>
+      callOpenRouterBatch(batch, firstResult.dimensions)
+    )
+    batchResults = [firstResult, ...remainingResults]
+  } else {
+    assertEmbeddingAggregateResponseWithinLimit(boundedInputs.length, options.dimensions)
+    const batches = createEmbeddingBatches(
+      boundedInputs,
+      model,
+      limits,
+      adapter.maxItemsPerRequest,
+      options.dimensions
+    )
+    batchResults = await mapWithConcurrency(batches, MAX_CONCURRENT_BATCHES, (batch) =>
+      callOpenRouterBatch(batch, options.dimensions)
+    )
+  }
   const result = combineEmbeddingBatches(batchResults)
 
-  if (result.embeddings.length !== boundedInputs.length) {
-    throw new Error(
-      `OpenRouter returned ${result.embeddings.length} embeddings for ${boundedInputs.length} inputs`
-    )
-  }
-  const dimensions = result.embeddings[0]?.length
-  if (!dimensions) throw new Error('OpenRouter returned an empty embedding vector')
-  if (result.embeddings.some((embedding) => embedding.length !== dimensions)) {
-    throw new Error('OpenRouter returned embedding vectors with inconsistent dimensions')
-  }
-  if (options.dimensions !== undefined && dimensions !== options.dimensions) {
-    throw new Error(
-      `OpenRouter returned ${dimensions} dimensions instead of the requested ${options.dimensions}`
-    )
+  const dimensions = result.dimensions
+  if (dimensions === undefined) {
+    throw new EmbeddingResponseValidationError('the response did not contain any vectors')
   }
 
   return {
@@ -585,6 +869,7 @@ export async function embedKnowledgeForDeployment(
   }
 
   const dimensions = resolveDimensions(info, options.dimensions)
+  assertEmbeddingAggregateResponseWithinLimit(texts.length, dimensions)
   const taskType = options.taskType ?? 'document'
   const boundedInputs = prepareEmbeddingInputs(
     texts,
@@ -608,6 +893,8 @@ export async function embedKnowledgeForDeployment(
           apiVersion: azure.apiVersion,
         }),
         info,
+        providerId: 'azure-openai',
+        quotaCircuitIdentity: createEmbeddingQuotaCircuitIdentity('azure-openai', azure.apiKey),
         modelName: azure.deployment,
         dimensions,
         isBYOK: false,
@@ -622,6 +909,8 @@ export async function embedKnowledgeForDeployment(
           nativeDimensions: info.nativeDimensions,
         }),
         info,
+        providerId: 'openai',
+        quotaCircuitIdentity: createEmbeddingQuotaCircuitIdentity('openai', apiKey),
         modelName: model,
         dimensions,
         isBYOK: Boolean(workspaceKey),
@@ -636,6 +925,11 @@ export async function embedKnowledgeForDeployment(
           nativeDimensions: info.nativeDimensions,
         }),
         info,
+        providerId: 'openrouter',
+        quotaCircuitIdentity: createEmbeddingQuotaCircuitIdentity(
+          'openrouter',
+          env.OPENROUTER_API_KEY
+        ),
         modelName: model,
         dimensions,
         isBYOK: false,
@@ -649,10 +943,12 @@ export async function embedKnowledgeForDeployment(
     factories,
     shouldFallback: isTransientEmbeddingError,
     onFailure(providerId, error) {
-      logger.warn('Knowledge embedding provider failed; continuing fallback chain', {
-        providerId,
-        error,
-      })
+      logger.warn(
+        'Knowledge embedding provider failed; continuing fallback chain',
+        isEmbeddingQuotaExhaustion(error)
+          ? { providerId, quotaExhausted: true }
+          : { providerId, error }
+      )
     },
   })
 
@@ -663,7 +959,8 @@ export async function embedKnowledgeForDeployment(
     boundedInputs,
     model,
     info,
-    itemLimits.length > 0 ? Math.min(...itemLimits) : undefined
+    itemLimits.length > 0 ? Math.min(...itemLimits) : undefined,
+    dimensions
   )
   const batchResults = await mapWithConcurrency(
     batches,
@@ -676,12 +973,20 @@ export async function embedKnowledgeForDeployment(
             provider.adapter,
             provider.info.tokenizerProvider,
             taskType,
-            options.dimensions
+            provider.providerId,
+            provider.quotaCircuitIdentity,
+            options.dimensions,
+            provider.dimensions
           )),
           provider,
         }))
       } catch (error) {
-        logger.error(`Failed to generate embeddings for batch ${i + 1}/${batches.length}:`, error)
+        const message = `Failed to generate embeddings for batch ${i + 1}/${batches.length}:`
+        if (isEmbeddingQuotaExhaustion(error)) {
+          logger.warn(message, { quotaExhausted: true })
+        } else {
+          logger.error(message, error)
+        }
         throw error
       }
     }

apps/sim/lib/embeddings/index.ts

@@ -9,7 +9,15 @@ export {
   findEmbeddingModelInfo,
   resolveDimensions,
 } from '@/lib/embeddings/catalog'
-export { embed, embedKnowledge, embedOpenRouter } from '@/lib/embeddings/client'
+export {
+  EMBEDDING_QUOTA_EXHAUSTED_MESSAGE,
+  EmbeddingOutputLimitError,
+  embed,
+  embedKnowledge,
+  embedOpenRouter,
+  getEmbeddingAggregateItemLimit,
+  isEmbeddingQuotaExhaustion,
+} from '@/lib/embeddings/client'
 export { DEFAULT_OPENROUTER_EMBEDDING_MODEL } from '@/lib/embeddings/openrouter-models'
 export type {
   EmbeddingTaskType,

apps/sim/lib/embeddings/quota-circuit.test.ts

@@ -0,0 +1,130 @@
+/**
+ * @vitest-environment node
+ */
+import { sha256Hex } from '@sim/security/hash'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockGetRedisClient } = vi.hoisted(() => ({
+  mockGetRedisClient: vi.fn(),
+}))
+
+vi.mock('@/lib/core/config/redis', () => ({
+  getRedisClient: mockGetRedisClient,
+}))
+
+import {
+  createEmbeddingQuotaCircuitIdentity,
+  EMBEDDING_QUOTA_CIRCUIT_TTL_MS,
+  isEmbeddingQuotaCircuitOpen,
+  openEmbeddingQuotaCircuit,
+  resetEmbeddingQuotaCircuitsForTesting,
+} from '@/lib/embeddings/quota-circuit'
+
+describe('embedding quota circuit', () => {
+  const values = new Map<string, string>()
+  const redis = {
+    get: vi.fn(async (key: string) => values.get(key) ?? null),
+    eval: vi.fn(async (_script: string, _keyCount: number, key: string, value: string) => {
+      const current = Number(values.get(key))
+      const proposed = Number(value)
+      const expiry = Number.isFinite(current) ? Math.max(current, proposed) : proposed
+      values.set(key, String(expiry))
+      return expiry
+    }),
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    values.clear()
+    resetEmbeddingQuotaCircuitsForTesting()
+    mockGetRedisClient.mockReturnValue(redis)
+  })
+
+  afterEach(() => {
+    resetEmbeddingQuotaCircuitsForTesting()
+  })
+
+  it('fingerprints credentials without retaining the secret', () => {
+    const identity = createEmbeddingQuotaCircuitIdentity('openai', 'sk-secret')
+
+    expect(identity).toEqual({
+      providerId: 'openai',
+      credentialFingerprint: sha256Hex('sk-secret'),
+    })
+    expect(JSON.stringify(identity)).not.toContain('sk-secret')
+  })
+
+  it('shares an exhausted credential across workers without extending its expiry', async () => {
+    const identity = createEmbeddingQuotaCircuitIdentity('openai', 'sk-shared')
+    const openedAt = 1_000_000
+
+    await openEmbeddingQuotaCircuit(identity, openedAt)
+    resetEmbeddingQuotaCircuitsForTesting()
+
+    expect(await isEmbeddingQuotaCircuitOpen(identity, openedAt + 1)).toBe(true)
+    expect(redis.eval).toHaveBeenCalledWith(
+      expect.stringContaining("'PXAT'"),
+      1,
+      expect.stringContaining('embedding-quota-circuit:openai:'),
+      String(openedAt + EMBEDDING_QUOTA_CIRCUIT_TTL_MS)
+    )
+    expect(
+      await isEmbeddingQuotaCircuitOpen(identity, openedAt + EMBEDDING_QUOTA_CIRCUIT_TTL_MS + 1)
+    ).toBe(false)
+  })
+
+  it('does not shorten an existing expiry when an older observation completes later', async () => {
+    const identity = createEmbeddingQuotaCircuitIdentity('openai', 'sk-concurrent')
+    const olderObservation = 1_000_000
+    const newerObservation = olderObservation + 60_000
+
+    await openEmbeddingQuotaCircuit(identity, newerObservation)
+    await openEmbeddingQuotaCircuit(identity, olderObservation)
+    resetEmbeddingQuotaCircuitsForTesting()
+
+    expect(
+      await isEmbeddingQuotaCircuitOpen(
+        identity,
+        olderObservation + EMBEDDING_QUOTA_CIRCUIT_TTL_MS + 1
+      )
+    ).toBe(true)
+    expect([...values.values()]).toEqual([
+      String(newerObservation + EMBEDDING_QUOTA_CIRCUIT_TTL_MS),
+    ])
+    const script = redis.eval.mock.calls.at(-1)?.[0]
+    expect(script).toMatch(/if\s+current\s+and\s+current\s*>\s*proposed\s+then/)
+  })
+
+  it('isolates different providers and credentials', async () => {
+    const exhausted = createEmbeddingQuotaCircuitIdentity('openai', 'sk-exhausted')
+
+    await openEmbeddingQuotaCircuit(exhausted, 1_000_000)
+
+    expect(await isEmbeddingQuotaCircuitOpen(exhausted, 1_000_001)).toBe(true)
+    expect(
+      await isEmbeddingQuotaCircuitOpen(
+        createEmbeddingQuotaCircuitIdentity('openai', 'sk-healthy'),
+        1_000_001
+      )
+    ).toBe(false)
+    expect(
+      await isEmbeddingQuotaCircuitOpen(
+        createEmbeddingQuotaCircuitIdentity('openrouter', 'sk-exhausted'),
+        1_000_001
+      )
+    ).toBe(false)
+  })
+
+  it('fails open when Redis is unavailable', async () => {
+    mockGetRedisClient.mockImplementation(() => {
+      throw new Error('Redis unavailable')
+    })
+
+    expect(
+      await isEmbeddingQuotaCircuitOpen(
+        createEmbeddingQuotaCircuitIdentity('openai', 'sk-test'),
+        1_000_000
+      )
+    ).toBe(false)
+  })
+})

apps/sim/lib/embeddings/quota-circuit.ts

@@ -0,0 +1,136 @@
+import { createLogger } from '@sim/logger'
+import { sha256Hex } from '@sim/security/hash'
+import { getRedisClient } from '@/lib/core/config/redis'
+import type { EmbeddingProviderKind } from '@/lib/embeddings/types'
+
+const logger = createLogger('EmbeddingQuotaCircuit')
+
+/**
+ * A billing change can take a couple of minutes to reach a provider. Keeping the
+ * circuit open for five minutes absorbs the document queue that observed the
+ * same exhausted credential while still probing again without operator action.
+ */
+export const EMBEDDING_QUOTA_CIRCUIT_TTL_MS = 5 * 60 * 1000
+
+const REDIS_KEY_PREFIX = 'embedding-quota-circuit:'
+const MAX_LOCAL_CIRCUITS = 1024
+const OPEN_QUOTA_CIRCUIT_SCRIPT = `
+local current = tonumber(redis.call('GET', KEYS[1]))
+local proposed = tonumber(ARGV[1])
+local expiry = proposed
+if current and current > proposed then
+  expiry = current
+end
+redis.call('SET', KEYS[1], tostring(expiry), 'PXAT', expiry)
+return expiry
+`
+
+export interface EmbeddingQuotaCircuitIdentity {
+  readonly providerId: EmbeddingProviderKind
+  /** SHA-256 fingerprint; the provider credential itself never reaches cache or logs. */
+  readonly credentialFingerprint: string
+}
+
+const localCircuits = new Map<string, number>()
+
+function writeLocalCircuit(key: string, expiresAt: number, now: number): void {
+  for (const [candidateKey, candidateExpiry] of localCircuits) {
+    if (candidateExpiry <= now) localCircuits.delete(candidateKey)
+  }
+  const currentExpiry = localCircuits.get(key)
+  if (currentExpiry !== undefined) {
+    localCircuits.set(key, Math.max(currentExpiry, expiresAt))
+    return
+  }
+  while (localCircuits.size >= MAX_LOCAL_CIRCUITS) {
+    const oldestKey = localCircuits.keys().next().value
+    if (oldestKey === undefined) break
+    localCircuits.delete(oldestKey)
+  }
+  localCircuits.set(key, expiresAt)
+}
+
+function circuitKey(identity: EmbeddingQuotaCircuitIdentity): string {
+  return `${identity.providerId}:${identity.credentialFingerprint}`
+}
+
+function redisKey(identity: EmbeddingQuotaCircuitIdentity): string {
+  return `${REDIS_KEY_PREFIX}${circuitKey(identity)}`
+}
+
+/** Isolates quota state to the exact provider credential without retaining the secret. */
+export function createEmbeddingQuotaCircuitIdentity(
+  providerId: EmbeddingProviderKind,
+  apiKey: string
+): EmbeddingQuotaCircuitIdentity {
+  return {
+    providerId,
+    credentialFingerprint: sha256Hex(apiKey),
+  }
+}
+
+function readLocalCircuit(identity: EmbeddingQuotaCircuitIdentity, now: number): boolean {
+  const key = circuitKey(identity)
+  const expiresAt = localCircuits.get(key)
+  if (expiresAt === undefined) return false
+  if (expiresAt > now) return true
+  localCircuits.delete(key)
+  return false
+}
+
+/**
+ * Returns whether another worker has already observed exhausted credit for this
+ * credential. Cache failures deliberately fail open: quota protection must not
+ * turn a Redis outage into a knowledge-search outage.
+ */
+export async function isEmbeddingQuotaCircuitOpen(
+  identity: EmbeddingQuotaCircuitIdentity,
+  now = Date.now()
+): Promise<boolean> {
+  if (readLocalCircuit(identity, now)) return true
+
+  try {
+    const redis = getRedisClient()
+    if (!redis) return false
+    const storedExpiry = await redis.get(redisKey(identity))
+    if (!storedExpiry) return false
+    const expiresAt = Number(storedExpiry)
+    if (!Number.isFinite(expiresAt) || expiresAt <= now) return false
+    writeLocalCircuit(circuitKey(identity), expiresAt, now)
+    return true
+  } catch (error) {
+    logger.warn('Failed to read embedding quota circuit; continuing with provider request', {
+      providerId: identity.providerId,
+      error,
+    })
+    return false
+  }
+}
+
+/**
+ * Shares a provider-declared exhausted-credit result with every worker using
+ * the same credential. The absolute expiry keeps a late reader from extending
+ * the circuit by another full TTL.
+ */
+export async function openEmbeddingQuotaCircuit(
+  identity: EmbeddingQuotaCircuitIdentity,
+  now = Date.now()
+): Promise<void> {
+  const expiresAt = now + EMBEDDING_QUOTA_CIRCUIT_TTL_MS
+  writeLocalCircuit(circuitKey(identity), expiresAt, now)
+
+  try {
+    const redis = getRedisClient()
+    if (!redis) return
+    await redis.eval(OPEN_QUOTA_CIRCUIT_SCRIPT, 1, redisKey(identity), String(expiresAt))
+  } catch (error) {
+    logger.warn('Failed to share embedding quota circuit; process-local circuit remains active', {
+      providerId: identity.providerId,
+      error,
+    })
+  }
+}
+
+export function resetEmbeddingQuotaCircuitsForTesting(): void {
+  localCircuits.clear()
+}

apps/sim/lib/file-parsers/csv-parser.ts

@@ -2,6 +2,7 @@ import { createReadStream, existsSync } from 'fs'
 import { Readable } from 'stream'
 import { createLogger } from '@sim/logger'
 import { type Options, parse } from 'csv-parse'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils'
 
@@ -111,13 +112,18 @@ export class CsvParser implements FileParser {
         if (errorCount >= CONFIG.MAX_ERRORS) {
           aborted = true
           parser.destroy()
-          reject(new Error(`Too many errors (${errorCount}). File may be corrupted.`))
+          reject(
+            new FileParserError(
+              'invalid_format',
+              `Too many errors (${errorCount}). File may be corrupted.`
+            )
+          )
         }
       })
 
       parser.on('error', (err: Error) => {
         logger.error('CSV parser error:', err)
-        reject(new Error(`CSV parsing failed: ${err.message}`))
+        reject(new FileParserError('invalid_format', `CSV parsing failed: ${err.message}`, err))
       })
 
       parser.on('end', () => {

apps/sim/lib/file-parsers/data-uri.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it, vi } from 'vitest'
+import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri'
+import { FileParserError } from '@/lib/file-parsers/errors'
+
+describe('decodeDataUriWithinLimit', () => {
+  it('preserves commas after the first delimiter', () => {
+    const decoded = decodeDataUriWithinLimit('data:text/plain,alpha,beta,gamma', 100)
+
+    expect(decoded.buffer.toString('utf8')).toBe('alpha,beta,gamma')
+    expect(decoded.mediaType).toBe('text/plain')
+  })
+
+  it('rejects malformed base64 instead of silently decoding a prefix', () => {
+    expect(() => decodeDataUriWithinLimit('data:text/plain;base64,@@@=', 100)).toThrowError(
+      expect.objectContaining({ code: 'invalid_format' })
+    )
+  })
+
+  it('accepts valid unpadded base64 without weakening alphabet validation', () => {
+    const decoded = decodeDataUriWithinLimit('data:text/plain;base64,aGk', 2)
+
+    expect(decoded.buffer.toString('utf8')).toBe('hi')
+  })
+
+  it('accepts percent-escaped base64 bytes before validating padding', () => {
+    const decoded = decodeDataUriWithinLimit('data:text/plain;base64,aGk%3D', 2)
+
+    expect(decoded.buffer.toString('utf8')).toBe('hi')
+  })
+
+  it('rejects malformed percent escapes in a base64 payload', () => {
+    expect(() => decodeDataUriWithinLimit('data:text/plain;base64,aGk%ZZ', 2)).toThrowError(
+      expect.objectContaining({ code: 'invalid_format' })
+    )
+  })
+
+  it('accepts a base64 payload at the encoded boundary, including whitespace', () => {
+    const payload = Buffer.from('12345').toString('base64')
+    const decoded = decodeDataUriWithinLimit(
+      `data:application/octet-stream;base64,${payload.slice(0, 4)}\n${payload.slice(4)}`,
+      5
+    )
+
+    expect(decoded.buffer.toString('utf8')).toBe('12345')
+    expect(decoded.mediaType).toBe('application/octet-stream')
+  })
+
+  it('preserves percent-encoded binary octets that are not standalone UTF-8', () => {
+    const decoded = decodeDataUriWithinLimit('data:application/octet-stream,%FF%00A', 3)
+
+    expect([...decoded.buffer]).toEqual([0xff, 0x00, 0x41])
+  })
+
+  it('rejects the encoded representation before decoding', () => {
+    const error = (() => {
+      try {
+        decodeDataUriWithinLimit(`data:text/plain,${'x'.repeat(50)}`, 10)
+      } catch (caught) {
+        return caught
+      }
+    })()
+
+    expect(error).toBeInstanceOf(FileParserError)
+    expect(error).toMatchObject({ code: 'complexity_limit' })
+  })
+
+  it('rejects an oversized descriptor before splitting its parameters', () => {
+    const descriptor = `text/plain;${'x;'.repeat(4096)}`
+
+    expect(() => decodeDataUriWithinLimit(`data:${descriptor},ok`, 10)).toThrowError(
+      expect.objectContaining({ code: 'complexity_limit' })
+    )
+  })
+
+  it('sizes non-base64 decode storage from the input instead of the file cap', () => {
+    const allocation = vi.spyOn(Buffer, 'allocUnsafe')
+    try {
+      const decoded = decodeDataUriWithinLimit('data:text/plain,hi', 100 * 1024 * 1024)
+
+      expect(decoded.buffer.toString('utf8')).toBe('hi')
+      expect(allocation).toHaveBeenCalledWith(7)
+    } finally {
+      allocation.mockRestore()
+    }
+  })
+
+  it('rejects decoded bytes beyond the cap', () => {
+    const payload = Buffer.from('eleven-bytes').toString('base64')
+
+    expect(() => decodeDataUriWithinLimit(`data:text/plain;base64,${payload}`, 10)).toThrowError(
+      expect.objectContaining({ code: 'complexity_limit' })
+    )
+  })
+
+  it('rejects a near-four-times base64 payload before decoding', () => {
+    const maxBytes = 1024
+    const payload = 'A'.repeat(maxBytes * 4)
+
+    expect(() =>
+      decodeDataUriWithinLimit(`data:application/octet-stream;base64,${payload}`, maxBytes)
+    ).toThrowError(expect.objectContaining({ code: 'complexity_limit' }))
+  })
+
+  it('bounds base64 transport whitespace before compacting the payload', () => {
+    const maxBytes = 1024
+    const encodedPayload = Buffer.alloc(maxBytes).toString('base64')
+    const whitespaceBomb = `${encodedPayload}${' '.repeat(2048)}`
+
+    expect(() =>
+      decodeDataUriWithinLimit(`data:application/octet-stream;base64,${whitespaceBomb}`, maxBytes)
+    ).toThrowError(expect.objectContaining({ code: 'complexity_limit' }))
+  })
+})

apps/sim/lib/file-parsers/data-uri.ts

@@ -0,0 +1,176 @@
+import { FileParserError } from '@/lib/file-parsers/errors'
+
+export interface DecodedDataUri {
+  buffer: Buffer
+  mediaType: string | null
+}
+
+const MAX_SAFE_BASE64_INPUT_BYTES = Math.floor(Number.MAX_SAFE_INTEGER / 4) * 3
+const MAX_SAFE_PERCENT_INPUT_BYTES = Math.floor((Number.MAX_SAFE_INTEGER - 4) / 4)
+const MAX_BASE64_WHITESPACE_BYTES = 4 * 1024 * 1024
+const MAX_DATA_URI_DESCRIPTOR_CHARACTERS = 4096
+
+function getMaxBase64EncodedLength(maxBytes: number): number {
+  if (maxBytes > MAX_SAFE_BASE64_INPUT_BYTES) return Number.MAX_SAFE_INTEGER
+  return Math.ceil(maxBytes / 3) * 4
+}
+
+function getBase64WhitespaceAllowance(maxEncodedLength: number): number {
+  return Math.min(Math.max(1024, Math.ceil(maxEncodedLength / 32)), MAX_BASE64_WHITESPACE_BYTES)
+}
+
+function getMaxBase64TransportLength(maxEncodedLength: number): number {
+  if (maxEncodedLength === Number.MAX_SAFE_INTEGER) return Number.MAX_SAFE_INTEGER
+  const whitespaceAllowance = getBase64WhitespaceAllowance(maxEncodedLength)
+  if (maxEncodedLength > Math.floor((Number.MAX_SAFE_INTEGER - whitespaceAllowance) / 3)) {
+    return Number.MAX_SAFE_INTEGER
+  }
+  return maxEncodedLength * 3 + whitespaceAllowance
+}
+
+function getMaxPercentEncodedLength(maxBytes: number): number {
+  if (maxBytes > MAX_SAFE_PERCENT_INPUT_BYTES) return Number.MAX_SAFE_INTEGER
+  return maxBytes * 4 + 4
+}
+
+/**
+ * Decodes a data URI without allowing its encoded or decoded representation to
+ * exceed the caller's byte budget. The first comma is the delimiter; later
+ * commas are payload and must be preserved.
+ */
+export function decodeDataUriWithinLimit(dataUri: string, maxBytes: number): DecodedDataUri {
+  if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) {
+    throw new RangeError('Data URI byte limit must be a non-negative safe integer')
+  }
+
+  const commaIndex = dataUri.indexOf(',')
+  if (dataUri.slice(0, 5).toLowerCase() !== 'data:' || commaIndex < 5) {
+    throw new FileParserError('invalid_format', 'Invalid data URI format')
+  }
+  if (commaIndex - 5 > MAX_DATA_URI_DESCRIPTOR_CHARACTERS) {
+    throw new FileParserError(
+      'complexity_limit',
+      `Data URI descriptor exceeds the safe limit of ${MAX_DATA_URI_DESCRIPTOR_CHARACTERS} characters`
+    )
+  }
+
+  const descriptor = dataUri.slice(5, commaIndex)
+  const descriptorParts = descriptor.split(';')
+  const mediaType = descriptorParts[0]?.trim() || null
+  const isBase64 = descriptorParts.slice(1).some((part) => part.toLowerCase() === 'base64')
+  const payloadStart = commaIndex + 1
+  const encodedPayloadLength = dataUri.length - payloadStart
+  const encodedCharacterLimit = isBase64
+    ? getMaxBase64TransportLength(getMaxBase64EncodedLength(maxBytes))
+    : getMaxPercentEncodedLength(maxBytes)
+  if (encodedPayloadLength > encodedCharacterLimit) {
+    throw new FileParserError(
+      'complexity_limit',
+      `Data URI encoded payload exceeds the safe limit for a ${maxBytes}-byte file`
+    )
+  }
+
+  if (isBase64) {
+    const maxBase64EncodedLength = getMaxBase64EncodedLength(maxBytes)
+    const whitespaceAllowance = getBase64WhitespaceAllowance(maxBase64EncodedLength)
+    let base64CharacterCount = 0
+    let whitespaceTransportLength = 0
+    for (let index = payloadStart; index < dataUri.length; index++) {
+      let character = dataUri[index]
+      let transportLength = 1
+      if (character === '%') {
+        const hex = dataUri.slice(index + 1, index + 3)
+        if (!/^[0-9A-Fa-f]{2}$/.test(hex)) {
+          throw new FileParserError('invalid_format', 'Invalid percent-encoded data URI payload')
+        }
+        character = String.fromCharCode(Number.parseInt(hex, 16))
+        transportLength = 3
+        index += 2
+      }
+      if (/\s/.test(character)) {
+        whitespaceTransportLength += transportLength
+        if (whitespaceTransportLength > whitespaceAllowance) {
+          throw new FileParserError(
+            'complexity_limit',
+            `Data URI encoded payload exceeds the safe limit for a ${maxBytes}-byte file`
+          )
+        }
+        continue
+      }
+      base64CharacterCount++
+      if (base64CharacterCount > maxBase64EncodedLength) {
+        throw new FileParserError(
+          'complexity_limit',
+          `Data URI encoded payload exceeds the safe limit for a ${maxBytes}-byte file`
+        )
+      }
+    }
+  }
+
+  const encodedPayload = dataUri.slice(payloadStart)
+
+  let buffer: Buffer
+  if (isBase64) {
+    const compactPayload = encodedPayload
+      .replace(/%([0-9A-Fa-f]{2})/g, (_, hex: string) =>
+        String.fromCharCode(Number.parseInt(hex, 16))
+      )
+      .replace(/\s/g, '')
+    const paddingIndex = compactPayload.indexOf('=')
+    const unpaddedLength = paddingIndex === -1 ? compactPayload.length : paddingIndex
+    const paddingLength = compactPayload.length - unpaddedLength
+    const remainder = unpaddedLength % 4
+    const invalidPadding =
+      paddingLength > 2 ||
+      (paddingLength > 0 && compactPayload.length % 4 !== 0) ||
+      (paddingLength === 1 && remainder !== 3) ||
+      (paddingLength === 2 && remainder !== 2)
+
+    if (!/^[A-Za-z0-9+/]*={0,2}$/.test(compactPayload) || remainder === 1 || invalidPadding) {
+      throw new FileParserError('invalid_format', 'Invalid base64 data URI payload')
+    }
+    const normalizedPayload =
+      paddingLength === 0
+        ? compactPayload.padEnd(compactPayload.length + ((4 - remainder) % 4), '=')
+        : compactPayload
+    buffer = Buffer.from(normalizedPayload, 'base64')
+  } else {
+    const inputBoundedCapacity =
+      encodedPayloadLength > MAX_SAFE_PERCENT_INPUT_BYTES
+        ? Number.MAX_SAFE_INTEGER
+        : encodedPayloadLength * 3 + 1
+    const decoded = Buffer.allocUnsafe(Math.min(maxBytes + 1, inputBoundedCapacity))
+    let decodedLength = 0
+    const appendByte = (byte: number): void => {
+      if (decodedLength <= maxBytes) decoded[decodedLength++] = byte
+    }
+
+    for (let index = 0; index < encodedPayload.length && decodedLength <= maxBytes; index++) {
+      const codeUnit = encodedPayload.charCodeAt(index)
+      if (codeUnit === 0x25) {
+        const hex = encodedPayload.slice(index + 1, index + 3)
+        if (!/^[0-9A-Fa-f]{2}$/.test(hex)) {
+          throw new FileParserError('invalid_format', 'Invalid percent-encoded data URI payload')
+        }
+        appendByte(Number.parseInt(hex, 16))
+        index += 2
+      } else if (codeUnit <= 0x7f) {
+        appendByte(codeUnit)
+      } else {
+        const codePoint = encodedPayload.codePointAt(index)!
+        for (const byte of Buffer.from(String.fromCodePoint(codePoint), 'utf8')) appendByte(byte)
+        if (codePoint > 0xffff) index++
+      }
+    }
+    buffer = decoded.subarray(0, decodedLength)
+  }
+
+  if (buffer.length > maxBytes) {
+    throw new FileParserError(
+      'complexity_limit',
+      `Data URI decoded payload is ${buffer.length} bytes, exceeding the safe limit of ${maxBytes} bytes`
+    )
+  }
+
+  return { buffer, mediaType }
+}

apps/sim/lib/file-parsers/doc-parser.test.ts

@@ -3,6 +3,7 @@
  */
 import JSZip from 'jszip'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { ZipBombError } from '@/lib/file-parsers/ooxml-limits'
 
 const { mockParseOfficeAsync, mockExtractRawText } = vi.hoisted(() => ({
   mockParseOfficeAsync: vi.fn(),
@@ -53,7 +54,7 @@ describe('DocParser.parseBuffer', () => {
   it('rejects a ZIP-shaped .doc whose declared expanded size exceeds the cap', async () => {
     const bomb = await buildDeclaredOversizeArchive(2 * 1024 * 1024 * 1024)
 
-    await expect(new DocParser().parseBuffer(bomb)).rejects.toThrow(/exceeds the maximum allowed/)
+    await expect(new DocParser().parseBuffer(bomb)).rejects.toBeInstanceOf(ZipBombError)
   })
 
   it('rejects the bomb before either decompression library sees the buffer', async () => {

apps/sim/lib/file-parsers/doc-parser.ts

@@ -1,6 +1,7 @@
 import { existsSync } from 'fs'
 import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
@@ -10,21 +11,16 @@ const logger = createLogger('DocParser')
 
 export class DocParser implements FileParser {
   async parseFile(filePath: string): Promise<FileParseResult> {
-    try {
-      if (!filePath) {
-        throw new Error('No file path provided')
-      }
-
-      if (!existsSync(filePath)) {
-        throw new Error(`File not found: ${filePath}`)
-      }
-
-      const buffer = await readFile(filePath)
-      return this.parseBuffer(buffer)
-    } catch (error) {
-      logger.error('DOC file parsing error:', error)
-      throw new Error(`Failed to parse DOC file: ${(error as Error).message}`)
+    if (!filePath) {
+      throw new Error('No file path provided')
     }
+
+    if (!existsSync(filePath)) {
+      throw new Error(`File not found: ${filePath}`)
+    }
+
+    const buffer = await readFile(filePath)
+    return this.parseBuffer(buffer)
   }
 
   /**
@@ -36,13 +32,14 @@ export class DocParser implements FileParser {
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
     try {
       if (!buffer || buffer.length === 0) {
-        throw new Error('Empty buffer provided')
+        throw new FileParserError('empty_input', 'Empty buffer provided')
       }
 
       assertOoxmlArchiveWithinLimits(buffer)
 
+      const parseOfficeAsync = await loadParseOfficeAsync()
+
       try {
-        const parseOfficeAsync = await loadParseOfficeAsync()
         const result = await parseOfficeAsync(buffer)
 
         if (result) {
@@ -85,7 +82,7 @@ export class DocParser implements FileParser {
       return this.fallbackExtraction(buffer)
     } catch (error) {
       logger.error('DOC parsing error:', error)
-      throw new Error(`Failed to parse DOC buffer: ${(error as Error).message}`)
+      throw error
     }
   }
 

apps/sim/lib/file-parsers/docx-parser.ts

@@ -1,6 +1,11 @@
 import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
 import mammoth from 'mammoth'
+import {
+  FileParserError,
+  isEncryptedOfficeParserError,
+  toFileParserError,
+} from '@/lib/file-parsers/errors'
 import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
@@ -20,27 +25,25 @@ interface MammothResult {
 
 export class DocxParser implements FileParser {
   async parseFile(filePath: string): Promise<FileParseResult> {
-    try {
-      if (!filePath) {
-        throw new Error('No file path provided')
-      }
-
-      const buffer = await readFile(filePath)
-      return this.parseBuffer(buffer)
-    } catch (error) {
-      logger.error('DOCX file error:', error)
-      throw new Error(`Failed to parse DOCX file: ${(error as Error).message}`)
+    if (!filePath) {
+      throw new Error('No file path provided')
     }
+
+    const buffer = await readFile(filePath)
+    return this.parseBuffer(buffer)
   }
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
     try {
       if (!buffer || buffer.length === 0) {
-        throw new Error('Empty buffer provided')
+        throw new FileParserError('empty_input', 'Empty buffer provided')
       }
 
       assertOoxmlArchiveWithinLimits(buffer)
 
+      const extractionErrors: unknown[] = []
+      let parserReturnedEmpty = false
+
       try {
         const result = await mammoth.extractRawText({ buffer })
 
@@ -61,12 +64,15 @@ export class DocxParser implements FileParser {
             },
           }
         }
+        parserReturnedEmpty = true
       } catch (mammothError) {
         logger.warn('mammoth failed, trying officeparser:', mammothError)
+        extractionErrors.push(mammothError)
       }
 
+      const parseOfficeAsync = await loadParseOfficeAsync()
+
       try {
-        const parseOfficeAsync = await loadParseOfficeAsync()
         const result = await parseOfficeAsync(buffer)
 
         if (result) {
@@ -83,8 +89,10 @@ export class DocxParser implements FileParser {
             }
           }
         }
+        parserReturnedEmpty = true
       } catch (officeError) {
         logger.warn('officeparser failed:', officeError)
+        extractionErrors.push(officeError)
       }
 
       const isZipFile = buffer.length >= 2 && buffer[0] === 0x50 && buffer[1] === 0x4b
@@ -102,10 +110,30 @@ export class DocxParser implements FileParser {
         }
       }
 
-      throw new Error('Failed to extract text from DOCX file')
+      if (extractionErrors.some(isEncryptedOfficeParserError)) {
+        throw new FileParserError(
+          'encrypted_file',
+          'This document is encrypted or password-protected',
+          new AggregateError(extractionErrors)
+        )
+      }
+
+      if (parserReturnedEmpty) {
+        throw new FileParserError(
+          'no_extractable_text',
+          'No text could be extracted from this DOCX file',
+          extractionErrors.length > 0 ? new AggregateError(extractionErrors) : undefined
+        )
+      }
+
+      throw new FileParserError(
+        'invalid_format',
+        'The DOCX container could not be read',
+        new AggregateError(extractionErrors)
+      )
     } catch (error) {
       logger.error('DOCX parsing error:', error)
-      throw new Error(`Failed to parse DOCX buffer: ${(error as Error).message}`)
+      throw toFileParserError(error, 'invalid_format', 'Failed to parse DOCX buffer')
     }
   }
 }

apps/sim/lib/file-parsers/errors.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from 'vitest'
+import {
+  FileParserError,
+  isEncryptedOfficeParserError,
+  toFileParserError,
+} from '@/lib/file-parsers/errors'
+import { ArchiveIntegrityError, ZipBombError } from '@/lib/file-parsers/ooxml-limits'
+
+describe('file parser errors', () => {
+  it('preserves an archive safety rejection through parser wrappers', () => {
+    const archiveError = new ZipBombError('Archive exceeds the expanded-size limit')
+
+    expect(toFileParserError(archiveError, 'invalid_format', 'DOCX parse failed')).toBe(
+      archiveError
+    )
+  })
+
+  it('preserves an archive integrity rejection through parser wrappers', () => {
+    const archiveError = new ArchiveIntegrityError('Archive entries overlap')
+
+    expect(toFileParserError(archiveError, 'invalid_format', 'DOCX parse failed')).toBe(
+      archiveError
+    )
+  })
+
+  it('preserves an existing typed parser failure', () => {
+    const parserError = new FileParserError('encrypted_file', 'Workbook is protected')
+
+    expect(toFileParserError(parserError, 'invalid_format', 'XLSX parse failed')).toBe(parserError)
+  })
+
+  it('retains an untyped parser-library exception as the cause', () => {
+    const libraryError = new Error('invalid central directory')
+    const parserError = toFileParserError(libraryError, 'invalid_format', 'DOCX parse failed')
+
+    expect(parserError).toBeInstanceOf(FileParserError)
+    expect(parserError.cause).toBe(libraryError)
+  })
+
+  it('bounds and normalizes an untyped parser-library diagnostic', () => {
+    const rawDiagnostic = `first line\nsecond line\u0000${'x'.repeat(1_000)}unbounded-tail`
+    const libraryError = new Error(rawDiagnostic)
+    const parserError = toFileParserError(libraryError, 'invalid_format', 'DOCX parse failed')
+
+    expect(parserError).toBeInstanceOf(FileParserError)
+    expect(parserError.message).not.toMatch(/[\n\u0000]/)
+    expect(parserError.message.length).toBeLessThanOrEqual('DOCX parse failed: '.length + 500)
+    expect(parserError.message).not.toContain('unbounded-tail')
+    expect(parserError.cause).toBe(libraryError)
+  })
+
+  it.each([
+    'File is password-protected',
+    'Password is required to open this workbook',
+    'Encrypted workbook is not supported',
+  ])('recognizes the SheetJS encrypted-workbook error: %s', (message) => {
+    expect(isEncryptedOfficeParserError(new Error(message))).toBe(true)
+  })
+})

apps/sim/lib/file-parsers/errors.ts

@@ -0,0 +1,71 @@
+import { getErrorMessage } from '@sim/utils/errors'
+import { truncate } from '@sim/utils/string'
+import { ArchiveIntegrityError, ZipBombError } from '@/lib/file-parsers/ooxml-limits'
+
+const FILE_PARSER_DIAGNOSTIC_MAX_LENGTH = 500
+
+export const FILE_PARSER_ERROR_CODES = [
+  'empty_input',
+  'unsupported_type',
+  'encrypted_file',
+  'no_extractable_text',
+  'invalid_format',
+  'complexity_limit',
+  'runtime_failure',
+] as const
+
+export type FileParserErrorCode = (typeof FILE_PARSER_ERROR_CODES)[number]
+
+/** A typed failure reported at the untrusted-file parsing boundary. */
+export class FileParserError extends Error {
+  readonly code: FileParserErrorCode
+
+  constructor(code: FileParserErrorCode, message: string, cause?: unknown) {
+    super(message, cause === undefined ? undefined : { cause })
+    this.name = 'FileParserError'
+    this.code = code
+  }
+}
+
+export function isFileParserError(error: unknown): error is FileParserError {
+  return error instanceof FileParserError
+}
+
+/**
+ * Wraps an untyped parser-library exception without erasing a typed inner cause.
+ * Archive safety and integrity failures remain typed so every caller can enforce
+ * the guard without knowing which parser happened to receive the archive.
+ */
+export function toFileParserError(
+  error: unknown,
+  code: FileParserErrorCode,
+  message: string
+): FileParserError | ZipBombError | ArchiveIntegrityError {
+  if (
+    error instanceof ZipBombError ||
+    error instanceof ArchiveIntegrityError ||
+    isFileParserError(error)
+  ) {
+    return error
+  }
+  const diagnostic = truncate(
+    getErrorMessage(error, 'Unknown parser error')
+      .replace(/[\u0000-\u001f\u007f]+/g, ' ')
+      .replace(/\s+/g, ' ')
+      .trim() || 'Unknown parser error',
+    FILE_PARSER_DIAGNOSTIC_MAX_LENGTH,
+    ''
+  )
+  return new FileParserError(code, `${message}: ${diagnostic}`, error)
+}
+
+/**
+ * SheetJS exposes encrypted-workbook failures only through its exception text.
+ * Localizing that adapter-specific check here converts it to a stable code before
+ * it crosses the parser boundary; domain code never needs to match vendor text.
+ */
+export function isEncryptedOfficeParserError(error: unknown): boolean {
+  return /password[- ]protected|password is required|encrypted (?:file|workbook|document)/i.test(
+    getErrorMessage(error)
+  )
+}

apps/sim/lib/file-parsers/html-parser.test.ts

@@ -10,6 +10,12 @@ import { HtmlComplexityError, HtmlParser } from '@/lib/file-parsers/html-parser'
 const parser = new HtmlParser()
 
 describe('HtmlParser', () => {
+  it('reports empty input with the typed parser taxonomy', async () => {
+    await expect(parser.parseBuffer(Buffer.alloc(0))).rejects.toMatchObject({
+      code: 'empty_input',
+    })
+  })
+
   describe('resource limits', () => {
     /**
      * Pinned by value: a 64 MB body aborts the process, so raising the cap

apps/sim/lib/file-parsers/html-parser.ts

@@ -2,6 +2,7 @@ import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import * as cheerio from 'cheerio'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
 
@@ -36,9 +37,9 @@ const MARKUP_TOKEN_BYTE = 0x3c
  * Raised when a document exceeds the limits above, so an input rejected on
  * resource grounds is not reported as a malformed file.
  */
-export class HtmlComplexityError extends Error {
+export class HtmlComplexityError extends FileParserError {
   constructor(message: string) {
-    super(message)
+    super('complexity_limit', message)
     this.name = 'HtmlComplexityError'
   }
 }
@@ -97,6 +98,10 @@ export class HtmlParser implements FileParser {
   }
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
+    if (!buffer || buffer.length === 0) {
+      throw new FileParserError('empty_input', 'Empty buffer provided')
+    }
+
     assertHtmlWithinLimits(buffer)
 
     try {
@@ -164,7 +169,11 @@ export class HtmlParser implements FileParser {
       }
 
       logger.error('HTML buffer parsing error:', error)
-      throw new Error(`Failed to parse HTML buffer: ${getErrorMessage(error, 'Unknown error')}`)
+      throw new FileParserError(
+        'invalid_format',
+        `Failed to parse HTML buffer: ${getErrorMessage(error, 'Unknown error')}`,
+        error
+      )
     }
   }
 

apps/sim/lib/file-parsers/index.ts

@@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger'
 import { CsvParser } from '@/lib/file-parsers/csv-parser'
 import { DocParser } from '@/lib/file-parsers/doc-parser'
 import { DocxParser } from '@/lib/file-parsers/docx-parser'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import { HtmlParser } from '@/lib/file-parsers/html-parser'
 import {
   parseJSON,
@@ -125,7 +126,7 @@ export async function parseFile(filePath: string): Promise<FileParseResult> {
 export async function parseBuffer(buffer: Buffer, extension: string): Promise<FileParseResult> {
   try {
     if (!buffer || buffer.length === 0) {
-      throw new Error('Empty buffer provided')
+      throw new FileParserError('empty_input', 'Empty buffer provided')
     }
 
     if (!extension) {
@@ -138,13 +139,17 @@ export async function parseBuffer(buffer: Buffer, extension: string): Promise<Fi
     const parser = PARSERS.get(normalizedExtension)
 
     if (!parser) {
-      throw new Error(
+      throw new FileParserError(
+        'unsupported_type',
         `Unsupported file type: ${normalizedExtension}. Supported types are: ${SUPPORTED_EXTENSIONS_TEXT}`
       )
     }
 
     if (!parser.parseBuffer) {
-      throw new Error(`Parser for ${normalizedExtension} does not support buffer parsing`)
+      throw new FileParserError(
+        'unsupported_type',
+        `Parser for ${normalizedExtension} does not support buffer parsing`
+      )
     }
 
     return await parser.parseBuffer(buffer)

apps/sim/lib/file-parsers/json-parser.test.ts

@@ -0,0 +1,45 @@
+/**
+ * @vitest-environment node
+ */
+
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { parseJSONBuffer, parseJSONLBuffer } from '@/lib/file-parsers/json-parser'
+
+describe('JSON parser complexity limits', () => {
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('rejects excessive nesting before serializing the parsed value', async () => {
+    const content = `${'['.repeat(501)}0${']'.repeat(501)}`
+    const stringify = vi.spyOn(JSON, 'stringify')
+
+    await expect(parseJSONBuffer(Buffer.from(content))).rejects.toMatchObject({
+      code: 'complexity_limit',
+    })
+    expect(stringify).not.toHaveBeenCalled()
+  })
+
+  it('validates every JSONL item before serializing the aggregate', async () => {
+    const nested = `${'['.repeat(501)}0${']'.repeat(501)}`
+    const stringify = vi.spyOn(JSON, 'stringify')
+
+    await expect(parseJSONLBuffer(Buffer.from(`{}\n${nested}`))).rejects.toMatchObject({
+      code: 'complexity_limit',
+    })
+    expect(stringify).not.toHaveBeenCalled()
+  })
+
+  it('reports the deepest JSONL item instead of only the first', async () => {
+    const result = await parseJSONLBuffer(Buffer.from('{}\n{"nested":{"value":true}}'))
+
+    expect(result.metadata).toMatchObject({ itemCount: 2, depth: 3 })
+  })
+
+  it('preserves ordinary JSON content and metadata', async () => {
+    const result = await parseJSONBuffer(Buffer.from('{"items":[1,2],"name":"test"}'))
+
+    expect(JSON.parse(result.content)).toEqual({ items: [1, 2], name: 'test' })
+    expect(result.metadata).toMatchObject({ isArray: false, keys: ['items', 'name'], depth: 2 })
+  })
+})

apps/sim/lib/file-parsers/json-parser.ts

@@ -1,118 +1,245 @@
 import { getErrorMessage } from '@sim/utils/errors'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import type { FileParseResult } from '@/lib/file-parsers/types'
 
+const MAX_JSON_DEPTH = 500
+const MAX_JSON_NODES = 1_000_000
+const MAX_JSON_SERIALIZED_UNITS = 64 * 1024 * 1024
+
+interface JsonComplexityBudget {
+  nodes: number
+  serializedUnits: number
+}
+
+interface JsonTraversalFrame {
+  value: unknown[] | Record<string, unknown>
+  depth: number
+  index: number
+  keys?: string[]
+}
+
 /**
- * Parse JSON files
+ * Returns the exact number of UTF-16 code units JSON serialization needs for a
+ * string, including quotes and escape expansion, without allocating the result.
  */
-export async function parseJSON(filePath: string): Promise<FileParseResult> {
-  const fs = await import('fs/promises')
-  const content = await fs.readFile(filePath, 'utf-8')
-
-  try {
-    // Parse to validate JSON
-    const jsonData = JSON.parse(content)
-
-    // Return pretty-printed JSON for better readability
-    const formattedContent = JSON.stringify(jsonData, null, 2)
-
-    // Extract metadata about the JSON structure
-    const metadata = {
-      type: 'json',
-      isArray: Array.isArray(jsonData),
-      keys: Array.isArray(jsonData) ? [] : Object.keys(jsonData),
-      itemCount: Array.isArray(jsonData) ? jsonData.length : undefined,
-      depth: getJsonDepth(jsonData),
+function serializedJsonStringLength(value: string): number {
+  let length = 2
+  for (let index = 0; index < value.length; index++) {
+    const code = value.charCodeAt(index)
+    if (code === 0x22 || code === 0x5c) {
+      length += 2
+    } else if (code < 0x20) {
+      length +=
+        code === 0x08 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d ? 2 : 6
+    } else if (code >= 0xd800 && code <= 0xdfff) {
+      const next = index + 1 < value.length ? value.charCodeAt(index + 1) : 0
+      if (code <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) {
+        length += 2
+        index++
+      } else {
+        length += 6
+      }
+    } else {
+      length++
     }
+  }
+  return length
+}
 
-    return {
-      content: formattedContent,
-      metadata,
-    }
-  } catch (error) {
-    throw new Error(`Invalid JSON: ${getErrorMessage(error, 'Unknown error')}`)
+function estimateJsonValueUnits(value: unknown, depth: number): number {
+  const formattingOverhead = depth * 2 + 4
+  if (typeof value === 'string') {
+    return formattingOverhead + serializedJsonStringLength(value)
+  }
+  if (typeof value === 'number') return formattingOverhead + String(value).length
+  if (typeof value === 'boolean') return formattingOverhead + (value ? 4 : 5)
+  if (value === null) return formattingOverhead + 4
+  return formattingOverhead + 2
+}
+
+function chargeJsonComplexity(
+  budget: JsonComplexityBudget,
+  value: unknown,
+  depth: number,
+  key?: string
+): void {
+  budget.nodes++
+  if (budget.nodes > MAX_JSON_NODES) {
+    throw new FileParserError(
+      'complexity_limit',
+      `JSON document exceeds the maximum of ${MAX_JSON_NODES.toLocaleString()} values`
+    )
+  }
+
+  budget.serializedUnits +=
+    estimateJsonValueUnits(value, depth) +
+    (key === undefined ? 0 : serializedJsonStringLength(key) + 2)
+  if (budget.serializedUnits > MAX_JSON_SERIALIZED_UNITS) {
+    throw new FileParserError(
+      'complexity_limit',
+      `JSON document expands beyond the maximum serialized size of ${MAX_JSON_SERIALIZED_UNITS} characters`
+    )
   }
 }
 
 /**
- * Parse JSON from buffer
+ * Iteratively validates a parsed JSON value before pretty-printing it. The
+ * traversal keeps only one frame per nesting level and avoids recursively
+ * materializing child-value arrays while trying to reject the document.
  */
-export async function parseJSONBuffer(buffer: Buffer): Promise<FileParseResult> {
-  const content = buffer.toString('utf-8')
+function assertJsonValueWithinLimits(
+  root: unknown,
+  budget: JsonComplexityBudget,
+  initialDepth = 0
+): number {
+  let maxDepth = initialDepth
+  const stack: JsonTraversalFrame[] = []
 
-  try {
-    const jsonData = JSON.parse(content)
-    const formattedContent = JSON.stringify(jsonData, null, 2)
-
-    const metadata = {
-      type: 'json',
-      isArray: Array.isArray(jsonData),
-      keys: Array.isArray(jsonData) ? [] : Object.keys(jsonData),
-      itemCount: Array.isArray(jsonData) ? jsonData.length : undefined,
-      depth: getJsonDepth(jsonData),
+  const enterValue = (value: unknown, depth: number, key?: string): void => {
+    chargeJsonComplexity(budget, value, depth, key)
+    if (value === null || typeof value !== 'object') return
+    if (depth >= MAX_JSON_DEPTH) {
+      throw new FileParserError(
+        'complexity_limit',
+        `JSON document exceeds the maximum nesting depth of ${MAX_JSON_DEPTH}`
+      )
     }
 
-    return {
-      content: formattedContent,
-      metadata,
-    }
-  } catch (error) {
-    throw new Error(`Invalid JSON: ${getErrorMessage(error, 'Unknown error')}`)
-  }
-}
-
-/**
- * Parse JSONL (JSON Lines) files — one JSON object per line
- */
-export async function parseJSONL(filePath: string): Promise<FileParseResult> {
-  const fs = await import('fs/promises')
-  const content = await fs.readFile(filePath, 'utf-8')
-  return parseJSONLContent(content)
-}
-
-/**
- * Parse JSONL from buffer
- */
-export async function parseJSONLBuffer(buffer: Buffer): Promise<FileParseResult> {
-  const content = buffer.toString('utf-8')
-  return parseJSONLContent(content)
-}
-
-function parseJSONLContent(content: string): FileParseResult {
-  const lines = content.split('\n').filter((line) => line.trim())
-  const items: unknown[] = []
-
-  for (const line of lines) {
-    try {
-      items.push(JSON.parse(line))
-    } catch {
-      throw new Error(`Invalid JSONL: failed to parse line: ${line.slice(0, 100)}`)
+    maxDepth = Math.max(maxDepth, depth + 1)
+    if (Array.isArray(value)) {
+      stack.push({ value, depth, index: 0 })
+    } else {
+      const record = value as Record<string, unknown>
+      stack.push({ value: record, depth, index: 0, keys: Object.keys(record) })
     }
   }
 
-  const formattedContent = JSON.stringify(items, null, 2)
+  enterValue(root, initialDepth)
+  while (stack.length > 0) {
+    const frame = stack.at(-1)!
+    if (Array.isArray(frame.value)) {
+      if (frame.index >= frame.value.length) {
+        stack.pop()
+        continue
+      }
+      const child = frame.value[frame.index]
+      frame.index++
+      enterValue(child, frame.depth + 1)
+      continue
+    }
+
+    const keys = frame.keys!
+    if (frame.index >= keys.length) {
+      stack.pop()
+      continue
+    }
+    const key = keys[frame.index]
+    frame.index++
+    enterValue(frame.value[key], frame.depth + 1, key)
+  }
+
+  return maxDepth
+}
+
+function buildJsonResult(jsonData: unknown): FileParseResult {
+  const budget = { nodes: 0, serializedUnits: 0 }
+  const depth = assertJsonValueWithinLimits(jsonData, budget)
+  const formattedContent = JSON.stringify(jsonData, null, 2)
+  const isArray = Array.isArray(jsonData)
+  const isRecord = jsonData !== null && typeof jsonData === 'object' && !isArray
 
   return {
     content: formattedContent,
     metadata: {
       type: 'json',
-      isArray: true,
-      keys: [],
-      itemCount: items.length,
-      depth: items.length > 0 ? 1 + getJsonDepth(items[0]) : 1,
+      isArray,
+      keys: isRecord ? Object.keys(jsonData as Record<string, unknown>) : [],
+      itemCount: isArray ? jsonData.length : undefined,
+      depth,
     },
   }
 }
 
-/**
- * Calculate the depth of a JSON object
- */
-function getJsonDepth(obj: any): number {
-  if (obj === null || typeof obj !== 'object') return 0
+function parseJsonContent(content: string): FileParseResult {
+  try {
+    return buildJsonResult(JSON.parse(content))
+  } catch (error) {
+    if (error instanceof FileParserError) throw error
+    if (!(error instanceof SyntaxError)) {
+      throw new FileParserError('runtime_failure', 'JSON processing failed unexpectedly', error)
+    }
+    throw new FileParserError(
+      'invalid_format',
+      `Invalid JSON: ${getErrorMessage(error, 'Unknown error')}`,
+      error
+    )
+  }
+}
 
-  if (Array.isArray(obj)) {
-    return obj.length > 0 ? 1 + Math.max(...obj.map(getJsonDepth)) : 1
+/** Parse a JSON file. */
+export async function parseJSON(filePath: string): Promise<FileParseResult> {
+  const fs = await import('fs/promises')
+  return parseJsonContent(await fs.readFile(filePath, 'utf-8'))
+}
+
+/** Parse JSON from a buffer. */
+export async function parseJSONBuffer(buffer: Buffer): Promise<FileParseResult> {
+  return parseJsonContent(buffer.toString('utf-8'))
+}
+
+function* iterateJsonLines(content: string): Generator<{ line: string; lineNumber: number }> {
+  let lineStart = 0
+  let lineNumber = 1
+  while (lineStart <= content.length) {
+    const newline = content.indexOf('\n', lineStart)
+    const lineEnd = newline === -1 ? content.length : newline
+    const line = content.slice(lineStart, lineEnd)
+    if (line.trim()) yield { line, lineNumber }
+    if (newline === -1) break
+    lineStart = newline + 1
+    lineNumber++
+  }
+}
+
+function parseJsonLinesContent(content: string): FileParseResult {
+  const items: unknown[] = []
+  const budget = { nodes: 0, serializedUnits: 0 }
+  let depth = assertJsonValueWithinLimits([], budget)
+
+  for (const { line, lineNumber } of iterateJsonLines(content)) {
+    let item: unknown
+    try {
+      item = JSON.parse(line)
+    } catch (error) {
+      throw new FileParserError(
+        'invalid_format',
+        `Invalid JSONL on line ${lineNumber}: ${line.slice(0, 100)}`,
+        error
+      )
+    }
+    depth = Math.max(depth, assertJsonValueWithinLimits(item, budget, 1))
+    items.push(item)
   }
 
-  const depths = Object.values(obj).map(getJsonDepth)
-  return depths.length > 0 ? 1 + Math.max(...depths) : 1
+  return {
+    content: JSON.stringify(items, null, 2),
+    metadata: {
+      type: 'json',
+      isArray: true,
+      keys: [],
+      itemCount: items.length,
+      depth,
+    },
+  }
+}
+
+/** Parse a JSON Lines file. */
+export async function parseJSONL(filePath: string): Promise<FileParseResult> {
+  const fs = await import('fs/promises')
+  return parseJsonLinesContent(await fs.readFile(filePath, 'utf-8'))
+}
+
+/** Parse JSON Lines from a buffer. */
+export async function parseJSONLBuffer(buffer: Buffer): Promise<FileParseResult> {
+  return parseJsonLinesContent(buffer.toString('utf-8'))
 }

apps/sim/lib/file-parsers/officeparser-module.ts

@@ -1,3 +1,5 @@
+import { FileParserError } from '@/lib/file-parsers/errors'
+
 /** `officeparser`'s single entry point, as every parser here calls it. */
 type ParseOfficeAsync = (input: Buffer) => Promise<string>
 
@@ -46,5 +48,13 @@ export function resolveParseOfficeAsync(mod: OfficeParserModule): ParseOfficeAsy
  * can only assert the shape that already worked.
  */
 export async function loadParseOfficeAsync(): Promise<ParseOfficeAsync> {
-  return resolveParseOfficeAsync((await import('officeparser')) as OfficeParserModule)
+  try {
+    return resolveParseOfficeAsync((await import('officeparser')) as OfficeParserModule)
+  } catch (error) {
+    throw new FileParserError(
+      'runtime_failure',
+      'The officeparser runtime could not be loaded',
+      error
+    )
+  }
 }

apps/sim/lib/file-parsers/ooxml-limits.ts

@@ -15,9 +15,23 @@ export const MAX_OOXML_TOTAL_UNCOMPRESSED_BYTES = 150 * 1024 * 1024
 /** Hard ceiling on any single entry's declared uncompressed size. */
 export const MAX_OOXML_ENTRY_UNCOMPRESSED_BYTES = 64 * 1024 * 1024
 
+/** Bounds the per-entry object graph created by OOXML ZIP parsers. */
+export const MAX_OOXML_CENTRAL_DIRECTORY_RECORDS = 10_000
+
+/** Bounds retained central-directory extra-field metadata before ZIP parsing. */
+export const MAX_OOXML_CENTRAL_DIRECTORY_EXTRA_BYTES = 4 * 1024 * 1024
+
 export class ZipBombError extends Error {
   constructor(message: string) {
     super(message)
     this.name = 'ZipBombError'
   }
 }
+
+/** A ZIP-shaped file whose structure cannot be verified safely or parsed consistently. */
+export class ArchiveIntegrityError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'ArchiveIntegrityError'
+  }
+}

apps/sim/lib/file-parsers/opendocument-parser.ts

@@ -1,6 +1,7 @@
 import { existsSync } from 'fs'
 import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
+import { FileParserError, isEncryptedOfficeParserError } from '@/lib/file-parsers/errors'
 import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
@@ -38,7 +39,7 @@ export class OpenDocumentParser implements FileParser {
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
     if (!buffer || buffer.length === 0) {
-      throw new Error('Empty buffer provided')
+      throw new FileParserError('empty_input', 'Empty buffer provided')
     }
 
     /**
@@ -55,12 +56,22 @@ export class OpenDocumentParser implements FileParser {
       extracted = typeof result === 'string' ? result : ''
     } catch (error) {
       logger.error('OpenDocument parsing failed', { error: (error as Error).message })
-      throw new Error(`Failed to parse OpenDocument file: ${(error as Error).message}`)
+      if (isEncryptedOfficeParserError(error)) {
+        throw new FileParserError(
+          'encrypted_file',
+          'This OpenDocument file is encrypted or password-protected',
+          error
+        )
+      }
+      throw new FileParserError('invalid_format', 'Failed to parse OpenDocument file', error)
     }
 
     const content = sanitizeTextForUTF8(extracted.trim())
     if (!content) {
-      throw new Error('Failed to extract text from OpenDocument file')
+      throw new FileParserError(
+        'no_extractable_text',
+        'No text could be extracted from this OpenDocument file'
+      )
     }
 
     return {

apps/sim/lib/file-parsers/parser-formats.test.ts

@@ -14,6 +14,7 @@ import * as XLSX from 'xlsx'
 import { parseBuffer } from '@/lib/file-parsers'
 import { DocParser } from '@/lib/file-parsers/doc-parser'
 import { DocxParser } from '@/lib/file-parsers/docx-parser'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import { OpenDocumentParser } from '@/lib/file-parsers/opendocument-parser'
 import { PptxParser } from '@/lib/file-parsers/pptx-parser'
 import { XlsxParser } from '@/lib/file-parsers/xlsx-parser'
@@ -184,6 +185,15 @@ describe('DocxParser', () => {
     expect(result.metadata?.degraded).toBeFalsy()
   })
 
+  it('reports a valid image-only or empty Word container as no extractable text', async () => {
+    const buffer = await buildDocx('<w:p><w:r><w:drawing/></w:r></w:p>')
+
+    const error = await new DocxParser().parseBuffer(buffer).catch((caught: unknown) => caught)
+
+    expect(error).toBeInstanceOf(FileParserError)
+    expect(error).toMatchObject({ code: 'no_extractable_text' })
+  })
+
   /**
    * A macro-enabled `.docm` is the same WordprocessingML package with a different
    * main-part content type. mammoth reads `word/document.xml` without consulting

apps/sim/lib/file-parsers/pptx-parser.test.ts

@@ -0,0 +1,30 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+
+const { mockParseOfficeAsync } = vi.hoisted(() => ({
+  mockParseOfficeAsync: vi.fn(),
+}))
+
+vi.mock('@/lib/file-parsers/officeparser-module', () => ({
+  loadParseOfficeAsync: vi.fn(async () => mockParseOfficeAsync),
+}))
+
+import type { FileParserError } from '@/lib/file-parsers/errors'
+import { PptxParser } from '@/lib/file-parsers/pptx-parser'
+
+describe('PptxParser', () => {
+  it('classifies encrypted legacy presentations before degraded extraction', async () => {
+    const libraryError = new Error('File is password-protected')
+    mockParseOfficeAsync.mockRejectedValueOnce(libraryError)
+    const legacyOleBuffer = Buffer.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1])
+
+    const result = new PptxParser().parseBuffer(legacyOleBuffer)
+
+    await expect(result).rejects.toMatchObject<FileParserError>({
+      code: 'encrypted_file',
+      cause: libraryError,
+    })
+  })
+})

apps/sim/lib/file-parsers/pptx-parser.ts

@@ -1,6 +1,7 @@
 import { existsSync } from 'fs'
 import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
+import { FileParserError, isEncryptedOfficeParserError } from '@/lib/file-parsers/errors'
 import { loadParseOfficeAsync } from '@/lib/file-parsers/officeparser-module'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
@@ -10,68 +11,69 @@ const logger = createLogger('PptxParser')
 
 export class PptxParser implements FileParser {
   async parseFile(filePath: string): Promise<FileParseResult> {
-    try {
-      if (!filePath) {
-        throw new Error('No file path provided')
-      }
-
-      if (!existsSync(filePath)) {
-        throw new Error(`File not found: ${filePath}`)
-      }
-
-      logger.info(`Parsing PowerPoint file: ${filePath}`)
-
-      const buffer = await readFile(filePath)
-      return this.parseBuffer(buffer)
-    } catch (error) {
-      logger.error('PowerPoint file parsing error:', error)
-      throw new Error(`Failed to parse PowerPoint file: ${(error as Error).message}`)
+    if (!filePath) {
+      throw new Error('No file path provided')
     }
+
+    if (!existsSync(filePath)) {
+      throw new Error(`File not found: ${filePath}`)
+    }
+
+    logger.info(`Parsing PowerPoint file: ${filePath}`)
+
+    const buffer = await readFile(filePath)
+    return this.parseBuffer(buffer)
   }
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
+    logger.info('Parsing PowerPoint buffer, size:', buffer.length)
+
+    if (!buffer || buffer.length === 0) {
+      throw new FileParserError('empty_input', 'Empty buffer provided')
+    }
+
+    assertOoxmlArchiveWithinLimits(buffer)
+
+    const parseOfficeAsync = await loadParseOfficeAsync()
+
     try {
-      logger.info('Parsing PowerPoint buffer, size:', buffer.length)
+      const result = await parseOfficeAsync(buffer)
 
-      if (!buffer || buffer.length === 0) {
-        throw new Error('Empty buffer provided')
-      }
-
-      assertOoxmlArchiveWithinLimits(buffer)
-
-      let parseOfficeAsync
-      try {
-        parseOfficeAsync = await loadParseOfficeAsync()
-      } catch (importError) {
-        logger.warn('officeparser not available, using fallback extraction')
+      if (!result || typeof result !== 'string') {
         return this.fallbackExtraction(buffer)
       }
 
-      try {
-        const result = await parseOfficeAsync(buffer)
+      const content = sanitizeTextForUTF8(result.trim())
 
-        if (!result || typeof result !== 'string') {
-          throw new Error('officeparser returned invalid result')
-        }
+      logger.info('PowerPoint parsing completed successfully with officeparser')
 
-        const content = sanitizeTextForUTF8(result.trim())
+      return {
+        content: content,
+        metadata: {
+          characterCount: content.length,
+          extractionMethod: 'officeparser',
+        },
+      }
+    } catch (extractError) {
+      if (isEncryptedOfficeParserError(extractError)) {
+        throw new FileParserError(
+          'encrypted_file',
+          'This presentation is encrypted or password-protected',
+          extractError
+        )
+      }
 
-        logger.info('PowerPoint parsing completed successfully with officeparser')
-
-        return {
-          content: content,
-          metadata: {
-            characterCount: content.length,
-            extractionMethod: 'officeparser',
-          },
-        }
-      } catch (extractError) {
-        logger.warn('officeparser failed, using fallback:', extractError)
+      const isZipFile = buffer.length >= 2 && buffer[0] === 0x50 && buffer[1] === 0x4b
+      if (!isZipFile) {
+        logger.warn('officeparser failed for legacy PowerPoint, using fallback:', extractError)
         return this.fallbackExtraction(buffer)
       }
-    } catch (error) {
-      logger.error('PowerPoint buffer parsing error:', error)
-      throw new Error(`Failed to parse PowerPoint buffer: ${(error as Error).message}`)
+
+      throw new FileParserError(
+        'invalid_format',
+        'The PowerPoint container could not be read',
+        extractError
+      )
     }
   }
 

apps/sim/lib/file-parsers/xlsx-parser.ts

@@ -3,6 +3,11 @@ import { readFile } from 'fs/promises'
 import { createLogger } from '@sim/logger'
 import { truncate } from '@sim/utils/string'
 import * as XLSX from 'xlsx'
+import {
+  FileParserError,
+  isEncryptedOfficeParserError,
+  toFileParserError,
+} from '@/lib/file-parsers/errors'
 import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
 import { sanitizeTextForUTF8, truncationNotice } from '@/lib/file-parsers/utils'
 import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard'
@@ -12,35 +17,48 @@ const logger = createLogger('XlsxParser')
 // Configuration for handling large XLSX files
 const CONFIG = {
   MAX_PREVIEW_ROWS: 1000, // Only keep first 1000 rows for preview
+  MAX_PREVIEW_COLUMNS: 256,
   MAX_SAMPLE_ROWS: 100, // Sample for metadata
-  ROWS_PER_CHUNK: 50, // Aggregate 50 rows per chunk to reduce chunk count
-  MAX_CELL_LENGTH: 1000, // Truncate very long cell values
+  MAX_SAMPLE_COLUMNS: 32,
+  MAX_SAMPLE_CELL_LENGTH: 256,
+  MAX_SAMPLE_CHARACTERS: 1024 * 1024,
   MAX_CONTENT_SIZE: 10 * 1024 * 1024, // 10MB max content size
 }
 
+const CONTENT_LIMIT_NOTICE = truncationNotice('Content truncated due to size limits')
+
+function sliceUtf8WithinByteLimit(value: string, maxBytes: number): string {
+  if (maxBytes <= 0) return ''
+  if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value
+
+  let low = 0
+  let high = value.length
+  while (low < high) {
+    const middle = Math.ceil((low + high) / 2)
+    if (Buffer.byteLength(value.slice(0, middle), 'utf8') <= maxBytes) low = middle
+    else high = middle - 1
+  }
+  return value.slice(0, low)
+}
+
 export class XlsxParser implements FileParser {
   /**
    * Read the file into a buffer and delegate to {@link parseBuffer} so the
    * decompression-bomb guard runs before SheetJS inflates the workbook.
    */
   async parseFile(filePath: string): Promise<FileParseResult> {
-    try {
-      if (!filePath) {
-        throw new Error('No file path provided')
-      }
-
-      if (!existsSync(filePath)) {
-        throw new Error(`File not found: ${filePath}`)
-      }
-
-      logger.info(`Parsing XLSX file: ${filePath}`)
-
-      const buffer = await readFile(filePath)
-      return this.parseBuffer(buffer)
-    } catch (error) {
-      logger.error('XLSX file parsing error:', error)
-      throw new Error(`Failed to parse XLSX file: ${(error as Error).message}`)
+    if (!filePath) {
+      throw new Error('No file path provided')
     }
+
+    if (!existsSync(filePath)) {
+      throw new Error(`File not found: ${filePath}`)
+    }
+
+    logger.info(`Parsing XLSX file: ${filePath}`)
+
+    const buffer = await readFile(filePath)
+    return this.parseBuffer(buffer)
   }
 
   async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
@@ -51,7 +69,7 @@ export class XlsxParser implements FileParser {
       )
 
       if (!buffer || buffer.length === 0) {
-        throw new Error('Empty buffer provided')
+        throw new FileParserError('empty_input', 'Empty buffer provided')
       }
 
       assertOoxmlArchiveWithinLimits(buffer)
@@ -65,7 +83,14 @@ export class XlsxParser implements FileParser {
       return this.processWorkbook(workbook)
     } catch (error) {
       logger.error('XLSX buffer parsing error:', error)
-      throw new Error(`Failed to parse XLSX buffer: ${(error as Error).message}`)
+      if (isEncryptedOfficeParserError(error)) {
+        throw new FileParserError(
+          'encrypted_file',
+          'This workbook is encrypted or password-protected',
+          error
+        )
+      }
+      throw toFileParserError(error, 'invalid_format', 'Failed to parse XLSX buffer')
     }
   }
 
@@ -75,7 +100,36 @@ export class XlsxParser implements FileParser {
     let totalRows = 0
     let truncated = false
     let contentSize = 0
-    const sampledData: any[] = []
+    let hasMeaningfulCellContent = false
+    let outputLimitReached = false
+    let sampleCharacters = 0
+    const sampledData: unknown[][] = []
+
+    const appendContent = (value: string): boolean => {
+      const remaining = CONFIG.MAX_CONTENT_SIZE - contentSize
+      const valueBytes = Buffer.byteLength(value, 'utf8')
+      if (valueBytes <= remaining) {
+        content += value
+        contentSize += valueBytes
+        return true
+      }
+
+      const noticeBytes = Buffer.byteLength(CONTENT_LIMIT_NOTICE, 'utf8')
+      const retainedContent = sliceUtf8WithinByteLimit(
+        content,
+        Math.max(0, CONFIG.MAX_CONTENT_SIZE - noticeBytes)
+      )
+      const retainedBytes = Buffer.byteLength(retainedContent, 'utf8')
+      const prefix = sliceUtf8WithinByteLimit(
+        value,
+        Math.max(0, CONFIG.MAX_CONTENT_SIZE - retainedBytes - noticeBytes)
+      )
+      content = retainedContent + prefix + CONTENT_LIMIT_NOTICE
+      contentSize = Buffer.byteLength(content, 'utf8')
+      truncated = true
+      outputLimitReached = true
+      return false
+    }
 
     for (const sheetName of sheetNames) {
       const worksheet = workbook.Sheets[sheetName]
@@ -83,6 +137,7 @@ export class XlsxParser implements FileParser {
       // Get sheet dimensions
       const range = XLSX.utils.decode_range(worksheet['!ref'] || 'A1')
       const rowCount = range.e.r - range.s.r + 1
+      const columnCount = range.e.c - range.s.c + 1
 
       logger.info(`Processing sheet: ${sheetName} with ${rowCount} rows`)
 
@@ -106,10 +161,14 @@ export class XlsxParser implements FileParser {
        * defeated the `blankrows: false` beside it.
        */
       const lastPreviewRow = Math.min(range.e.r, range.s.r + CONFIG.MAX_PREVIEW_ROWS - 1)
-      const sheetData = XLSX.utils.sheet_to_json(worksheet, {
+      const lastPreviewColumn = Math.min(range.e.c, range.s.c + CONFIG.MAX_PREVIEW_COLUMNS - 1)
+      const sheetData = XLSX.utils.sheet_to_json<unknown[]>(worksheet, {
         header: 1,
         blankrows: false, // Skip blank rows
-        range: { s: { r: range.s.r, c: range.s.c }, e: { r: lastPreviewRow, c: range.e.c } },
+        range: {
+          s: { r: range.s.r, c: range.s.c },
+          e: { r: lastPreviewRow, c: lastPreviewColumn },
+        },
       })
 
       // Reported from the declared range, as before, so bounding the conversion
@@ -117,10 +176,34 @@ export class XlsxParser implements FileParser {
       const actualRowCount = sheetData.length
       totalRows += rowCount
 
+      for (const row of sheetData) {
+        if (row.some((cell) => this.truncateCell(cell).trim().length > 0)) {
+          hasMeaningfulCellContent = true
+          break
+        }
+      }
+
       // Store limited sample for metadata
       if (sampledData.length < CONFIG.MAX_SAMPLE_ROWS) {
         const sampleSize = Math.min(CONFIG.MAX_SAMPLE_ROWS - sampledData.length, actualRowCount)
-        sampledData.push(...sheetData.slice(0, sampleSize))
+        for (const row of sheetData.slice(0, sampleSize)) {
+          const sampleRow: string[] = []
+
+          for (const cell of row.slice(0, CONFIG.MAX_SAMPLE_COLUMNS)) {
+            const remaining = CONFIG.MAX_SAMPLE_CHARACTERS - sampleCharacters
+            if (remaining <= 0) break
+
+            const value = this.truncateCell(
+              cell,
+              Math.min(CONFIG.MAX_SAMPLE_CELL_LENGTH, remaining)
+            )
+            sampleRow.push(value)
+            sampleCharacters += value.length
+          }
+
+          if (sampleRow.length > 0) sampledData.push(sampleRow)
+          if (sampleCharacters >= CONFIG.MAX_SAMPLE_CHARACTERS) break
+        }
       }
 
       // Already bounded by the conversion window above.
@@ -129,53 +212,28 @@ export class XlsxParser implements FileParser {
 
       // Add sheet header
       const sheetHeader = `\n=== Sheet: ${cleanSheetName} ===\n`
-      content += sheetHeader
-      contentSize += sheetHeader.length
+      if (!appendContent(sheetHeader)) break
 
       if (actualRowCount > 0) {
         // Get headers if available
-        const headers = sheetData[0] as any[]
+        const headers = sheetData[0]
         if (headers && headers.length > 0) {
           const headerRow = headers.map((h) => this.truncateCell(h)).join('\t')
-          content += `${headerRow}\n`
-          content += `${'-'.repeat(Math.min(80, headerRow.length))}\n`
-          contentSize += headerRow.length + 82
-        }
-
-        // Process data rows in chunks
-        let chunkContent = ''
-        let chunkRowCount = 0
-
-        for (let i = 1; i < rowsToProcess; i++) {
-          const row = sheetData[i] as any[]
-          if (row && row.length > 0) {
-            const rowString = row.map((cell) => this.truncateCell(cell)).join('\t')
-
-            chunkContent += `${rowString}\n`
-            chunkRowCount++
-
-            // Add chunk separator every N rows for better readability
-            if (chunkRowCount >= CONFIG.ROWS_PER_CHUNK) {
-              content += chunkContent
-              contentSize += chunkContent.length
-              chunkContent = ''
-              chunkRowCount = 0
-
-              // Check content size limit
-              if (contentSize > CONFIG.MAX_CONTENT_SIZE) {
-                truncated = true
-                break
-              }
-            }
+          if (!appendContent(`${headerRow}\n${'-'.repeat(Math.min(80, headerRow.length))}\n`)) {
+            break
           }
         }
 
-        // Add remaining chunk content
-        if (chunkContent && contentSize < CONFIG.MAX_CONTENT_SIZE) {
-          content += chunkContent
-          contentSize += chunkContent.length
+        for (let i = 1; i < rowsToProcess; i++) {
+          const row = sheetData[i]
+          if (row && row.length > 0) {
+            const rowString = row.map((cell) => this.truncateCell(cell)).join('\t')
+            if (!appendContent(`${rowString}\n`)) break
+          }
         }
 
+        if (outputLimitReached) break
+
         /**
          * Truncated means the WINDOW cut the sheet short, which is a question
          * about the declared range against the cap — not about how many rows
@@ -186,21 +244,26 @@ export class XlsxParser implements FileParser {
          * rows, since those are now skipped. The CSV parser asks the same
          * question the same way.
          */
-        if (rowCount > CONFIG.MAX_PREVIEW_ROWS) {
-          content += truncationNotice(
-            `${rowCount.toLocaleString()} total rows, showing first ${rowsToProcess.toLocaleString()}`
+        if (rowCount > CONFIG.MAX_PREVIEW_ROWS || columnCount > CONFIG.MAX_PREVIEW_COLUMNS) {
+          const rowSummary = `${rowCount.toLocaleString()} total rows, showing first ${rowsToProcess.toLocaleString()}`
+          const columnSummary = `${columnCount.toLocaleString()} total columns, showing first ${Math.min(columnCount, CONFIG.MAX_PREVIEW_COLUMNS).toLocaleString()}`
+          appendContent(
+            truncationNotice(
+              [
+                rowCount > CONFIG.MAX_PREVIEW_ROWS ? rowSummary : null,
+                columnCount > CONFIG.MAX_PREVIEW_COLUMNS ? columnSummary : null,
+              ]
+                .filter((summary): summary is string => summary !== null)
+                .join('; ')
+            )
           )
           truncated = true
         }
       } else {
-        content += '[Empty sheet]\n'
+        appendContent('[Empty sheet]\n')
       }
 
-      if (contentSize > CONFIG.MAX_CONTENT_SIZE) {
-        content += truncationNotice('Content truncated due to size limits')
-        truncated = true
-        break
-      }
+      if (outputLimitReached) break
     }
 
     logger.info(
@@ -216,21 +279,27 @@ export class XlsxParser implements FileParser {
         sheetNames: sheetNames,
         totalRows: totalRows,
         truncated: truncated,
+        degraded: !hasMeaningfulCellContent,
         sampledData: sampledData.slice(0, CONFIG.MAX_SAMPLE_ROWS),
         contentSize: contentSize,
       },
     }
   }
 
-  private truncateCell(cell: any): string {
+  private truncateCell(cell: unknown, maxLength?: number): string {
     if (cell === null || cell === undefined) {
       return ''
     }
 
     let cellStr = String(cell)
 
-    // Truncate very long cells
-    cellStr = truncate(cellStr, CONFIG.MAX_CELL_LENGTH)
+    /**
+     * Samples are previews; canonical content is bounded only by the aggregate
+     * output ceiling so an otherwise valid long cell is never silently partial.
+     */
+    if (maxLength !== undefined && cellStr.length > maxLength) {
+      cellStr = truncate(cellStr, Math.max(0, maxLength - 3))
+    }
 
     return sanitizeTextForUTF8(cellStr)
   }

apps/sim/lib/file-parsers/xlsx-preview-bound.test.ts

@@ -34,7 +34,7 @@ describe('XlsxParser preview bound', () => {
 
     expect(toJson).toHaveBeenCalled()
     const options = toJson.mock.calls[0][1] as {
-      range?: { s: { r: number }; e: { r: number } }
+      range?: { s: { r: number; c: number }; e: { r: number; c: number } }
       defval?: unknown
     }
 
@@ -49,6 +49,11 @@ describe('XlsxParser preview bound', () => {
       (options.range as { s: { r: number }; e: { r: number } }).s.r +
       1
     expect(rowsRequested).toBeLessThanOrEqual(1000)
+    const columnsRequested =
+      (options.range as { s: { c: number }; e: { c: number } }).e.c -
+      (options.range as { s: { c: number }; e: { c: number } }).s.c +
+      1
+    expect(columnsRequested).toBeLessThanOrEqual(256)
 
     /**
      * `defval` made every cell in the range materialize, so allocation scaled
@@ -58,6 +63,43 @@ describe('XlsxParser preview bound', () => {
     expect(options.defval).toBeUndefined()
   })
 
+  it('caps a sheet with an inflated declared column range before conversion', async () => {
+    const sheet = XLSX.utils.aoa_to_sheet([['header'], ['value']])
+    sheet['!ref'] = 'A1:XFD2'
+    const book = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(book, sheet, 'Wide')
+    const buffer = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+    const toJson = vi.spyOn(XLSX.utils, 'sheet_to_json')
+
+    const result = await new XlsxParser().parseBuffer(buffer)
+    const options = toJson.mock.calls[0][1] as {
+      range: { s: { c: number }; e: { c: number } }
+    }
+
+    expect(options.range.e.c - options.range.s.c + 1).toBe(256)
+    expect(result.metadata?.truncated).toBe(true)
+    expect(result.content).toContain('16,384 total columns')
+  })
+
+  it('hard-caps rendered content and bounds sampled metadata independently', async () => {
+    const repeatedCell = 'x'.repeat(500)
+    const rows = Array.from({ length: 100 }, () => Array.from({ length: 256 }, () => repeatedCell))
+    const sheet = XLSX.utils.aoa_to_sheet(rows)
+    const book = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(book, sheet, 'Dense')
+    const buffer = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+
+    const result = await new XlsxParser().parseBuffer(buffer)
+    const sampledData = result.metadata?.sampledData as string[][]
+    const sampledCharacters = sampledData.flat().reduce((sum, value) => sum + value.length, 0)
+
+    expect(result.metadata?.contentSize).toBeLessThanOrEqual(10 * 1024 * 1024)
+    expect(result.content.length).toBeLessThan(10 * 1024 * 1024 + 200)
+    expect(sampledData.every((row) => row.length <= 32)).toBe(true)
+    expect(sampledData.flat().every((value) => value.length <= 256)).toBe(true)
+    expect(sampledCharacters).toBe(100 * 32 * 256)
+  })
+
   it('still reports the workbook the sheet declares', async () => {
     const result = await new XlsxParser().parseBuffer(inflatedRangeWorkbook())
 
@@ -98,4 +140,46 @@ describe('XlsxParser preview bound', () => {
     expect(result.metadata?.truncated).toBe(false)
     expect(result.content).not.toContain('total rows, showing first')
   })
+
+  it('preserves a long cell in full when the aggregate output remains within budget', async () => {
+    const longCell = 'x'.repeat(2_000)
+    const sheet = XLSX.utils.aoa_to_sheet([['header'], [longCell]])
+    const book = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(book, sheet, 'Long cell')
+    const buffer = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+
+    const result = await new XlsxParser().parseBuffer(buffer)
+
+    expect(result.metadata?.truncated).toBe(false)
+    expect(result.content).toContain(longCell)
+  })
+
+  it.each(['', '   '])(
+    'does not treat an empty-string cell as extractable content',
+    async (cell) => {
+      const sheet = XLSX.utils.aoa_to_sheet([[cell]])
+      const book = XLSX.utils.book_new()
+      XLSX.utils.book_append_sheet(book, sheet, 'Empty')
+      const buffer = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+
+      const result = await new XlsxParser().parseBuffer(buffer)
+
+      expect(result.metadata?.degraded).toBe(true)
+    }
+  )
+
+  it('measures the rendered content cap in UTF-8 bytes', async () => {
+    const rows = Array.from({ length: 50 }, () =>
+      Array.from({ length: 256 }, () => '🚀'.repeat(300))
+    )
+    const sheet = XLSX.utils.aoa_to_sheet(rows)
+    const book = XLSX.utils.book_new()
+    XLSX.utils.book_append_sheet(book, sheet, 'Unicode')
+    const buffer = XLSX.write(book, { type: 'buffer', bookType: 'xlsx' }) as Buffer
+
+    const result = await new XlsxParser().parseBuffer(buffer)
+
+    expect(Buffer.byteLength(result.content, 'utf8')).toBeLessThanOrEqual(10 * 1024 * 1024)
+    expect(result.content).toContain('Content truncated due to size limits')
+  })
 })

apps/sim/lib/file-parsers/yaml-parser.test.ts

@@ -26,6 +26,10 @@ function buildAliasBomb(levels: number, width: number): string {
 }
 
 describe('parseYAMLBuffer', () => {
+  it('reports empty input with the typed parser taxonomy', async () => {
+    await expect(parseYAMLBuffer(Buffer.alloc(0))).rejects.toMatchObject({ code: 'empty_input' })
+  })
+
   it('parses a normal YAML document', async () => {
     const result = await parseYAMLBuffer(
       Buffer.from('name: sim\nlist:\n  - a\n  - b\nnested:\n  key: value\n')

apps/sim/lib/file-parsers/yaml-parser.ts

@@ -1,5 +1,6 @@
 import { getErrorMessage } from '@sim/utils/errors'
 import * as yaml from 'js-yaml'
+import { FileParserError } from '@/lib/file-parsers/errors'
 import type { FileParseResult } from '@/lib/file-parsers/types'
 
 /**
@@ -34,9 +35,9 @@ const MAX_YAML_DEPTH = 500
  * Distinct from a syntax error so callers can tell a malformed file apart from
  * a resource-exhaustion (alias-expansion DoS) attempt.
  */
-export class YamlComplexityError extends Error {
+export class YamlComplexityError extends FileParserError {
   constructor(message: string) {
-    super(message)
+    super('complexity_limit', message)
     this.name = 'YamlComplexityError'
   }
 }
@@ -179,6 +180,10 @@ export function assertYamlWithinLimits(root: unknown): number {
  * that its expanded form stays within safe complexity limits.
  */
 function buildYamlResult(yamlData: unknown): FileParseResult {
+  if (yamlData === undefined) {
+    throw new FileParserError('empty_input', 'Empty YAML input provided')
+  }
+
   const depth = assertYamlWithinLimits(yamlData)
   const jsonContent = JSON.stringify(yamlData, null, 2)
 
@@ -207,8 +212,12 @@ export async function parseYAML(filePath: string): Promise<FileParseResult> {
     const yamlData = yaml.load(content)
     return buildYamlResult(yamlData)
   } catch (error) {
-    if (error instanceof YamlComplexityError) throw error
-    throw new Error(`Invalid YAML: ${getErrorMessage(error, 'Unknown error')}`)
+    if (error instanceof FileParserError) throw error
+    throw new FileParserError(
+      'invalid_format',
+      `Invalid YAML: ${getErrorMessage(error, 'Unknown error')}`,
+      error
+    )
   }
 }
 
@@ -216,13 +225,21 @@ export async function parseYAML(filePath: string): Promise<FileParseResult> {
  * Parse YAML from buffer
  */
 export async function parseYAMLBuffer(buffer: Buffer): Promise<FileParseResult> {
+  if (!buffer || buffer.length === 0) {
+    throw new FileParserError('empty_input', 'Empty buffer provided')
+  }
+
   const content = buffer.toString('utf-8')
 
   try {
     const yamlData = yaml.load(content)
     return buildYamlResult(yamlData)
   } catch (error) {
-    if (error instanceof YamlComplexityError) throw error
-    throw new Error(`Invalid YAML: ${getErrorMessage(error, 'Unknown error')}`)
+    if (error instanceof FileParserError) throw error
+    throw new FileParserError(
+      'invalid_format',
+      `Invalid YAML: ${getErrorMessage(error, 'Unknown error')}`,
+      error
+    )
   }
 }

apps/sim/lib/file-parsers/zip-guard.test.ts

@@ -3,7 +3,7 @@
  */
 import JSZip from 'jszip'
 import { describe, expect, it } from 'vitest'
-import { ZipBombError } from '@/lib/file-parsers/ooxml-limits'
+import { ArchiveIntegrityError, ZipBombError } from '@/lib/file-parsers/ooxml-limits'
 import { assertOoxmlArchiveWithinLimits, type OoxmlSizeLimits } from '@/lib/file-parsers/zip-guard'
 
 const HIGH_LIMITS: OoxmlSizeLimits = {
@@ -31,6 +31,24 @@ async function buildZip(
 const CENTRAL_DIRECTORY_HEADER_SIGNATURE = 0x02014b50
 const LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50
 
+function buildCentralDirectoryOnly(entryCount: number, extraFieldBytesPerEntry = 0): Buffer {
+  const recordSize = 46 + extraFieldBytesPerEntry
+  const centralDirectory = Buffer.alloc(recordSize * entryCount)
+  for (let index = 0; index < entryCount; index++) {
+    const offset = index * recordSize
+    centralDirectory.writeUInt32LE(CENTRAL_DIRECTORY_HEADER_SIGNATURE, offset)
+    centralDirectory.writeUInt16LE(extraFieldBytesPerEntry, offset + 30)
+  }
+
+  const eocd = Buffer.alloc(22)
+  eocd.writeUInt32LE(0x06054b50, 0)
+  eocd.writeUInt16LE(entryCount, 8)
+  eocd.writeUInt16LE(entryCount, 10)
+  eocd.writeUInt32LE(centralDirectory.length, 12)
+  eocd.writeUInt32LE(0, 16)
+  return Buffer.concat([centralDirectory, eocd])
+}
+
 /**
  * Rewrite every declared uncompressed size — in both the central directory and
  * the local file headers — so the archive under-reports how much it expands to.
@@ -91,6 +109,22 @@ describe('assertOoxmlArchiveWithinLimits', () => {
     expect(() => assertOoxmlArchiveWithinLimits(buffer, HIGH_LIMITS)).not.toThrow()
   })
 
+  it('rejects a small archive with an excessive central-directory object count', () => {
+    const buffer = buildCentralDirectoryOnly(10_001)
+
+    expect(() => assertOoxmlArchiveWithinLimits(buffer)).toThrow(
+      /10001 entries, exceeding the maximum allowed 10000/
+    )
+  })
+
+  it('rejects excessive central-directory extra-field metadata', () => {
+    const buffer = buildCentralDirectoryOnly(65, 65_535)
+
+    expect(() => assertOoxmlArchiveWithinLimits(buffer)).toThrow(
+      /central-directory metadata .* exceeds the maximum allowed 4194304 bytes/
+    )
+  })
+
   it('rejects an archive whose declared expanded size exceeds the absolute cap', async () => {
     const buffer = await buildZip({ 'xl/worksheets/sheet1.xml': 'A'.repeat(200_000) })
     expect(() =>
@@ -202,7 +236,7 @@ describe('assertOoxmlArchiveWithinLimits', () => {
   it('fails closed for a ZIP-shaped buffer whose central directory is unparseable', () => {
     const buffer = Buffer.alloc(64)
     buffer.writeUInt32LE(0x04034b50, 0) // local file header signature, no valid EOCD
-    expect(() => assertOoxmlArchiveWithinLimits(buffer)).toThrow(ZipBombError)
+    expect(() => assertOoxmlArchiveWithinLimits(buffer)).toThrow(ArchiveIntegrityError)
   })
 
   it('rejects a decoy EOCD signature that does not validate against the buffer tail', async () => {
@@ -213,7 +247,7 @@ describe('assertOoxmlArchiveWithinLimits', () => {
     const decoy = Buffer.alloc(64)
     decoy.writeUInt32LE(0x06054b50, 0)
     const tampered = Buffer.concat([realZip, decoy])
-    expect(() => assertOoxmlArchiveWithinLimits(tampered)).toThrow(ZipBombError)
+    expect(() => assertOoxmlArchiveWithinLimits(tampered)).toThrow(ArchiveIntegrityError)
   })
 
   it('rejects an archive that under-declares its uncompressed size', async () => {
@@ -222,7 +256,7 @@ describe('assertOoxmlArchiveWithinLimits', () => {
     const honest = await buildZip({ 'word/document.xml': 'A'.repeat(200_000) })
     const lying = underDeclareSizes(honest, 1000)
 
-    expect(() => assertOoxmlArchiveWithinLimits(lying, HIGH_LIMITS)).toThrow(ZipBombError)
+    expect(() => assertOoxmlArchiveWithinLimits(lying, HIGH_LIMITS)).toThrow(ArchiveIntegrityError)
     expect(() => assertOoxmlArchiveWithinLimits(lying, HIGH_LIMITS)).toThrow(
       /inflates beyond the 1000 bytes it declares/
     )
@@ -259,7 +293,7 @@ describe('assertOoxmlArchiveWithinLimits', () => {
     const honest = await buildZip({ 'xl/worksheets/sheet1.xml': 'A'.repeat(200_000) })
     const split = setCompressionMethod(honest, 0, 'central')
 
-    expect(() => assertOoxmlArchiveWithinLimits(split, HIGH_LIMITS)).toThrow(ZipBombError)
+    expect(() => assertOoxmlArchiveWithinLimits(split, HIGH_LIMITS)).toThrow(ArchiveIntegrityError)
     expect(() => assertOoxmlArchiveWithinLimits(split, HIGH_LIMITS)).toThrow(
       /compression method 0 centrally but 8 locally/
     )

apps/sim/lib/file-parsers/zip-guard.ts

@@ -1,6 +1,9 @@
 import { inflateRawSync } from 'zlib'
 import { createLogger } from '@sim/logger'
 import {
+  ArchiveIntegrityError,
+  MAX_OOXML_CENTRAL_DIRECTORY_EXTRA_BYTES,
+  MAX_OOXML_CENTRAL_DIRECTORY_RECORDS,
   MAX_OOXML_ENTRY_UNCOMPRESSED_BYTES,
   MAX_OOXML_TOTAL_UNCOMPRESSED_BYTES,
   ZipBombError,
@@ -224,6 +227,10 @@ interface DeclaredSizeStats {
   total: number
   /** Largest single entry's declared uncompressed size. */
   largestEntry: number
+  /** Records a downstream ZIP parser would materialize. */
+  entryCount: number
+  /** Summed central-directory extra-field bytes retained by ZIP parsers. */
+  totalExtraFieldBytes: number
 }
 
 /**
@@ -267,6 +274,7 @@ function sumDeclaredUncompressedSize(
   let total = 0
   let largestEntry = 0
   let counted = 0
+  let totalExtraFieldBytes = 0
   let cursor = location.offset
   while (
     cursor + CENTRAL_DIRECTORY_HEADER_MIN_SIZE <= buffer.length &&
@@ -282,12 +290,13 @@ function sumDeclaredUncompressedSize(
       fileNameLength,
       extraFieldLength
     ).uncompressedSize
+    totalExtraFieldBytes += extraFieldLength
     total += entryBytes
     if (entryBytes > largestEntry) {
       largestEntry = entryBytes
     }
     if (total > limits.maxTotalUncompressedBytes || entryBytes > limits.maxEntryUncompressedBytes) {
-      return { total, largestEntry }
+      return { total, largestEntry, entryCount: counted + 1, totalExtraFieldBytes }
     }
 
     counted += 1
@@ -300,7 +309,7 @@ function sumDeclaredUncompressedSize(
     return null
   }
 
-  return { total, largestEntry }
+  return { total, largestEntry, entryCount: counted, totalExtraFieldBytes }
 }
 
 /**
@@ -506,14 +515,14 @@ export function assertOoxmlArchiveWithinLimits(
       logger.warn('Rejected ZIP-shaped archive: central directory could not be parsed', {
         compressedBytes: buffer.length,
       })
-      throw new ZipBombError(
+      throw new ArchiveIntegrityError(
         'Unable to inspect ZIP central directory; refusing to parse an unverifiable ZIP-shaped archive'
       )
     }
     return
   }
 
-  const { total: totalUncompressed, largestEntry } = declared
+  const { total: totalUncompressed, largestEntry, entryCount, totalExtraFieldBytes } = declared
 
   if (largestEntry > limits.maxEntryUncompressedBytes) {
     logger.warn('Rejected OOXML archive: a single entry exceeds the per-entry limit', {
@@ -537,6 +546,28 @@ export function assertOoxmlArchiveWithinLimits(
     )
   }
 
+  if (entryCount > MAX_OOXML_CENTRAL_DIRECTORY_RECORDS) {
+    logger.warn('Rejected OOXML archive: central-directory record count exceeds limit', {
+      entryCount,
+      maxEntryCount: MAX_OOXML_CENTRAL_DIRECTORY_RECORDS,
+      compressedBytes: buffer.length,
+    })
+    throw new ZipBombError(
+      `Archive contains ${entryCount} entries, exceeding the maximum allowed ${MAX_OOXML_CENTRAL_DIRECTORY_RECORDS}`
+    )
+  }
+
+  if (totalExtraFieldBytes > MAX_OOXML_CENTRAL_DIRECTORY_EXTRA_BYTES) {
+    logger.warn('Rejected OOXML archive: central-directory metadata exceeds limit', {
+      totalExtraFieldBytes,
+      maxExtraFieldBytes: MAX_OOXML_CENTRAL_DIRECTORY_EXTRA_BYTES,
+      compressedBytes: buffer.length,
+    })
+    throw new ZipBombError(
+      `Archive central-directory metadata (${totalExtraFieldBytes} bytes) exceeds the maximum allowed ${MAX_OOXML_CENTRAL_DIRECTORY_EXTRA_BYTES} bytes`
+    )
+  }
+
   const ratio = totalUncompressed / Math.max(buffer.length, 1)
   if (totalUncompressed > limits.ratioCheckFloorBytes && ratio > limits.maxCompressionRatio) {
     logger.warn('Rejected OOXML archive: compression ratio exceeds limit', {
@@ -556,7 +587,7 @@ export function assertOoxmlArchiveWithinLimits(
     logger.warn('Rejected ZIP-shaped archive: central directory could not be re-read', {
       compressedBytes: buffer.length,
     })
-    throw new ZipBombError(
+    throw new ArchiveIntegrityError(
       'Unable to inspect ZIP central directory; refusing to parse an unverifiable ZIP-shaped archive'
     )
   }
@@ -568,6 +599,6 @@ export function assertOoxmlArchiveWithinLimits(
       declaredTotalUncompressed: totalUncompressed,
       compressedBytes: buffer.length,
     })
-    throw new ZipBombError(`Archive contents do not match declared sizes: ${mismatch}`)
+    throw new ArchiveIntegrityError(`Archive contents do not match declared sizes: ${mismatch}`)
   }
 }

apps/sim/lib/knowledge/application/documents.ts

@@ -865,7 +865,10 @@ export const updateKnowledgeDocument = defineAuthorizedKnowledgeUseCase({
   async execute({ principal, input, context }) {
     if (input.markFailedDueToTimeout || input.retryProcessing) {
       const outcome = input.markFailedDueToTimeout
-        ? await performMarkKnowledgeDocumentTimedOut({ document: context.document })
+        ? await performMarkKnowledgeDocumentTimedOut({
+            knowledgeBaseId: context.knowledgeBaseId,
+            document: context.document,
+          })
         : await performRetryKnowledgeDocumentProcessing({
             knowledgeBaseId: context.knowledgeBaseId,
             document: context.document,

apps/sim/lib/knowledge/constants.ts

@@ -179,10 +179,12 @@ export function getPlaceholderForFieldType(fieldType: string): string {
 }
 
 /**
- * How long a document may sit in `processing` before its run is treated as dead.
+ * Minimum time the client waits before asking the server to classify an active
+ * document-processing run as dead.
  *
- * Lives here rather than beside the server-side claim helpers so the client can
- * read the same number without importing a module that pulls in the database
- * client. `processing-claim.ts` re-exports it for its own callers.
+ * Lives here so the client does not import server configuration. The server
+ * derives its authoritative threshold from the configured task
+ * duration and retry budget and may require longer. Keeping the client at the
+ * same 45-minute floor prevents the default UI from racing a legitimate run.
  */
-export const KNOWLEDGE_DOCUMENT_PROCESSING_STALE_THRESHOLD_MS = 10 * 60 * 1000
+export const KNOWLEDGE_DOCUMENT_PROCESSING_STALE_THRESHOLD_MS = 45 * 60 * 1000

apps/sim/lib/knowledge/documents/document-indexing-usage.test.ts

@@ -42,6 +42,7 @@ vi.mock('@/lib/knowledge/documents/document-processor', () => ({
 }))
 
 vi.mock('@/lib/knowledge/embedding-models', () => ({
+  EMBEDDING_DIMENSIONS: 1536,
   getEmbeddingModelInfo: vi.fn(() => ({ tokenizerProvider: 'openai' })),
 }))
 

apps/sim/lib/knowledge/documents/document-processing-error.test.ts

@@ -0,0 +1,222 @@
+import { describe, expect, it } from 'vitest'
+import { FileParserError, type FileParserErrorCode } from '@/lib/file-parsers/errors'
+import { ArchiveIntegrityError, ZipBombError } from '@/lib/file-parsers/ooxml-limits'
+import {
+  assertDocumentChunkCountWithinLimit,
+  classifyDocumentProcessingFailure,
+  MAX_DOCUMENT_CHUNKS,
+  PermanentDocumentProcessingError,
+  toPermanentDocumentProcessingError,
+} from '@/lib/knowledge/documents/document-processing-error'
+
+describe('document processing failure taxonomy', () => {
+  it('classifies archive safety rejections without exposing technical limits as the remedy', () => {
+    const failure = classifyDocumentProcessingFailure(
+      new ZipBombError('Archive entry xl/worksheets/sheet1.xml exceeds 67108864 bytes'),
+      'Vendor Spend.xlsx'
+    )
+
+    expect(failure).toEqual({
+      disposition: 'permanent',
+      code: 'archive_safety_limit',
+      userMessage:
+        'This file expands beyond the safe processing limit and was not indexed. Reduce its size or split it into smaller files, then retry.',
+    })
+  })
+
+  it('does not infer archive safety from untyped exception text', () => {
+    const failure = classifyDocumentProcessingFailure(
+      new Error(
+        'Failed to parse XLSX buffer: Archive total uncompressed size exceeds 157286400 bytes'
+      ),
+      'Marketing plan.xlsx'
+    )
+
+    expect(failure).toMatchObject({
+      disposition: 'transient',
+      code: 'transient_processing_failure',
+    })
+  })
+
+  it('does not infer encryption from untyped exception text', () => {
+    const failure = classifyDocumentProcessingFailure(
+      new Error('Failed to parse XLSX buffer: File is password-protected'),
+      'Reconciliation.xlsx'
+    )
+
+    expect(failure).toMatchObject({
+      disposition: 'transient',
+      code: 'transient_processing_failure',
+    })
+  })
+
+  it('does not dead-letter Office files from untyped exception text', () => {
+    const failure = classifyDocumentProcessingFailure(
+      new Error('Failed to parse DOCX buffer: Failed to extract text from DOCX file'),
+      'Letterhead.dotx'
+    )
+
+    expect(failure).toMatchObject({
+      disposition: 'transient',
+      code: 'transient_processing_failure',
+    })
+  })
+
+  it('preserves typed no-text guidance', () => {
+    const error = new PermanentDocumentProcessingError(
+      'no_extractable_text',
+      'No text could be extracted. Re-save it as DOCX to index it.'
+    )
+
+    expect(classifyDocumentProcessingFailure(error, 'Contract.doc')).toEqual({
+      disposition: 'permanent',
+      code: 'no_extractable_text',
+      userMessage: error.message,
+    })
+    expect(toPermanentDocumentProcessingError(error, 'Contract.doc')).toBe(error)
+  })
+
+  it.each(['Contract.doc', 'Budget.xls', 'Deck.ppt'])(
+    'classifies an unreadable legacy Office file as repairable: %s',
+    (filename) => {
+      const failure = classifyDocumentProcessingFailure(
+        new FileParserError('invalid_format', 'The legacy Office file could not be parsed'),
+        filename
+      )
+
+      expect(failure).toMatchObject({
+        disposition: 'permanent',
+        code: 'unreadable_office_file',
+        userMessage: expect.stringContaining('re-save'),
+      })
+    }
+  )
+
+  it.each<{
+    parserCode: FileParserErrorCode
+    filename: string
+    disposition: 'permanent' | 'transient'
+    documentCode: string
+  }>([
+    {
+      parserCode: 'empty_input',
+      filename: 'empty.pdf',
+      disposition: 'permanent',
+      documentCode: 'invalid_file',
+    },
+    {
+      parserCode: 'unsupported_type',
+      filename: 'diagram.vsdx',
+      disposition: 'permanent',
+      documentCode: 'unsupported_file_type',
+    },
+    {
+      parserCode: 'encrypted_file',
+      filename: 'protected.xlsx',
+      disposition: 'permanent',
+      documentCode: 'encrypted_file',
+    },
+    {
+      parserCode: 'no_extractable_text',
+      filename: 'scan.pdf',
+      disposition: 'permanent',
+      documentCode: 'no_extractable_text',
+    },
+    {
+      parserCode: 'invalid_format',
+      filename: 'damaged.dotx',
+      disposition: 'permanent',
+      documentCode: 'unreadable_office_file',
+    },
+    {
+      parserCode: 'runtime_failure',
+      filename: 'valid.docx',
+      disposition: 'transient',
+      documentCode: 'transient_processing_failure',
+    },
+    {
+      parserCode: 'complexity_limit',
+      filename: 'large.yaml',
+      disposition: 'permanent',
+      documentCode: 'document_complexity_limit',
+    },
+  ])(
+    'maps typed parser code $parserCode exhaustively into the document taxonomy',
+    ({ parserCode, filename, disposition, documentCode }) => {
+      const failure = classifyDocumentProcessingFailure(
+        new FileParserError(parserCode, `parser failure: ${parserCode}`),
+        filename
+      )
+
+      expect(failure).toMatchObject({ disposition, code: documentCode })
+    }
+  )
+
+  it('keeps a successful-but-empty OCR result permanent and an OCR timeout transient', () => {
+    const empty = new PermanentDocumentProcessingError(
+      'no_extractable_text',
+      'No text could be extracted from this file.'
+    )
+
+    expect(classifyDocumentProcessingFailure(empty, 'scan.pdf')).toMatchObject({
+      disposition: 'permanent',
+      code: 'no_extractable_text',
+    })
+    expect(
+      classifyDocumentProcessingFailure(new Error('OCR API request timed out'), 'scan.pdf')
+    ).toMatchObject({
+      disposition: 'transient',
+      code: 'transient_processing_failure',
+    })
+  })
+
+  it('keeps stale downloads and access failures transient', () => {
+    for (const error of [
+      Object.assign(new Error('Not Found'), { status: 404 }),
+      Object.assign(new Error('Access denied'), { status: 403 }),
+    ]) {
+      expect(classifyDocumentProcessingFailure(error, 'Report.docx')).toMatchObject({
+        disposition: 'transient',
+        code: 'transient_processing_failure',
+      })
+    }
+  })
+
+  it('leaves infrastructure and provider failures transient', () => {
+    for (const error of [
+      new Error('Storage request timed out'),
+      new Error('Database connection terminated unexpectedly'),
+      new Error('Embedding provider returned 503'),
+      new TypeError('parseOfficeAsync is not a function'),
+    ]) {
+      expect(classifyDocumentProcessingFailure(error, 'Report.docx')).toMatchObject({
+        disposition: 'transient',
+        code: 'transient_processing_failure',
+      })
+      expect(toPermanentDocumentProcessingError(error, 'Report.docx')).toBeNull()
+    }
+  })
+
+  it('rejects excessive chunk counts before embedding allocation without truncating content', () => {
+    expect(() => assertDocumentChunkCountWithinLimit(MAX_DOCUMENT_CHUNKS)).not.toThrow()
+    expect(() => assertDocumentChunkCountWithinLimit(MAX_DOCUMENT_CHUNKS + 1)).toThrowError(
+      expect.objectContaining({
+        code: 'document_complexity_limit',
+        message: expect.stringContaining('Split it into smaller files'),
+      })
+    )
+  })
+
+  it('distinguishes malformed archive integrity from an expanded-size limit', () => {
+    const failure = classifyDocumentProcessingFailure(
+      new ArchiveIntegrityError('Unable to inspect ZIP central directory'),
+      'damaged.docx'
+    )
+
+    expect(failure).toMatchObject({
+      disposition: 'permanent',
+      code: 'unreadable_office_file',
+      userMessage: expect.stringContaining('re-save'),
+    })
+  })
+})

apps/sim/lib/knowledge/documents/document-processing-error.ts

@@ -0,0 +1,244 @@
+import { getErrorMessage } from '@sim/utils/errors'
+import { isFileParserError } from '@/lib/file-parsers/errors'
+import { ArchiveIntegrityError, ZipBombError } from '@/lib/file-parsers/ooxml-limits'
+import { getFileExtension } from '@/lib/uploads/utils/file-utils'
+
+export const DOCUMENT_PROCESSING_FAILURE_CODES = [
+  'archive_safety_limit',
+  'encrypted_file',
+  'no_extractable_text',
+  'unreadable_office_file',
+  'unsupported_file_type',
+  'invalid_file',
+  'document_complexity_limit',
+  'transient_processing_failure',
+] as const
+
+export type DocumentProcessingFailureCode = (typeof DOCUMENT_PROCESSING_FAILURE_CODES)[number]
+
+export type DocumentProcessingFailure =
+  | {
+      readonly disposition: 'permanent'
+      readonly code: Exclude<DocumentProcessingFailureCode, 'transient_processing_failure'>
+      readonly userMessage: string
+    }
+  | {
+      readonly disposition: 'transient'
+      readonly code: 'transient_processing_failure'
+      readonly userMessage: string
+    }
+
+/**
+ * A deterministic failure caused by the document bytes or format.
+ *
+ * The row remains `failed` and can still be retried explicitly after its
+ * content is replaced or repaired. The distinction is only about unattended
+ * retries: rerunning the same bytes cannot change this outcome.
+ */
+export class PermanentDocumentProcessingError extends Error {
+  readonly code: Exclude<DocumentProcessingFailureCode, 'transient_processing_failure'>
+
+  constructor(
+    code: Exclude<DocumentProcessingFailureCode, 'transient_processing_failure'>,
+    userMessage: string,
+    cause?: unknown
+  ) {
+    super(userMessage, cause === undefined ? undefined : { cause })
+    this.name = 'PermanentDocumentProcessingError'
+    this.code = code
+  }
+}
+
+/**
+ * A mutable billing gate that must stop this attempt without consuming the
+ * document's unattended retry budget. A plan upgrade or credit top-up can make
+ * the same bytes processable, so this is intentionally not a permanent input
+ * failure and must not be retried immediately by Trigger.dev.
+ */
+export class UsageLimitDocumentProcessingError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'UsageLimitDocumentProcessingError'
+  }
+}
+
+export function isUsageLimitDocumentProcessingError(
+  error: unknown
+): error is UsageLimitDocumentProcessingError {
+  return error instanceof UsageLimitDocumentProcessingError
+}
+
+/**
+ * Maximum vectors and embedding records retained before the atomic index swap.
+ *
+ * Each knowledge-base vector is 1,536 JavaScript numbers. The old 100,000-chunk
+ * ceiling could retain well over a gigabyte before response JSON, array
+ * overhead, chunk text, provenance, and insert records. Five thousand bounds
+ * raw vector values to roughly 59 MiB while preserving the atomic replacement
+ * behavior instead of silently truncating indexed content.
+ */
+export const MAX_DOCUMENT_CHUNKS = 5_000
+
+export function assertDocumentChunkCountWithinLimit(chunkCount: number): void {
+  if (chunkCount <= MAX_DOCUMENT_CHUNKS) return
+  throw new PermanentDocumentProcessingError(
+    'document_complexity_limit',
+    `This document produced ${chunkCount.toLocaleString()} index chunks, exceeding the safe limit of ${MAX_DOCUMENT_CHUNKS.toLocaleString()}. Split it into smaller files or increase its knowledge-base chunk size, then retry.`
+  )
+}
+
+export function isPermanentDocumentProcessingError(
+  error: unknown
+): error is PermanentDocumentProcessingError {
+  return error instanceof PermanentDocumentProcessingError
+}
+
+const OFFICE_REPAIR_EXTENSIONS = new Set([
+  'doc',
+  'docx',
+  'docm',
+  'dotx',
+  'xls',
+  'xlsx',
+  'xlsm',
+  'xlsb',
+  'xltx',
+  'ppt',
+  'pptx',
+  'pptm',
+  'potx',
+  'odt',
+  'ods',
+  'odp',
+])
+
+function officeFormatName(filename: string): string {
+  const extension = getFileExtension(filename)
+  return extension ? extension.toUpperCase() : 'Office'
+}
+
+/**
+ * Classifies only failures whose retry behavior is known from stable parser
+ * evidence. Unknown exceptions stay transient so code, storage, database, and
+ * provider outages are never silently dead-lettered as bad user input.
+ */
+export function classifyDocumentProcessingFailure(
+  error: unknown,
+  filename: string
+): DocumentProcessingFailure {
+  if (isPermanentDocumentProcessingError(error)) {
+    return {
+      disposition: 'permanent',
+      code: error.code,
+      userMessage: error.message,
+    }
+  }
+
+  const extension = getFileExtension(filename)
+
+  if (isFileParserError(error)) {
+    switch (error.code) {
+      case 'empty_input':
+        return {
+          disposition: 'permanent',
+          code: 'invalid_file',
+          userMessage: 'This file is empty or invalid. Replace it with a valid file and retry.',
+        }
+      case 'unsupported_type':
+        return {
+          disposition: 'permanent',
+          code: 'unsupported_file_type',
+          userMessage: 'This file type is not supported for indexing. Convert it and retry.',
+        }
+      case 'encrypted_file':
+        return {
+          disposition: 'permanent',
+          code: 'encrypted_file',
+          userMessage:
+            'This file is encrypted or password-protected. Remove the protection and retry.',
+        }
+      case 'no_extractable_text':
+        return {
+          disposition: 'permanent',
+          code: 'no_extractable_text',
+          userMessage: error.message,
+        }
+      case 'invalid_format':
+        return OFFICE_REPAIR_EXTENSIONS.has(extension)
+          ? {
+              disposition: 'permanent',
+              code: 'unreadable_office_file',
+              userMessage: `This ${officeFormatName(filename)} file could not be read. Open and re-save it as a valid ${officeFormatName(filename)} file, then retry.`,
+            }
+          : {
+              disposition: 'permanent',
+              code: 'invalid_file',
+              userMessage:
+                'This file is invalid or unreadable. Replace it with a valid file and retry.',
+            }
+      case 'complexity_limit':
+        return {
+          disposition: 'permanent',
+          code: 'document_complexity_limit',
+          userMessage:
+            'This document exceeds safe processing complexity limits. Simplify it or split it into smaller files, then retry.',
+        }
+      case 'runtime_failure':
+        return {
+          disposition: 'transient',
+          code: 'transient_processing_failure',
+          userMessage: error.message,
+        }
+      default: {
+        const exhaustiveCode: never = error.code
+        return exhaustiveCode
+      }
+    }
+  }
+
+  if (error instanceof ArchiveIntegrityError) {
+    return OFFICE_REPAIR_EXTENSIONS.has(extension)
+      ? {
+          disposition: 'permanent',
+          code: 'unreadable_office_file',
+          userMessage: `This ${officeFormatName(filename)} file could not be read. Open and re-save it as a valid ${officeFormatName(filename)} file, then retry.`,
+        }
+      : {
+          disposition: 'permanent',
+          code: 'invalid_file',
+          userMessage:
+            'This archive is invalid or unreadable. Replace it with a valid file and retry.',
+        }
+  }
+
+  if (error instanceof ZipBombError) {
+    return {
+      disposition: 'permanent',
+      code: 'archive_safety_limit',
+      userMessage:
+        'This file expands beyond the safe processing limit and was not indexed. Reduce its size or split it into smaller files, then retry.',
+    }
+  }
+
+  return {
+    disposition: 'transient',
+    code: 'transient_processing_failure',
+    userMessage: getErrorMessage(error, 'Document processing failed. Please retry.'),
+  }
+}
+
+/**
+ * Preserves a typed permanent error or converts stable parser evidence into
+ * one. Transient exceptions are returned unchanged by callers.
+ */
+export function toPermanentDocumentProcessingError(
+  error: unknown,
+  filename: string
+): PermanentDocumentProcessingError | null {
+  if (isPermanentDocumentProcessingError(error)) return error
+
+  const failure = classifyDocumentProcessingFailure(error, filename)
+  return failure.disposition === 'permanent'
+    ? new PermanentDocumentProcessingError(failure.code, failure.userMessage, error)
+    : null
+}

apps/sim/lib/knowledge/documents/document-processing-source.test.ts

@@ -3,25 +3,38 @@
  */
 import {
   dbChainMockFns,
+  defaultMockEnv,
   hasMockCondition,
   type MockCondition,
   resetDbChainMock,
+  resetEnvFlagsMock,
   schemaMock,
+  setEnvFlags,
 } from '@sim/testing'
-import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 
 const {
   mockCheckActorUsageLimits,
+  mockBatchTrigger,
   mockGenerateEmbeddings,
   mockGetBoundWorkspaceFileSecretProvenanceByMetadata,
+  mockGetEmbeddingModelInfo,
   mockGetFileMetadataByKeys,
   mockProcessDocument,
+  mockTrigger,
 } = vi.hoisted(() => ({
   mockCheckActorUsageLimits: vi.fn(),
+  mockBatchTrigger: vi.fn(),
   mockGenerateEmbeddings: vi.fn(),
   mockGetBoundWorkspaceFileSecretProvenanceByMetadata: vi.fn(),
+  mockGetEmbeddingModelInfo: vi.fn(),
   mockGetFileMetadataByKeys: vi.fn(),
   mockProcessDocument: vi.fn(),
+  mockTrigger: vi.fn(),
+}))
+
+vi.mock('@trigger.dev/sdk', () => ({
+  tasks: { batchTrigger: mockBatchTrigger, trigger: mockTrigger },
 }))
 
 vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
@@ -33,7 +46,8 @@ vi.mock('@/lib/knowledge/documents/document-processor', () => ({
 }))
 
 vi.mock('@/lib/knowledge/embedding-models', () => ({
-  getEmbeddingModelInfo: vi.fn(() => ({ tokenizerProvider: 'openai' })),
+  EMBEDDING_DIMENSIONS: 1536,
+  getEmbeddingModelInfo: mockGetEmbeddingModelInfo,
 }))
 
 vi.mock('@/lib/knowledge/embeddings', () => ({
@@ -54,7 +68,19 @@ vi.mock('@/lib/uploads/server/metadata', () => ({
   getFileMetadataByKeys: mockGetFileMetadataByKeys,
 }))
 
-import { processDocumentAsync } from '@/lib/knowledge/documents/service'
+import { env } from '@/lib/core/config/env'
+import {
+  markInsideTriggerRun,
+  resetInsideTriggerRunForTests,
+} from '@/lib/core/config/trigger-runtime'
+import { EMBEDDING_QUOTA_EXHAUSTED_MESSAGE } from '@/lib/embeddings'
+import { EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client'
+import {
+  PermanentDocumentProcessingError,
+  UsageLimitDocumentProcessingError,
+} from '@/lib/knowledge/documents/document-processing-error'
+import { processDocumentAsync, processDocumentsWithQueue } from '@/lib/knowledge/documents/service'
+import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types'
 
 const PERSISTED_KEY = 'workspace/workspace-1/persisted.pdf'
 const PERSISTED_URL = `/api/files/serve/${encodeURIComponent(PERSISTED_KEY)}?context=workspace`
@@ -158,6 +184,10 @@ describe('knowledge document processing source', () => {
       chunks: [],
       metadata: { chunkCount: 0, tokenCount: 0, characterCount: 0 },
     })
+    mockGetEmbeddingModelInfo.mockReturnValue({
+      tokenizerProvider: 'openai',
+      maxInputTokens: 8191,
+    })
   })
 
   it('uses the persisted document source instead of stale queued source fields', async () => {
@@ -274,29 +304,25 @@ describe('processDocumentAsync write guards', () => {
       chunks: [],
       metadata: { chunkCount: 0, tokenCount: 0, characterCount: 0 },
     })
+    mockGetEmbeddingModelInfo.mockReturnValue({
+      tokenizerProvider: 'openai',
+      maxInputTokens: 8191,
+    })
   })
 
   /** Asserts the write that set `status` exists, and returns its guard clause. */
   function guardForStatusWrite(status: string): unknown {
-    expect(
-      dbChainMockFns.set.mock.calls.some(
-        (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === status
-      )
-    ).toBe(true)
-
-    // `set` and `where` are separate shared spies, so they cannot be correlated
-    // by index; the guard is identified by its own shape instead.
-    const guard = dbChainMockFns.where.mock.calls.find((call) =>
-      hasMockCondition(
-        call[0],
-        (node: MockCondition) =>
-          node.type === 'ne' &&
-          node.left === schemaMock.document.processingStatus &&
-          node.right === 'completed'
-      )
+    const setIndex = dbChainMockFns.set.mock.calls.findIndex(
+      (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === status
     )
-    expect(guard).toBeDefined()
-    return guard?.[0]
+    expect(setIndex).toBeGreaterThanOrEqual(0)
+
+    const setOrder = dbChainMockFns.set.mock.invocationCallOrder[setIndex]
+    const whereIndex = dbChainMockFns.where.mock.invocationCallOrder.findIndex(
+      (whereOrder) => whereOrder > setOrder
+    )
+    expect(whereIndex).toBeGreaterThanOrEqual(0)
+    return dbChainMockFns.where.mock.calls[whereIndex]?.[0]
   }
 
   it('never claims a document whose pass already completed', async () => {
@@ -323,6 +349,187 @@ describe('processDocumentAsync write guards', () => {
     expect(guardForStatusWrite('processing')).toBeDefined()
   })
 
+  it('signals ownership before a claimed processing attempt can fail', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    mockProcessDocument.mockRejectedValueOnce(new Error('processor failed after claim'))
+    const onClaimed = vi.fn()
+
+    await expect(
+      processDocumentAsync(
+        'knowledge-base-1',
+        'document-1',
+        {
+          filename: 'a.pdf',
+          fileUrl: 'https://example.com/a.pdf',
+          fileSize: 1,
+          mimeType: 'text/plain',
+        },
+        {},
+        undefined,
+        'request-1',
+        { chargedAtDispatch: true, onClaimed }
+      )
+    ).rejects.toThrow('processor failed after claim')
+
+    expect(onClaimed).toHaveBeenCalledTimes(1)
+    expect(guardForStatusWrite('processing')).toBeDefined()
+    expect(guardForStatusWrite('failed')).toBeDefined()
+  })
+
+  it('accepts a legacy queuedAt-only payload only while the row has no token', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    const processingQueuedAt = new Date('2026-08-24T22:00:00.000Z')
+
+    await processDocumentAsync(
+      'knowledge-base-1',
+      'document-1',
+      {
+        filename: 'a.pdf',
+        fileUrl: 'https://example.com/a.pdf',
+        fileSize: 1,
+        mimeType: 'text/plain',
+      },
+      {},
+      undefined,
+      'request-1',
+      { chargedAtDispatch: true, processingQueuedAt }
+    )
+
+    const claimGuard = guardForStatusWrite('processing')
+    expect(
+      hasMockCondition(
+        claimGuard,
+        (node: MockCondition) =>
+          node.type === 'eq' &&
+          node.left === schemaMock.document.processingQueuedAt &&
+          node.right === processingQueuedAt
+      )
+    ).toBe(true)
+    expect(
+      hasMockCondition(
+        claimGuard,
+        (node: MockCondition) =>
+          node.type === 'isNull' && node.column === schemaMock.document.processingQueueToken
+      )
+    ).toBe(true)
+  })
+
+  it('accepts a pre-rollout payload only while the row has no token', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+
+    await processDocumentAsync(
+      'knowledge-base-1',
+      'document-1',
+      {
+        filename: 'a.pdf',
+        fileUrl: 'https://example.com/a.pdf',
+        fileSize: 1,
+        mimeType: 'text/plain',
+      },
+      {},
+      undefined,
+      'request-1',
+      { chargedAtDispatch: false }
+    )
+
+    const claimGuard = guardForStatusWrite('processing')
+    expect(
+      hasMockCondition(
+        claimGuard,
+        (node: MockCondition) =>
+          node.type === 'isNull' && node.column === schemaMock.document.processingQueueToken
+      )
+    ).toBe(true)
+    expect(
+      hasMockCondition(
+        claimGuard,
+        (node: MockCondition) =>
+          node.type === 'eq' && node.left === schemaMock.document.processingQueuedAt
+      )
+    ).toBe(false)
+  })
+
+  it('uses the queue token as the authoritative claim and final-write generation', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+
+    await processDocumentAsync(
+      'knowledge-base-1',
+      'document-1',
+      {
+        filename: 'a.pdf',
+        fileUrl: 'https://example.com/a.pdf',
+        fileSize: 1,
+        mimeType: 'text/plain',
+      },
+      {},
+      undefined,
+      'request-1',
+      {
+        chargedAtDispatch: true,
+        processingQueueToken: 'request-1',
+        processingQueuedAt: new Date('2026-08-24T22:00:00.000Z'),
+      }
+    )
+
+    for (const status of ['processing', 'completed']) {
+      const guard = guardForStatusWrite(status)
+      expect(
+        hasMockCondition(
+          guard,
+          (node: MockCondition) =>
+            node.type === 'eq' &&
+            node.left === schemaMock.document.processingQueueToken &&
+            node.right === 'request-1'
+        )
+      ).toBe(true)
+      expect(
+        hasMockCondition(
+          guard,
+          (node: MockCondition) =>
+            node.type === 'eq' &&
+            node.left === schemaMock.document.userExcluded &&
+            node.right === false
+        )
+      ).toBe(true)
+    }
+
+    const completion = dbChainMockFns.set.mock.calls.find(
+      (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === 'completed'
+    )
+    expect(completion?.[0]).toMatchObject({
+      processingQueueToken: null,
+      processingQueuedAt: null,
+    })
+  })
+
   it('does not process or bill a document it failed to claim', async () => {
     dbChainMockFns.limit
       .mockResolvedValueOnce([PERSISTED_CONTEXT])
@@ -391,4 +598,432 @@ describe('processDocumentAsync write guards', () => {
     expect(completion).toBeDefined()
     expect((completion?.[0] as Record<string, unknown>).processingAttempts).toBe(0)
   })
+
+  it('records a mutable usage-limit failure and refunds a charged dispatch attempt', async () => {
+    dbChainMockFns.limit.mockResolvedValueOnce([PERSISTED_CONTEXT])
+    mockCheckActorUsageLimits.mockResolvedValue({
+      isExceeded: true,
+      message: 'Usage limit exceeded. Upgrade to continue.',
+    })
+
+    await expect(
+      processDocumentAsync(
+        'knowledge-base-1',
+        'document-1',
+        {
+          filename: 'a.pdf',
+          fileUrl: 'https://example.com/a.pdf',
+          fileSize: 1,
+          mimeType: 'text/plain',
+        },
+        {},
+        undefined,
+        undefined,
+        {
+          chargedAtDispatch: true,
+          processingQueuedAt: new Date('2026-08-24T22:00:00.000Z'),
+        }
+      )
+    ).rejects.toBeInstanceOf(UsageLimitDocumentProcessingError)
+
+    expect(mockProcessDocument).not.toHaveBeenCalled()
+    const failure = dbChainMockFns.set.mock.calls.find(
+      (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === 'failed'
+    )
+    expect(failure?.[0]).toMatchObject({
+      processingError: 'Usage limit exceeded. Upgrade to continue.',
+    })
+    const attempts = (failure?.[0] as Record<string, unknown>).processingAttempts as {
+      toSQL: () => { params: unknown[]; sql: string }
+    }
+    expect(attempts.toSQL().sql).toBe('GREATEST(? - 1, 0)')
+    expect(attempts.toSQL().params).toEqual([schemaMock.document.processingAttempts])
+  })
+
+  it('fails before embedding when a stored chunk exceeds the model input ceiling', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    mockGetEmbeddingModelInfo.mockReturnValue({
+      tokenizerProvider: 'openai',
+      maxInputTokens: 1,
+    })
+    mockProcessDocument.mockResolvedValue({
+      chunks: [
+        {
+          text: 'This chunk is too large for the selected embedding model.',
+          metadata: { startIndex: 0, endIndex: 57 },
+        },
+      ],
+      metadata: { chunkCount: 1, tokenCount: 12, characterCount: 57 },
+    })
+
+    await expect(
+      processDocumentAsync('knowledge-base-1', 'document-1', {
+        filename: 'large.txt',
+        fileUrl: 'https://example.com/large.txt',
+        fileSize: 57,
+        mimeType: 'text/plain',
+      })
+    ).rejects.toMatchObject({
+      name: 'PermanentDocumentProcessingError',
+      code: 'document_complexity_limit',
+    })
+
+    expect(mockGenerateEmbeddings).not.toHaveBeenCalled()
+  })
+
+  it('dead-letters deterministic input failures after recording an actionable reason', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    mockProcessDocument.mockRejectedValue(
+      new PermanentDocumentProcessingError(
+        'encrypted_file',
+        'This file is encrypted or password-protected. Remove the protection and retry.'
+      )
+    )
+
+    await expect(
+      processDocumentAsync('knowledge-base-1', 'document-1', {
+        filename: 'protected.xlsx',
+        fileUrl: 'https://example.com/protected.xlsx',
+        fileSize: 1,
+        mimeType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+      })
+    ).rejects.toMatchObject({
+      code: 'encrypted_file',
+    })
+
+    const failure = dbChainMockFns.set.mock.calls.find(
+      (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === 'failed'
+    )
+    expect(failure?.[0]).toMatchObject({
+      processingError:
+        'This file is encrypted or password-protected. Remove the protection and retry.',
+      processingAttempts: 5,
+    })
+  })
+
+  it('keeps infrastructure failures eligible for automatic recovery', async () => {
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    mockProcessDocument.mockRejectedValue(new Error('Storage request timed out'))
+
+    await expect(
+      processDocumentAsync('knowledge-base-1', 'document-1', {
+        filename: 'report.docx',
+        fileUrl: 'https://example.com/report.docx',
+        fileSize: 1,
+        mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      })
+    ).rejects.toThrow('Storage request timed out')
+
+    const failure = dbChainMockFns.set.mock.calls.find(
+      (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === 'failed'
+    )
+    expect(failure).toBeDefined()
+    expect(failure![0]).not.toHaveProperty('processingAttempts')
+  })
+
+  it.each([
+    { chargedAtDispatch: true, refundsAttempt: true },
+    { chargedAtDispatch: false, refundsAttempt: false },
+  ])(
+    'refunds only a charged document attempt when provider credit is exhausted',
+    async ({ chargedAtDispatch, refundsAttempt }) => {
+      dbChainMockFns.limit
+        .mockResolvedValueOnce([PERSISTED_CONTEXT])
+        .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+        .mockResolvedValueOnce([{ id: 'document-1' }])
+      mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+      mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+        new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+      )
+      mockProcessDocument.mockResolvedValue({
+        chunks: [{ text: 'Index me', metadata: { startIndex: 0, endIndex: 8 } }],
+        metadata: { chunkCount: 1, tokenCount: 2, characterCount: 8 },
+      })
+      mockGenerateEmbeddings.mockRejectedValue(new EmbeddingQuotaExhaustedError('openai'))
+
+      await expect(
+        processDocumentAsync(
+          'knowledge-base-1',
+          'document-1',
+          {
+            filename: 'report.docx',
+            fileUrl: 'https://example.com/report.docx',
+            fileSize: 1,
+            mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+          },
+          {},
+          undefined,
+          undefined,
+          {
+            chargedAtDispatch,
+            processingQueuedAt: new Date('2026-08-24T22:00:00.000Z'),
+          }
+        )
+      ).rejects.toBeInstanceOf(EmbeddingQuotaExhaustedError)
+
+      const failure = dbChainMockFns.set.mock.calls.find(
+        (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === 'failed'
+      )
+      expect(failure).toBeDefined()
+      expect(failure?.[0]).toMatchObject({
+        processingError: EMBEDDING_QUOTA_EXHAUSTED_MESSAGE,
+      })
+      const attempts = (failure![0] as Record<string, unknown>).processingAttempts as
+        | { toSQL: () => { params: unknown[]; sql: string } }
+        | undefined
+      if (refundsAttempt) {
+        expect(attempts?.toSQL().sql).toBe('GREATEST(? - 1, 0)')
+        expect(attempts?.toSQL().params).toEqual([schemaMock.document.processingAttempts])
+      } else {
+        expect(failure![0]).not.toHaveProperty('processingAttempts')
+      }
+    }
+  )
+})
+
+describe('in-process quota continuation dispatch', () => {
+  const envSnapshot = { ...env }
+  const queuedDocument = {
+    documentId: 'document-1',
+    filename: PERSISTED_CONTEXT.filename,
+    fileUrl: PERSISTED_CONTEXT.fileUrl,
+    fileSize: PERSISTED_CONTEXT.fileSize,
+    mimeType: PERSISTED_CONTEXT.mimeType,
+  }
+
+  beforeEach(() => {
+    vi.clearAllMocks()
+    resetDbChainMock()
+    resetInsideTriggerRunForTests()
+    setEnvFlags({ isTriggerDevEnabled: false })
+    for (const key of Object.keys(env)) delete (env as Record<string, unknown>)[key]
+    Object.assign(env, { ...defaultMockEnv, TRIGGER_SECRET_KEY: undefined })
+    dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }])
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([{ userId: 'knowledge-owner', workspaceId: null }])
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false })
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    mockProcessDocument.mockResolvedValue({
+      chunks: [{ text: 'Index me', metadata: { startIndex: 0, endIndex: 8 } }],
+      metadata: { chunkCount: 1, tokenCount: 2, characterCount: 8 },
+    })
+    mockGetEmbeddingModelInfo.mockReturnValue({
+      tokenizerProvider: 'openai',
+      maxInputTokens: 8191,
+    })
+    mockGenerateEmbeddings.mockRejectedValue(new EmbeddingQuotaExhaustedError('openai'))
+    mockTrigger.mockResolvedValue({ id: 'quota-continuation-run' })
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+    resetInsideTriggerRunForTests()
+    resetEnvFlagsMock()
+  })
+
+  afterAll(() => {
+    for (const key of Object.keys(env)) delete (env as Record<string, unknown>)[key]
+    Object.assign(env, envSnapshot)
+  })
+
+  it('durably defers quota exhaustion in direct mode before reporting acceptance', async () => {
+    vi.spyOn(Date, 'now').mockReturnValue(1_000)
+
+    await expect(
+      processDocumentsWithQueue([queuedDocument], 'knowledge-base-1', {}, 'request-1', undefined)
+    ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] })
+
+    expect(mockTrigger).toHaveBeenCalledWith(
+      'knowledge-process-document',
+      expect.objectContaining({
+        documentId: 'document-1',
+        processingQueuedAt: expect.any(String),
+        quotaRetryCount: 1,
+      }),
+      expect.objectContaining({
+        idempotencyKey: 'knowledge-quota-document-1-request-1-1',
+        delay: expect.any(Date),
+      })
+    )
+
+    const deferredUntil = mockTrigger.mock.calls[0]?.[2]?.delay as Date
+    expect(deferredUntil.getTime()).toBeGreaterThanOrEqual(1_000 + 5 * 60 * 1000 * 0.8)
+    expect(deferredUntil.getTime()).toBeLessThanOrEqual(1_000 + 5 * 60 * 1000 * 1.2)
+
+    const deferredWriteIndex = dbChainMockFns.set.mock.calls.findIndex(
+      (call) =>
+        (call[0] as Record<string, unknown> | undefined)?.processingDeferredUntil instanceof Date
+    )
+    expect(deferredWriteIndex).toBeGreaterThanOrEqual(0)
+    expect(dbChainMockFns.set.mock.calls[deferredWriteIndex]?.[0]).toMatchObject({
+      processingStatus: 'pending',
+      processingQueuedAt: deferredUntil,
+      processingStartedAt: null,
+      processingDeferredUntil: deferredUntil,
+      processingCompletedAt: null,
+      processingError: null,
+    })
+    expect(mockTrigger.mock.invocationCallOrder[0]).toBeLessThan(
+      dbChainMockFns.set.mock.invocationCallOrder[deferredWriteIndex]
+    )
+  })
+
+  it('preserves a tokenless queue stamp across an accepted quota continuation', async () => {
+    const originalQueuedAt = new Date('2026-08-24T22:00:00.000Z')
+    const deferredUntil = new Date('2026-08-24T23:00:00.000Z')
+    resetDbChainMock()
+    dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }])
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+    mockGetFileMetadataByKeys.mockResolvedValue([SOURCE_BINDING])
+    mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
+      new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
+    )
+    mockProcessDocument.mockResolvedValue({
+      chunks: [{ text: 'Index me', metadata: { startIndex: 0, endIndex: 8 } }],
+      metadata: { chunkCount: 1, tokenCount: 2, characterCount: 8 },
+    })
+    mockGenerateEmbeddings.mockRejectedValue(new EmbeddingQuotaExhaustedError('openai'))
+
+    await expect(
+      processDocumentAsync(
+        'knowledge-base-1',
+        'document-1',
+        {
+          filename: 'report.docx',
+          fileUrl: 'https://example.com/report.docx',
+          fileSize: 1,
+          mimeType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        },
+        {},
+        undefined,
+        'request-1',
+        {
+          chargedAtDispatch: false,
+          processingQueuedAt: originalQueuedAt,
+          scheduleQuotaContinuation: vi.fn().mockResolvedValue(deferredUntil),
+        }
+      )
+    ).rejects.toBeInstanceOf(EmbeddingQuotaExhaustedError)
+
+    const deferredWrite = dbChainMockFns.set.mock.calls.find(
+      (call) =>
+        (call[0] as Record<string, unknown> | undefined)?.processingDeferredUntil === deferredUntil
+    )
+    expect(deferredWrite?.[0]).toMatchObject({
+      processingStatus: 'pending',
+      processingDeferredUntil: deferredUntil,
+    })
+    expect(deferredWrite?.[0]).not.toHaveProperty('processingQueuedAt')
+    expect(
+      dbChainMockFns.where.mock.calls.some((call) =>
+        hasMockCondition(
+          call[0],
+          (node) =>
+            node.type === 'eq' &&
+            node.left === schemaMock.document.processingQueuedAt &&
+            node.right === originalQueuedAt
+        )
+      )
+    ).toBe(true)
+  })
+
+  it('stops automatic retries after the bounded quota continuation chain is exhausted', async () => {
+    resetDbChainMock()
+    dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }])
+    dbChainMockFns.limit
+      .mockResolvedValueOnce([PERSISTED_CONTEXT])
+      .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
+      .mockResolvedValueOnce([{ id: 'document-1' }])
+
+    await expect(
+      processDocumentAsync(
+        'knowledge-base-1',
+        'document-1',
+        queuedDocument,
+        {},
+        undefined,
+        'request-1',
+        {
+          chargedAtDispatch: false,
+          processingQueuedAt: new Date('2026-08-24T22:00:00.000Z'),
+          quotaContinuationExhausted: true,
+        }
+      )
+    ).rejects.toBeInstanceOf(EmbeddingQuotaExhaustedError)
+
+    const failure = dbChainMockFns.set.mock.calls.find(
+      (call) => (call[0] as Record<string, unknown> | undefined)?.processingStatus === 'failed'
+    )
+    expect(failure?.[0]).toMatchObject({
+      processingError: EMBEDDING_QUOTA_EXHAUSTED_MESSAGE,
+      processingAttempts: MAX_PROCESSING_ATTEMPTS,
+    })
+    expect(mockTrigger).not.toHaveBeenCalled()
+  })
+
+  it('durably defers quota exhaustion after a failed Trigger batch fallback', async () => {
+    markInsideTriggerRun()
+    mockBatchTrigger.mockRejectedValue(new Error('batch unavailable'))
+
+    await expect(
+      processDocumentsWithQueue([queuedDocument], 'knowledge-base-1', {}, 'request-1', undefined)
+    ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] })
+
+    expect(mockBatchTrigger).toHaveBeenCalledTimes(1)
+    expect(mockTrigger).toHaveBeenCalledWith(
+      'knowledge-process-document',
+      expect.objectContaining({ documentId: 'document-1', quotaRetryCount: 1 }),
+      expect.objectContaining({
+        idempotencyKey: 'knowledge-quota-document-1-request-1-1',
+      })
+    )
+  })
+
+  it('keeps a claimed direct dispatch accepted when quota continuation handoff fails', async () => {
+    mockTrigger.mockRejectedValue(new Error('continuation unavailable'))
+
+    await expect(
+      processDocumentsWithQueue([queuedDocument], 'knowledge-base-1', {}, 'request-1', undefined)
+    ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] })
+
+    const failure = dbChainMockFns.set.mock.calls.find(
+      (call) =>
+        (call[0] as Record<string, unknown> | undefined)?.processingError ===
+        'continuation unavailable'
+    )
+    expect(failure?.[0]).toMatchObject({
+      processingStatus: 'failed',
+      processingDeferredUntil: null,
+    })
+  })
 })

apps/sim/lib/knowledge/documents/document-processor-chunk-limit.test.ts

@@ -0,0 +1,98 @@
+/**
+ * @vitest-environment node
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockParseBuffer } = vi.hoisted(() => ({
+  mockParseBuffer: vi.fn(),
+}))
+
+vi.mock('@/lib/file-parsers', () => ({
+  parseBuffer: mockParseBuffer,
+}))
+
+import { ChunkLimitExceededError } from '@/lib/chunkers/chunk-budget'
+import { TokenChunker } from '@/lib/chunkers/token-chunker'
+import {
+  MAX_DOCUMENT_CHUNKS,
+  type PermanentDocumentProcessingError,
+} from '@/lib/knowledge/documents/document-processing-error'
+import { processDocument } from '@/lib/knowledge/documents/document-processor'
+
+describe('document chunk production ceiling', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('translates the shared chunk limit once into a permanent complexity failure', async () => {
+    mockParseBuffer.mockResolvedValue({
+      content: Array.from({ length: MAX_DOCUMENT_CHUNKS + 1 }, () => 'word').join(' '),
+      metadata: {},
+    })
+
+    const processing = processDocument(
+      'data:text/plain;base64,dGVzdA==',
+      'large.txt',
+      'text/plain',
+      1,
+      0,
+      1,
+      undefined,
+      undefined,
+      'token'
+    )
+
+    await expect(processing).rejects.toMatchObject({
+      name: 'PermanentDocumentProcessingError',
+      code: 'document_complexity_limit',
+      cause: expect.any(ChunkLimitExceededError),
+    } satisfies Partial<PermanentDocumentProcessingError>)
+  })
+
+  it('never completes nonempty content with zero chunks', async () => {
+    mockParseBuffer.mockResolvedValue({ content: 'x', metadata: {} })
+    vi.spyOn(TokenChunker.prototype, 'chunk').mockResolvedValue([])
+
+    await expect(
+      processDocument(
+        'data:text/plain;base64,eA==',
+        'filtered.txt',
+        'text/plain',
+        100,
+        0,
+        10,
+        undefined,
+        undefined,
+        'token'
+      )
+    ).rejects.toMatchObject({
+      name: 'PermanentDocumentProcessingError',
+      code: 'no_extractable_text',
+    } satisfies Partial<PermanentDocumentProcessingError>)
+  })
+
+  it('preserves bounded partial indexing for a parser-limited document', async () => {
+    const content = 'name,value\nfirst,1\nsecond,2\n[Content truncated: showing first 1,000 rows]'
+    mockParseBuffer.mockResolvedValue({
+      content,
+      metadata: { truncated: true, headers: ['name', 'value'], rowCount: 2_000 },
+    })
+
+    const result = await processDocument(
+      'data:text/csv;base64,dGVzdA==',
+      'large.csv',
+      'text/csv',
+      100,
+      0,
+      10
+    )
+
+    expect(result.chunks.length).toBeGreaterThan(0)
+    expect(result.metadata.characterCount).toBe(content.length)
+  })
+})