improvement(kb): improve chunkers, respect user-specified chunk configurations, added tests (#2539)

* improvement(kb): improve chunkers, respect user-specified chunk configurations, added tests

* ack PR commnets

* updated docs

* cleanup

apps/docs/content/docs/en/knowledgebase/index.mdx

@@ -34,9 +34,15 @@ Once your documents are processed, you can view and edit the individual chunks.
 <Image src="/static/knowledgebase/knowledgebase.png" alt="Document chunks view showing processed content" width={800} height={500} />
 
 ### Chunk Configuration
-- **Default chunk size**: 1,024 characters
-- **Configurable range**: 100-4,000 characters per chunk
-- **Smart overlap**: 200 characters by default for context preservation
+
+When creating a knowledge base, you can configure how documents are split into chunks:
+
+| Setting | Unit | Default | Range | Description |
+|---------|------|---------|-------|-------------|
+| **Max Chunk Size** | tokens | 1,024 | 100-4,000 | Maximum size of each chunk (1 token ≈ 4 characters) |
+| **Min Chunk Size** | characters | 1 | 1-2,000 | Minimum chunk size to avoid tiny fragments |
+| **Overlap** | characters | 200 | 0-500 | Context overlap between consecutive chunks |
+
 - **Hierarchical splitting**: Respects document structure (sections, paragraphs, sentences)
 
 ### Editing Capabilities

apps/docs/content/docs/en/tools/memory.mdx

@@ -1,6 +1,6 @@
 ---
 title: Memory
-description: Store and retrieve conversation history
+description: Add memory store
 ---
 
 import { BlockInfoCard } from "@/components/ui/block-info-card"
@@ -10,94 +10,95 @@ import { BlockInfoCard } from "@/components/ui/block-info-card"
   color="#F64F9E"
 />
 
-## Overview
+## Usage Instructions
+
+Integrate Memory into the workflow. Can add, get a memory, get all memories, and delete memories.
 
-The Memory block stores conversation history for agents. Each memory is identified by a `conversationId` that you provide. Multiple agents can share the same memory by using the same `conversationId`.
 
-Memory stores only user and assistant messages. System messages are not stored—they are configured in the Agent block and prefixed at runtime.
 
 ## Tools
 
 ### `memory_add`
 
-Add a message to memory. Creates a new memory if the `conversationId` doesn't exist, or appends to existing memory.
+Add a new memory to the database or append to existing memory with the same ID.
 
 #### Input
 
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `conversationId` | string | Yes | Unique identifier for the conversation (e.g., `user-123`, `session-abc`) |
-| `role` | string | Yes | Message role: `user` or `assistant` |
-| `content` | string | Yes | Message content |
+| `conversationId` | string | No | Conversation identifier \(e.g., user-123, session-abc\). If a memory with this conversationId already exists, the new message will be appended to it. |
+| `id` | string | No | Legacy parameter for conversation identifier. Use conversationId instead. Provided for backwards compatibility. |
+| `role` | string | Yes | Role for agent memory \(user, assistant, or system\) |
+| `content` | string | Yes | Content for agent memory |
 
 #### Output
 
 | Parameter | Type | Description |
 | --------- | ---- | ----------- |
-| `success` | boolean | Whether the operation succeeded |
-| `memories` | array | Updated memory array |
-| `error` | string | Error message if failed |
+| `success` | boolean | Whether the memory was added successfully |
+| `memories` | array | Array of memory objects including the new or updated memory |
+| `error` | string | Error message if operation failed |
 
 ### `memory_get`
 
-Retrieve memory by conversation ID.
+Retrieve memory by conversationId. Returns matching memories.
 
 #### Input
 
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `conversationId` | string | Yes | Conversation identifier |
+| `conversationId` | string | No | Conversation identifier \(e.g., user-123, session-abc\). Returns memories for this conversation. |
+| `id` | string | No | Legacy parameter for conversation identifier. Use conversationId instead. Provided for backwards compatibility. |
 
 #### Output
 
 | Parameter | Type | Description |
 | --------- | ---- | ----------- |
-| `success` | boolean | Whether the operation succeeded |
-| `memories` | array | Array of messages with `role` and `content` |
-| `error` | string | Error message if failed |
+| `success` | boolean | Whether the memory was retrieved successfully |
+| `memories` | array | Array of memory objects with conversationId and data fields |
+| `message` | string | Success or error message |
+| `error` | string | Error message if operation failed |
 
 ### `memory_get_all`
 
-Retrieve all memories for the current workspace.
-
-#### Output
-
-| Parameter | Type | Description |
-| --------- | ---- | ----------- |
-| `success` | boolean | Whether the operation succeeded |
-| `memories` | array | All memory objects with `conversationId` and `data` fields |
-| `error` | string | Error message if failed |
-
-### `memory_delete`
-
-Delete memory by conversation ID.
+Retrieve all memories from the database
 
 #### Input
 
 | Parameter | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `conversationId` | string | Yes | Conversation identifier to delete |
 
 #### Output
 
 | Parameter | Type | Description |
 | --------- | ---- | ----------- |
-| `success` | boolean | Whether the operation succeeded |
-| `message` | string | Confirmation message |
-| `error` | string | Error message if failed |
+| `success` | boolean | Whether all memories were retrieved successfully |
+| `memories` | array | Array of all memory objects with key, conversationId, and data fields |
+| `message` | string | Success or error message |
+| `error` | string | Error message if operation failed |
 
-## Agent Memory Types
+### `memory_delete`
+
+Delete memories by conversationId.
+
+#### Input
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `conversationId` | string | No | Conversation identifier \(e.g., user-123, session-abc\). Deletes all memories for this conversation. |
+| `id` | string | No | Legacy parameter for conversation identifier. Use conversationId instead. Provided for backwards compatibility. |
+
+#### Output
+
+| Parameter | Type | Description |
+| --------- | ---- | ----------- |
+| `success` | boolean | Whether the memory was deleted successfully |
+| `message` | string | Success or error message |
+| `error` | string | Error message if operation failed |
 
-When using memory with an Agent block, you can configure how conversation history is managed:
 
-| Type | Description |
-| ---- | ----------- |
-| **Full Conversation** | Stores all messages, limited by model's context window (uses 90% to leave room for response) |
-| **Sliding Window (Messages)** | Keeps the last N messages (default: 10) |
-| **Sliding Window (Tokens)** | Keeps messages that fit within a token limit (default: 4000) |
 
 ## Notes
 
-- Memory is scoped per workspace—workflows in the same workspace share the memory store
-- Use unique `conversationId` values to keep conversations separate (e.g., session IDs, user IDs, or UUIDs)
-- System messages belong in the Agent block configuration, not in memory
+- Category: `blocks`
+- Type: `memory`

apps/sim/app/api/knowledge/[id]/documents/route.ts

@@ -34,13 +34,24 @@ const CreateDocumentSchema = z.object({
   documentTagsData: z.string().optional(),
 })
 
+/**
+ * Schema for bulk document creation with processing options
+ *
+ * Processing options units:
+ * - chunkSize: tokens (1 token ≈ 4 characters)
+ * - minCharactersPerChunk: characters
+ * - chunkOverlap: characters
+ */
 const BulkCreateDocumentsSchema = z.object({
   documents: z.array(CreateDocumentSchema),
   processingOptions: z.object({
+    /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
     chunkSize: z.number().min(100).max(4000),
+    /** Minimum chunk size in characters */
     minCharactersPerChunk: z.number().min(1).max(2000),
     recipe: z.string(),
     lang: z.string(),
+    /** Overlap between chunks in characters */
     chunkOverlap: z.number().min(0).max(500),
   }),
   bulk: z.literal(true),

apps/sim/app/api/knowledge/[id]/route.ts

@@ -12,6 +12,14 @@ import { checkKnowledgeBaseAccess, checkKnowledgeBaseWriteAccess } from '@/app/a
 
 const logger = createLogger('KnowledgeBaseByIdAPI')
 
+/**
+ * Schema for updating a knowledge base
+ *
+ * Chunking config units:
+ * - maxSize: tokens (1 token ≈ 4 characters)
+ * - minSize: characters
+ * - overlap: tokens (1 token ≈ 4 characters)
+ */
 const UpdateKnowledgeBaseSchema = z.object({
   name: z.string().min(1, 'Name is required').optional(),
   description: z.string().optional(),
@@ -20,10 +28,23 @@ const UpdateKnowledgeBaseSchema = z.object({
   workspaceId: z.string().nullable().optional(),
   chunkingConfig: z
     .object({
-      maxSize: z.number(),
-      minSize: z.number(),
-      overlap: z.number(),
+      /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
+      maxSize: z.number().min(100).max(4000),
+      /** Minimum chunk size in characters */
+      minSize: z.number().min(1).max(2000),
+      /** Overlap between chunks in characters */
+      overlap: z.number().min(0).max(500),
     })
+    .refine(
+      (data) => {
+        // Convert maxSize from tokens to characters for comparison (1 token ≈ 4 chars)
+        const maxSizeInChars = data.maxSize * 4
+        return data.minSize < maxSizeInChars
+      },
+      {
+        message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
+      }
+    )
     .optional(),
 })
 

apps/sim/app/api/knowledge/route.test.ts

@@ -139,8 +139,8 @@ describe('Knowledge Base API Route', () => {
       const invalidData = {
         name: 'Test KB',
         chunkingConfig: {
-          maxSize: 100,
-          minSize: 200, // Invalid: minSize > maxSize
+          maxSize: 100, // 100 tokens = 400 characters
+          minSize: 500, // Invalid: minSize (500 chars) > maxSize (400 chars)
           overlap: 50,
         },
       }
@@ -168,7 +168,7 @@ describe('Knowledge Base API Route', () => {
       expect(data.data.embeddingDimension).toBe(1536)
       expect(data.data.chunkingConfig).toEqual({
         maxSize: 1024,
-        minSize: 1,
+        minSize: 100,
         overlap: 200,
       })
     })

apps/sim/app/api/knowledge/route.ts

@@ -7,6 +7,14 @@ import { createLogger } from '@/lib/logs/console/logger'
 
 const logger = createLogger('KnowledgeBaseAPI')
 
+/**
+ * Schema for creating a knowledge base
+ *
+ * Chunking config units:
+ * - maxSize: tokens (1 token ≈ 4 characters)
+ * - minSize: characters
+ * - overlap: tokens (1 token ≈ 4 characters)
+ */
 const CreateKnowledgeBaseSchema = z.object({
   name: z.string().min(1, 'Name is required'),
   description: z.string().optional(),
@@ -15,18 +23,28 @@ const CreateKnowledgeBaseSchema = z.object({
   embeddingDimension: z.literal(1536).default(1536),
   chunkingConfig: z
     .object({
+      /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
       maxSize: z.number().min(100).max(4000).default(1024),
-      minSize: z.number().min(1).max(2000).default(1),
+      /** Minimum chunk size in characters */
+      minSize: z.number().min(1).max(2000).default(100),
+      /** Overlap between chunks in tokens (1 token ≈ 4 characters) */
       overlap: z.number().min(0).max(500).default(200),
     })
     .default({
       maxSize: 1024,
-      minSize: 1,
+      minSize: 100,
       overlap: 200,
     })
-    .refine((data) => data.minSize < data.maxSize, {
-      message: 'Min chunk size must be less than max chunk size',
-    }),
+    .refine(
+      (data) => {
+        // Convert maxSize from tokens to characters for comparison (1 token ≈ 4 chars)
+        const maxSizeInChars = data.maxSize * 4
+        return data.minSize < maxSizeInChars
+      },
+      {
+        message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
+      }
+    ),
 })
 
 export async function GET(req: NextRequest) {

apps/sim/app/api/knowledge/utils.test.ts

@@ -183,7 +183,12 @@ describe('Knowledge Utils', () => {
 
   describe('processDocumentAsync', () => {
     it.concurrent('should insert embeddings before updating document counters', async () => {
-      kbRows.push({ id: 'kb1', userId: 'user1', workspaceId: null })
+      kbRows.push({
+        id: 'kb1',
+        userId: 'user1',
+        workspaceId: null,
+        chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 200 },
+      })
       docRows.push({ id: 'doc1', knowledgeBaseId: 'kb1' })
 
       await processDocumentAsync(

apps/sim/app/workspace/[workspaceId]/knowledge/components/create-base-modal/create-base-modal.tsx

@@ -44,23 +44,33 @@ const FormSchema = z
       .max(100, 'Name must be less than 100 characters')
       .refine((value) => value.trim().length > 0, 'Name cannot be empty'),
     description: z.string().max(500, 'Description must be less than 500 characters').optional(),
+    /** Minimum chunk size in characters */
     minChunkSize: z
       .number()
-      .min(1, 'Min chunk size must be at least 1')
-      .max(2000, 'Min chunk size must be less than 2000'),
+      .min(1, 'Min chunk size must be at least 1 character')
+      .max(2000, 'Min chunk size must be less than 2000 characters'),
+    /** Maximum chunk size in tokens (1 token ≈ 4 characters) */
     maxChunkSize: z
       .number()
-      .min(100, 'Max chunk size must be at least 100')
-      .max(4000, 'Max chunk size must be less than 4000'),
+      .min(100, 'Max chunk size must be at least 100 tokens')
+      .max(4000, 'Max chunk size must be less than 4000 tokens'),
+    /** Overlap between chunks in tokens */
     overlapSize: z
       .number()
-      .min(0, 'Overlap size must be non-negative')
-      .max(500, 'Overlap size must be less than 500'),
-  })
-  .refine((data) => data.minChunkSize < data.maxChunkSize, {
-    message: 'Min chunk size must be less than max chunk size',
-    path: ['minChunkSize'],
+      .min(0, 'Overlap must be non-negative')
+      .max(500, 'Overlap must be less than 500 tokens'),
   })
+  .refine(
+    (data) => {
+      // Convert maxChunkSize from tokens to characters for comparison (1 token ≈ 4 chars)
+      const maxChunkSizeInChars = data.maxChunkSize * 4
+      return data.minChunkSize < maxChunkSizeInChars
+    },
+    {
+      message: 'Min chunk size (characters) must be less than max chunk size (tokens × 4)',
+      path: ['minChunkSize'],
+    }
+  )
 
 type FormValues = z.infer<typeof FormSchema>
 
@@ -123,7 +133,7 @@ export function CreateBaseModal({
     defaultValues: {
       name: '',
       description: '',
-      minChunkSize: 1,
+      minChunkSize: 100,
       maxChunkSize: 1024,
       overlapSize: 200,
     },
@@ -143,7 +153,7 @@ export function CreateBaseModal({
       reset({
         name: '',
         description: '',
-        minChunkSize: 1,
+        minChunkSize: 100,
         maxChunkSize: 1024,
         overlapSize: 200,
       })
@@ -381,10 +391,10 @@ export function CreateBaseModal({
                 <div className='space-y-[12px] rounded-[6px] bg-[var(--surface-6)] px-[12px] py-[14px]'>
                   <div className='grid grid-cols-2 gap-[12px]'>
                     <div className='flex flex-col gap-[8px]'>
-                      <Label htmlFor='minChunkSize'>Min Chunk Size</Label>
+                      <Label htmlFor='minChunkSize'>Min Chunk Size (characters)</Label>
                       <Input
                         id='minChunkSize'
-                        placeholder='1'
+                        placeholder='100'
                         {...register('minChunkSize', { valueAsNumber: true })}
                         className={cn(errors.minChunkSize && 'border-[var(--text-error)]')}
                         autoComplete='off'
@@ -394,7 +404,7 @@ export function CreateBaseModal({
                     </div>
 
                     <div className='flex flex-col gap-[8px]'>
-                      <Label htmlFor='maxChunkSize'>Max Chunk Size</Label>
+                      <Label htmlFor='maxChunkSize'>Max Chunk Size (tokens)</Label>
                       <Input
                         id='maxChunkSize'
                         placeholder='1024'
@@ -408,7 +418,7 @@ export function CreateBaseModal({
                   </div>
 
                   <div className='flex flex-col gap-[8px]'>
-                    <Label htmlFor='overlapSize'>Overlap Size</Label>
+                    <Label htmlFor='overlapSize'>Overlap (tokens)</Label>
                     <Input
                       id='overlapSize'
                       placeholder='200'
@@ -419,6 +429,9 @@ export function CreateBaseModal({
                       name='overlap-size'
                     />
                   </div>
+                  <p className='text-[11px] text-[var(--text-muted)]'>
+                    1 token ≈ 4 characters. Max chunk size and overlap are in tokens.
+                  </p>
                 </div>
 
                 <div className='flex flex-col gap-[8px]'>

apps/sim/lib/chunkers/docs-chunker.ts

@@ -32,8 +32,8 @@ export class DocsChunker {
     // Use the existing TextChunker for chunking logic
     this.textChunker = new TextChunker({
       chunkSize: options.chunkSize ?? 300, // Max 300 tokens per chunk
-      minChunkSize: options.minChunkSize ?? 1,
-      overlap: options.overlap ?? 50,
+      minCharactersPerChunk: options.minCharactersPerChunk ?? 1,
+      chunkOverlap: options.chunkOverlap ?? 50,
     })
     // Use localhost docs in development, production docs otherwise
     this.baseUrl = options.baseUrl ?? 'https://docs.sim.ai'

apps/sim/lib/chunkers/json-yaml-chunker.ts

@@ -21,19 +21,20 @@ function getTokenCount(text: string): number {
  * Reduced limits to ensure we stay well under OpenAI's 8,191 token limit per embedding request
  */
 const JSON_YAML_CHUNKING_CONFIG = {
-  TARGET_CHUNK_SIZE: 1000, // Target tokens per chunk
-  MIN_CHUNK_SIZE: 100, // Minimum tokens per chunk
+  TARGET_CHUNK_SIZE: 1024, // Target tokens per chunk
+  MIN_CHARACTERS_PER_CHUNK: 100, // Minimum characters per chunk to filter tiny fragments
   MAX_CHUNK_SIZE: 1500, // Maximum tokens per chunk
   MAX_DEPTH_FOR_SPLITTING: 5, // Maximum depth to traverse for splitting
 }
 
 export class JsonYamlChunker {
-  private chunkSize: number
-  private minChunkSize: number
+  private chunkSize: number // in tokens
+  private minCharactersPerChunk: number // in characters
 
   constructor(options: ChunkerOptions = {}) {
-    this.chunkSize = options.chunkSize || JSON_YAML_CHUNKING_CONFIG.TARGET_CHUNK_SIZE
-    this.minChunkSize = options.minChunkSize || JSON_YAML_CHUNKING_CONFIG.MIN_CHUNK_SIZE
+    this.chunkSize = options.chunkSize ?? JSON_YAML_CHUNKING_CONFIG.TARGET_CHUNK_SIZE
+    this.minCharactersPerChunk =
+      options.minCharactersPerChunk ?? JSON_YAML_CHUNKING_CONFIG.MIN_CHARACTERS_PER_CHUNK
   }
 
   /**
@@ -99,7 +100,8 @@ export class JsonYamlChunker {
     const content = JSON.stringify(data, null, 2)
     const tokenCount = getTokenCount(content)
 
-    if (tokenCount >= this.minChunkSize) {
+    // Filter tiny fragments using character count
+    if (content.length >= this.minCharactersPerChunk) {
       chunks.push({
         text: content,
         tokenCount,
@@ -318,7 +320,8 @@ export class JsonYamlChunker {
       }
     }
 
-    if (currentChunk && currentTokens >= this.minChunkSize) {
+    // Filter tiny fragments using character count
+    if (currentChunk && currentChunk.length >= this.minCharactersPerChunk) {
       chunks.push({
         text: currentChunk,
         tokenCount: currentTokens,

apps/sim/lib/chunkers/structured-data-chunker.ts

@@ -3,15 +3,18 @@ import { createLogger } from '@/lib/logs/console/logger'
 
 const logger = createLogger('StructuredDataChunker')
 
-// Configuration for structured data chunking (CSV, XLSX, etc.)
-const STRUCTURED_CHUNKING_CONFIG = {
-  // Target 2000-3000 tokens per chunk for better semantic meaning
-  TARGET_CHUNK_SIZE: 2500,
-  MIN_CHUNK_SIZE: 500,
+/**
+ * Default configuration for structured data chunking (CSV, XLSX, etc.)
+ * These are used when user doesn't provide preferences
+ */
+const DEFAULT_CONFIG = {
+  // Target chunk size in tokens
+  TARGET_CHUNK_SIZE: 1024,
+  MIN_CHUNK_SIZE: 100,
   MAX_CHUNK_SIZE: 4000,
 
   // For spreadsheets, group rows together
-  ROWS_PER_CHUNK: 100, // Start with 100 rows per chunk
+  ROWS_PER_CHUNK: 100,
   MIN_ROWS_PER_CHUNK: 20,
   MAX_ROWS_PER_CHUNK: 500,
 
@@ -22,10 +25,12 @@ const STRUCTURED_CHUNKING_CONFIG = {
 
 /**
  * Smart chunker for structured data (CSV, XLSX) that preserves semantic meaning
+ * Preserves headers in each chunk for better semantic context
  */
 export class StructuredDataChunker {
   /**
    * Chunk structured data intelligently based on rows and semantic boundaries
+   * Respects user's chunkSize preference when provided
    */
   static async chunkStructuredData(
     content: string,
@@ -38,19 +43,24 @@ export class StructuredDataChunker {
       return chunks
     }
 
+    // Use user's chunk size or fall back to default
+    const targetChunkSize = options.chunkSize ?? DEFAULT_CONFIG.TARGET_CHUNK_SIZE
+
     // Detect headers (first line or provided)
     const headerLine = options.headers?.join('\t') || lines[0]
     const dataStartIndex = options.headers ? 0 : 1
 
-    // Calculate optimal rows per chunk based on content
+    // Calculate optimal rows per chunk based on content and user's target size
     const estimatedTokensPerRow = StructuredDataChunker.estimateTokensPerRow(
       lines.slice(dataStartIndex, Math.min(10, lines.length))
     )
-    const optimalRowsPerChunk =
-      StructuredDataChunker.calculateOptimalRowsPerChunk(estimatedTokensPerRow)
+    const optimalRowsPerChunk = StructuredDataChunker.calculateOptimalRowsPerChunk(
+      estimatedTokensPerRow,
+      targetChunkSize
+    )
 
     logger.info(
-      `Structured data chunking: ${lines.length} rows, ~${estimatedTokensPerRow} tokens/row, ${optimalRowsPerChunk} rows/chunk`
+      `Structured data chunking: ${lines.length} rows, ~${estimatedTokensPerRow} tokens/row, ${optimalRowsPerChunk} rows/chunk, target: ${targetChunkSize} tokens`
     )
 
     let currentChunkRows: string[] = []
@@ -66,11 +76,11 @@ export class StructuredDataChunker {
       const projectedTokens =
         currentTokenEstimate +
         rowTokens +
-        (STRUCTURED_CHUNKING_CONFIG.INCLUDE_HEADERS_IN_EACH_CHUNK ? headerTokens : 0)
+        (DEFAULT_CONFIG.INCLUDE_HEADERS_IN_EACH_CHUNK ? headerTokens : 0)
 
       const shouldCreateChunk =
-        (projectedTokens > STRUCTURED_CHUNKING_CONFIG.TARGET_CHUNK_SIZE &&
-          currentChunkRows.length >= STRUCTURED_CHUNKING_CONFIG.MIN_ROWS_PER_CHUNK) ||
+        (projectedTokens > targetChunkSize &&
+          currentChunkRows.length >= DEFAULT_CONFIG.MIN_ROWS_PER_CHUNK) ||
         currentChunkRows.length >= optimalRowsPerChunk
 
       if (shouldCreateChunk && currentChunkRows.length > 0) {
@@ -119,7 +129,7 @@ export class StructuredDataChunker {
     }
 
     // Add headers for context
-    if (STRUCTURED_CHUNKING_CONFIG.INCLUDE_HEADERS_IN_EACH_CHUNK) {
+    if (DEFAULT_CONFIG.INCLUDE_HEADERS_IN_EACH_CHUNK) {
       content += `Headers: ${headerLine}\n`
       content += `${'-'.repeat(Math.min(80, headerLine.length))}\n`
     }
@@ -151,10 +161,9 @@ export class StructuredDataChunker {
 
   /**
    * Estimate tokens in text (rough approximation)
+   * For structured data with numbers, uses 1 token per 3 characters
    */
   private static estimateTokens(text: string): number {
-    // Rough estimate: 1 token per 4 characters for English text
-    // For structured data with numbers, it's closer to 1 token per 3 characters
     return Math.ceil(text.length / 3)
   }
 
@@ -172,14 +181,17 @@ export class StructuredDataChunker {
   }
 
   /**
-   * Calculate optimal rows per chunk based on token estimates
+   * Calculate optimal rows per chunk based on token estimates and target size
    */
-  private static calculateOptimalRowsPerChunk(tokensPerRow: number): number {
-    const optimal = Math.floor(STRUCTURED_CHUNKING_CONFIG.TARGET_CHUNK_SIZE / tokensPerRow)
+  private static calculateOptimalRowsPerChunk(
+    tokensPerRow: number,
+    targetChunkSize: number
+  ): number {
+    const optimal = Math.floor(targetChunkSize / tokensPerRow)
 
     return Math.min(
-      Math.max(optimal, STRUCTURED_CHUNKING_CONFIG.MIN_ROWS_PER_CHUNK),
-      STRUCTURED_CHUNKING_CONFIG.MAX_ROWS_PER_CHUNK
+      Math.max(optimal, DEFAULT_CONFIG.MIN_ROWS_PER_CHUNK),
+      DEFAULT_CONFIG.MAX_ROWS_PER_CHUNK
     )
   }
 

apps/sim/lib/chunkers/text-chunker.test.ts

@@ -0,0 +1,265 @@
+/**
+ * @vitest-environment node
+ */
+
+import { describe, expect, it } from 'vitest'
+import { TextChunker } from './text-chunker'
+
+describe('TextChunker', () => {
+  describe('basic chunking', () => {
+    it.concurrent('should return empty array for empty text', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const chunks = await chunker.chunk('')
+      expect(chunks).toEqual([])
+    })
+
+    it.concurrent('should return empty array for whitespace-only text', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const chunks = await chunker.chunk('   \n\n   ')
+      expect(chunks).toEqual([])
+    })
+
+    it.concurrent('should return single chunk for small text', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'This is a short text.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks).toHaveLength(1)
+      expect(chunks[0].text).toBe(text)
+    })
+
+    it.concurrent('should include token count in chunk metadata', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'Hello world' // ~3 tokens (11 chars / 4)
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks[0].tokenCount).toBe(3)
+    })
+  })
+
+  describe('chunk size limits (tokens)', () => {
+    it.concurrent('should split text that exceeds chunk size', async () => {
+      const chunker = new TextChunker({ chunkSize: 50 })
+      const text = 'This is a test sentence. '.repeat(20)
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(1)
+    })
+
+    it.concurrent('should respect chunk size limit', async () => {
+      const chunkSize = 50
+      const chunker = new TextChunker({ chunkSize })
+      const text = 'This is a test sentence. '.repeat(20)
+      const chunks = await chunker.chunk(text)
+
+      for (const chunk of chunks) {
+        expect(chunk.tokenCount).toBeLessThanOrEqual(chunkSize + 5)
+      }
+    })
+
+    it.concurrent('should create more chunks with smaller chunk size', async () => {
+      const text = 'This is a test sentence. '.repeat(20)
+
+      const largeChunker = new TextChunker({ chunkSize: 200 })
+      const smallChunker = new TextChunker({ chunkSize: 50 })
+
+      const largeChunks = await largeChunker.chunk(text)
+      const smallChunks = await smallChunker.chunk(text)
+
+      expect(smallChunks.length).toBeGreaterThan(largeChunks.length)
+    })
+  })
+
+  describe('overlap (tokens)', () => {
+    it.concurrent('should not add overlap when overlap is 0', async () => {
+      const chunker = new TextChunker({ chunkSize: 20, chunkOverlap: 0 })
+      const text = 'First sentence here. Second sentence here. Third sentence here.'
+      const chunks = await chunker.chunk(text)
+
+      if (chunks.length > 1) {
+        const firstChunkEnd = chunks[0].text.slice(-10)
+        const secondChunkStart = chunks[1].text.slice(0, 10)
+        expect(chunks[1].text.startsWith(firstChunkEnd)).toBe(false)
+      }
+    })
+
+    it.concurrent('should add overlap from previous chunk', async () => {
+      const chunker = new TextChunker({ chunkSize: 20, chunkOverlap: 50 })
+      const text =
+        'First paragraph with some content here.\n\nSecond paragraph with different content here.\n\nThird paragraph with more content.'
+      const chunks = await chunker.chunk(text)
+
+      if (chunks.length > 1) {
+        expect(chunks[1].text.length).toBeGreaterThan(0)
+      }
+    })
+
+    it.concurrent('should handle overlap larger than previous chunk', async () => {
+      const chunker = new TextChunker({ chunkSize: 10, chunkOverlap: 500 })
+      const text = 'Short. Another short.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(0)
+    })
+  })
+
+  describe('hierarchical splitting', () => {
+    it.concurrent('should prefer splitting on paragraph boundaries', async () => {
+      const chunker = new TextChunker({ chunkSize: 30 })
+      const text =
+        'First paragraph content here.\n\nSecond paragraph content here.\n\nThird paragraph content here.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThanOrEqual(1)
+    })
+
+    it.concurrent('should split on sentences when paragraphs are too large', async () => {
+      const chunker = new TextChunker({ chunkSize: 20 })
+      const text =
+        'First sentence. Second sentence. Third sentence. Fourth sentence. Fifth sentence.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(1)
+    })
+
+    it.concurrent('should split on words as last resort', async () => {
+      const chunker = new TextChunker({ chunkSize: 5 })
+      const text = 'Supercalifragilisticexpialidocious is a very long word that needs splitting.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(1)
+    })
+
+    it.concurrent('should handle markdown headings', async () => {
+      const chunker = new TextChunker({ chunkSize: 30 })
+      const text =
+        '# Heading 1\n\nContent under heading 1.\n\n## Heading 2\n\nContent under heading 2.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThanOrEqual(1)
+    })
+  })
+
+  describe('text cleaning', () => {
+    it.concurrent('should normalize Windows line endings', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'Line 1\r\nLine 2\r\nLine 3'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks[0].text).not.toContain('\r')
+    })
+
+    it.concurrent('should collapse multiple spaces', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'Word1    Word2     Word3'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks[0].text).not.toContain('  ')
+    })
+
+    it.concurrent('should limit consecutive newlines', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'Para 1\n\n\n\n\nPara 2'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks[0].text).not.toContain('\n\n\n')
+    })
+  })
+
+  describe('chunk metadata', () => {
+    it.concurrent('should include startIndex and endIndex', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'This is test content.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks[0].metadata.startIndex).toBeDefined()
+      expect(chunks[0].metadata.endIndex).toBeDefined()
+      expect(chunks[0].metadata.startIndex).toBe(0)
+    })
+
+    it.concurrent('should have sequential indices across chunks', async () => {
+      const chunker = new TextChunker({ chunkSize: 20, chunkOverlap: 0 })
+      const text = 'First part of text. Second part of text. Third part of text.'
+      const chunks = await chunker.chunk(text)
+
+      if (chunks.length > 1) {
+        for (let i = 1; i < chunks.length; i++) {
+          expect(chunks[i].metadata.startIndex).toBeGreaterThanOrEqual(0)
+        }
+      }
+    })
+  })
+
+  describe('default values', () => {
+    it.concurrent('should use default chunkSize of 1024 tokens', async () => {
+      const chunker = new TextChunker({})
+      const text = 'Word '.repeat(400)
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks).toHaveLength(1)
+    })
+
+    it.concurrent('should use default minCharactersPerChunk of 100', async () => {
+      const chunker = new TextChunker({ chunkSize: 10 })
+      // Text with 150+ characters to ensure chunks pass the 100 character minimum
+      const text = 'This is a longer sentence with more content. '.repeat(5)
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(0)
+    })
+
+    it.concurrent('should use default overlap of 0', async () => {
+      const chunker = new TextChunker({ chunkSize: 20 })
+      const text = 'First sentence here. Second sentence here.'
+      const chunks = await chunker.chunk(text)
+
+      if (chunks.length > 1) {
+        const firstEnd = chunks[0].text.slice(-5)
+        expect(chunks[1].text.startsWith(firstEnd)).toBe(false)
+      }
+    })
+  })
+
+  describe('edge cases', () => {
+    it.concurrent('should handle very long text', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = 'This is a sentence. '.repeat(1000)
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(1)
+      const totalLength = chunks.reduce((sum, c) => sum + c.text.length, 0)
+      expect(totalLength).toBeGreaterThan(0)
+    })
+
+    it.concurrent('should handle text with only special characters', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = '!@#$%^&*()_+-=[]{}|;:,.<>?'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks).toHaveLength(1)
+      expect(chunks[0].text).toBe(text)
+    })
+
+    it.concurrent('should handle unicode text', async () => {
+      const chunker = new TextChunker({ chunkSize: 100 })
+      const text = '这是中文测试。日本語テスト。한국어 테스트.'
+      const chunks = await chunker.chunk(text)
+
+      expect(chunks.length).toBeGreaterThan(0)
+      expect(chunks[0].text).toContain('中文')
+    })
+
+    it.concurrent('should not lose any content during chunking', async () => {
+      const chunker = new TextChunker({ chunkSize: 30, chunkOverlap: 0 })
+      const originalText =
+        'The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs.'
+      const chunks = await chunker.chunk(originalText)
+
+      const allText = chunks.map((c) => c.text).join(' ')
+      expect(allText).toContain('quick')
+      expect(allText).toContain('fox')
+      expect(allText).toContain('lazy')
+      expect(allText).toContain('dog')
+    })
+  })
+})

apps/sim/lib/chunkers/text-chunker.ts

@@ -3,11 +3,16 @@ import type { Chunk, ChunkerOptions } from '@/lib/chunkers/types'
 /**
  * Lightweight text chunker optimized for RAG applications
  * Uses hierarchical splitting with simple character-based token estimation
+ *
+ * Parameters:
+ * - chunkSize: Maximum chunk size in TOKENS (default: 1024)
+ * - chunkOverlap: Overlap between chunks in TOKENS (default: 0)
+ * - minCharactersPerChunk: Minimum characters to keep a chunk (default: 100)
  */
 export class TextChunker {
-  private readonly chunkSize: number
-  private readonly minChunkSize: number
-  private readonly overlap: number
+  private readonly chunkSize: number // Max chunk size in tokens
+  private readonly chunkOverlap: number // Overlap in tokens
+  private readonly minCharactersPerChunk: number // Min characters per chunk
 
   // Hierarchical separators ordered from largest to smallest semantic units
   private readonly separators = [
@@ -32,31 +37,40 @@ export class TextChunker {
   ]
 
   constructor(options: ChunkerOptions = {}) {
-    this.chunkSize = options.chunkSize ?? 512
-    this.minChunkSize = options.minChunkSize ?? 1
-    this.overlap = options.overlap ?? 0
+    this.chunkSize = options.chunkSize ?? 1024
+    // Clamp overlap to prevent exceeding chunk size (max 50% of chunk size)
+    const maxOverlap = Math.floor(this.chunkSize * 0.5)
+    this.chunkOverlap = Math.min(options.chunkOverlap ?? 0, maxOverlap)
+    this.minCharactersPerChunk = options.minCharactersPerChunk ?? 100
   }
 
   /**
    * Simple token estimation using character count
+   * 1 token ≈ 4 characters for English text
    */
   private estimateTokens(text: string): number {
-    // Handle empty or whitespace-only text
     if (!text?.trim()) return 0
-
-    // Simple estimation: ~4 characters per token
     return Math.ceil(text.length / 4)
   }
 
+  /**
+   * Convert tokens to approximate character count
+   */
+  private tokensToChars(tokens: number): number {
+    return tokens * 4
+  }
+
   /**
    * Split text recursively using hierarchical separators
    */
   private async splitRecursively(text: string, separatorIndex = 0): Promise<string[]> {
     const tokenCount = this.estimateTokens(text)
 
-    // If chunk is small enough, return it
+    // If chunk is small enough (within max token limit), return it
+    // Keep chunks even if below minCharactersPerChunk to avoid data loss
     if (tokenCount <= this.chunkSize) {
-      return text.length >= this.minChunkSize ? [text] : []
+      // Only filter out empty/whitespace-only text, not small chunks
+      return text.trim() ? [text] : []
     }
 
     // If we've run out of separators, force split by character count
@@ -66,7 +80,8 @@ export class TextChunker {
 
       for (let i = 0; i < text.length; i += targetLength) {
         const chunk = text.slice(i, i + targetLength).trim()
-        if (chunk.length >= this.minChunkSize) {
+        // Keep all non-empty chunks to avoid data loss
+        if (chunk) {
           chunks.push(chunk)
         }
       }
@@ -90,8 +105,8 @@ export class TextChunker {
       if (this.estimateTokens(testChunk) <= this.chunkSize) {
         currentChunk = testChunk
       } else {
-        // Save current chunk if it meets minimum size
-        if (currentChunk.trim() && currentChunk.length >= this.minChunkSize) {
+        // Save current chunk - keep even if below minCharactersPerChunk to avoid data loss
+        if (currentChunk.trim()) {
           chunks.push(currentChunk.trim())
         }
 
@@ -106,8 +121,8 @@ export class TextChunker {
       }
     }
 
-    // Add final chunk if it exists and meets minimum size
-    if (currentChunk.trim() && currentChunk.length >= this.minChunkSize) {
+    // Add final chunk if it exists - keep even if below minCharactersPerChunk to avoid data loss
+    if (currentChunk.trim()) {
       chunks.push(currentChunk.trim())
     }
 
@@ -115,26 +130,35 @@ export class TextChunker {
   }
 
   /**
-   * Add overlap between chunks if specified
+   * Add overlap between chunks (overlap is in tokens, converted to characters)
    */
   private addOverlap(chunks: string[]): string[] {
-    if (this.overlap <= 0 || chunks.length <= 1) {
+    if (this.chunkOverlap <= 0 || chunks.length <= 1) {
       return chunks
     }
 
     const overlappedChunks: string[] = []
+    // Convert token overlap to character overlap
+    const overlapChars = this.tokensToChars(this.chunkOverlap)
 
     for (let i = 0; i < chunks.length; i++) {
       let chunk = chunks[i]
 
-      // Add overlap from previous chunk
+      // Add overlap from previous chunk (converted from tokens to characters)
       if (i > 0) {
         const prevChunk = chunks[i - 1]
-        const words = prevChunk.split(/\s+/)
-        const overlapWords = words.slice(-Math.min(this.overlap, words.length))
+        // Take the last N characters from previous chunk (based on token overlap)
+        const overlapLength = Math.min(overlapChars, prevChunk.length)
+        const overlapText = prevChunk.slice(-overlapLength)
 
-        if (overlapWords.length > 0) {
-          chunk = `${overlapWords.join(' ')} ${chunk}`
+        // Try to start overlap at a word boundary for cleaner text
+        const wordBoundaryMatch = overlapText.match(/^\s*\S/)
+        const cleanOverlap = wordBoundaryMatch
+          ? overlapText.slice(overlapText.indexOf(wordBoundaryMatch[0].trim()))
+          : overlapText
+
+        if (cleanOverlap.trim()) {
+          chunk = `${cleanOverlap.trim()} ${chunk}`
         }
       }
 
@@ -180,26 +204,22 @@ export class TextChunker {
       let startIndex: number
       let actualContentLength: number
 
-      if (index === 0 || this.overlap <= 0) {
+      if (index === 0 || this.chunkOverlap <= 0) {
         // First chunk or no overlap - start from previous end
         startIndex = previousEndIndex
         actualContentLength = chunkText.length
       } else {
-        // Calculate overlap length in characters
+        // Calculate overlap length in characters (converted from tokens)
         const prevChunk = chunks[index - 1]
-        const prevWords = prevChunk.split(/\s+/)
-        const overlapWords = prevWords.slice(-Math.min(this.overlap, prevWords.length))
-        const overlapLength = Math.min(
-          chunkText.length,
-          overlapWords.length > 0 ? overlapWords.join(' ').length + 1 : 0 // +1 for space
-        )
+        const overlapChars = this.tokensToChars(this.chunkOverlap)
+        const overlapLength = Math.min(overlapChars, prevChunk.length, chunkText.length)
 
         startIndex = previousEndIndex - overlapLength
         actualContentLength = chunkText.length - overlapLength
       }
 
       const safeStart = Math.max(0, startIndex)
-      const endIndexSafe = safeStart + actualContentLength
+      const endIndexSafe = safeStart + Math.max(0, actualContentLength)
 
       const chunk: Chunk = {
         text: chunkText,

apps/sim/lib/chunkers/types.ts

@@ -1,18 +1,18 @@
-export interface ChunkMetadata {
-  startIndex: number
-  endIndex: number
-  tokenCount: number
-}
-
-export interface TextChunk {
-  text: string
-  metadata: ChunkMetadata
-}
-
+/**
+ * Options for configuring text chunkers
+ *
+ * Units:
+ * - chunkSize: Maximum chunk size in TOKENS (1 token ≈ 4 characters)
+ * - chunkOverlap: Overlap between chunks in TOKENS
+ * - minCharactersPerChunk: Minimum chunk size in CHARACTERS (filters tiny fragments)
+ */
 export interface ChunkerOptions {
+  /** Maximum chunk size in tokens (default: 1024) */
   chunkSize?: number
-  minChunkSize?: number
-  overlap?: number
+  /** Overlap between chunks in tokens (default: 0) */
+  chunkOverlap?: number
+  /** Minimum chunk size in characters to avoid tiny fragments (default: 100) */
+  minCharactersPerChunk?: number
 }
 
 export interface Chunk {
@@ -24,7 +24,7 @@ export interface Chunk {
   }
 }
 
-export interface StructuredDataOptions {
+export interface StructuredDataOptions extends ChunkerOptions {
   headers?: string[]
   totalRows?: number
   sheetName?: string

apps/sim/lib/knowledge/documents/document-processor.ts

@@ -54,9 +54,9 @@ export async function processDocument(
   fileUrl: string,
   filename: string,
   mimeType: string,
-  chunkSize = 1000,
+  chunkSize = 1024,
   chunkOverlap = 200,
-  minChunkSize = 1,
+  minCharactersPerChunk = 100,
   userId?: string,
   workspaceId?: string | null
 ): Promise<{
@@ -92,17 +92,18 @@ export async function processDocument(
       logger.info('Using JSON/YAML chunker for structured data')
       chunks = await JsonYamlChunker.chunkJsonYaml(content, {
         chunkSize,
-        minChunkSize,
+        minCharactersPerChunk,
       })
     } else if (StructuredDataChunker.isStructuredData(content, mimeType)) {
       logger.info('Using structured data chunker for spreadsheet/CSV content')
       chunks = await StructuredDataChunker.chunkStructuredData(content, {
+        chunkSize,
         headers: metadata.headers,
         totalRows: metadata.totalRows || metadata.rowCount,
         sheetName: metadata.sheetNames?.[0],
       })
     } else {
-      const chunker = new TextChunker({ chunkSize, overlap: chunkOverlap, minChunkSize })
+      const chunker = new TextChunker({ chunkSize, chunkOverlap, minCharactersPerChunk })
       chunks = await chunker.chunk(content)
     }
 

apps/sim/lib/knowledge/documents/service.ts

@@ -334,13 +334,7 @@ export async function processDocumentsWithQueue(
           fileSize: doc.fileSize,
           mimeType: doc.mimeType,
         },
-        processingOptions: {
-          chunkSize: processingOptions.chunkSize || 1024,
-          minCharactersPerChunk: processingOptions.minCharactersPerChunk || 1,
-          recipe: processingOptions.recipe || 'default',
-          lang: processingOptions.lang || 'en',
-          chunkOverlap: processingOptions.chunkOverlap || 200,
-        },
+        processingOptions,
         requestId,
       }))
 
@@ -425,6 +419,7 @@ export async function processDocumentAsync(
       .select({
         userId: knowledgeBase.userId,
         workspaceId: knowledgeBase.workspaceId,
+        chunkingConfig: knowledgeBase.chunkingConfig,
       })
       .from(knowledgeBase)
       .where(eq(knowledgeBase.id, knowledgeBaseId))
@@ -445,15 +440,18 @@ export async function processDocumentAsync(
 
     logger.info(`[${documentId}] Status updated to 'processing', starting document processor`)
 
+    // Use KB's chunkingConfig as fallback if processingOptions not provided
+    const kbConfig = kb[0].chunkingConfig as { maxSize: number; minSize: number; overlap: number }
+
     await withTimeout(
       (async () => {
         const processed = await processDocument(
           docData.fileUrl,
           docData.filename,
           docData.mimeType,
-          processingOptions.chunkSize || 512,
-          processingOptions.chunkOverlap || 200,
-          processingOptions.minCharactersPerChunk || 1,
+          processingOptions.chunkSize ?? kbConfig.maxSize,
+          processingOptions.chunkOverlap ?? kbConfig.overlap,
+          processingOptions.minCharactersPerChunk ?? kbConfig.minSize,
           kb[0].userId,
           kb[0].workspaceId
         )
@@ -1345,6 +1343,17 @@ export async function retryDocumentProcessing(
   },
   requestId: string
 ): Promise<{ success: boolean; status: string; message: string }> {
+  // Fetch KB's chunkingConfig for retry processing
+  const kb = await db
+    .select({
+      chunkingConfig: knowledgeBase.chunkingConfig,
+    })
+    .from(knowledgeBase)
+    .where(eq(knowledgeBase.id, knowledgeBaseId))
+    .limit(1)
+
+  const kbConfig = kb[0].chunkingConfig as { maxSize: number; minSize: number; overlap: number }
+
   // Clear existing embeddings and reset document state
   await db.transaction(async (tx) => {
     await tx.delete(embedding).where(eq(embedding.documentId, documentId))
@@ -1364,11 +1373,11 @@ export async function retryDocumentProcessing(
   })
 
   const processingOptions = {
-    chunkSize: 512,
-    minCharactersPerChunk: 24,
+    chunkSize: kbConfig.maxSize,
+    minCharactersPerChunk: kbConfig.minSize,
     recipe: 'default',
     lang: 'en',
-    chunkOverlap: 100,
+    chunkOverlap: kbConfig.overlap,
   }
 
   // Start processing in the background

apps/sim/lib/knowledge/documents/types.ts

@@ -13,49 +13,6 @@ export interface DocumentSortOptions {
   sortOrder?: SortOrder
 }
 
-export interface DocChunk {
-  /** The chunk text content */
-  text: string
-  /** Token count estimate for the chunk */
-  tokenCount: number
-  /** Source document path relative to docs/ */
-  sourceDocument: string
-  /** Link to the most relevant header section */
-  headerLink: string
-  /** The header text that this chunk belongs to */
-  headerText: string
-  /** Header level (1-6) */
-  headerLevel: number
-  /** OpenAI text embedding vector (1536 dimensions for text-embedding-3-small) */
-  embedding: number[]
-  /** Model used to generate the embedding */
-  embeddingModel: string
-  /** Metadata about the chunk */
-  metadata: {
-    /** Start position in the original document */
-    startIndex: number
-    /** End position in the original document */
-    endIndex: number
-    /** Whether this chunk contains the document frontmatter */
-    hasFrontmatter?: boolean
-    /** Document title from frontmatter */
-    documentTitle?: string
-    /** Document description from frontmatter */
-    documentDescription?: string
-  }
-}
-
-export interface DocsChunkerOptions {
-  /** Target chunk size in tokens */
-  chunkSize?: number
-  /** Minimum chunk size in tokens */
-  minChunkSize?: number
-  /** Overlap between chunks in tokens */
-  overlap?: number
-  /** Base URL for generating links */
-  baseUrl?: string
-}
-
 export interface HeaderInfo {
   /** Header text */
   text: string

apps/sim/lib/knowledge/types.ts

@@ -1,6 +1,17 @@
+/**
+ * Configuration for document chunking in knowledge bases
+ *
+ * Units:
+ * - maxSize: Maximum chunk size in TOKENS (1 token ≈ 4 characters)
+ * - minSize: Minimum chunk size in CHARACTERS (floor to avoid tiny fragments)
+ * - overlap: Overlap between chunks in TOKENS (1 token ≈ 4 characters)
+ */
 export interface ChunkingConfig {
+  /** Maximum chunk size in tokens (default: 1024, range: 100-4000) */
   maxSize: number
+  /** Minimum chunk size in characters (default: 100, range: 1-2000) */
   minSize: number
+  /** Overlap between chunks in tokens (default: 200, range: 0-500) */
   overlap: number
 }
 

apps/sim/scripts/process-docs.ts

@@ -19,10 +19,10 @@ interface ProcessingOptions {
   baseUrl?: string
   /** Chunk size in tokens */
   chunkSize?: number
-  /** Minimum chunk size */
-  minChunkSize?: number
-  /** Overlap between chunks */
-  overlap?: number
+  /** Minimum chunk size in characters */
+  minCharactersPerChunk?: number
+  /** Overlap between chunks in tokens */
+  chunkOverlap?: number
   /** Dry run - only display results, don't save to DB */
   dryRun?: boolean
   /** Verbose output */
@@ -37,8 +37,8 @@ async function processDocs(options: ProcessingOptions = {}) {
     docsPath: options.docsPath || path.join(process.cwd(), '../../apps/docs/content/docs/en'),
     baseUrl: options.baseUrl || (isDev ? 'http://localhost:4000' : 'https://docs.sim.ai'),
     chunkSize: options.chunkSize || 1024,
-    minChunkSize: options.minChunkSize || 100,
-    overlap: options.overlap || 200,
+    minCharactersPerChunk: options.minCharactersPerChunk || 100,
+    chunkOverlap: options.chunkOverlap || 200,
     clearExisting: options.clearExisting ?? false,
     dryRun: options.dryRun ?? false,
     verbose: options.verbose ?? false,
@@ -59,8 +59,8 @@ async function processDocs(options: ProcessingOptions = {}) {
     // Initialize the chunker
     const chunker = new DocsChunker({
       chunkSize: config.chunkSize,
-      minChunkSize: config.minChunkSize,
-      overlap: config.overlap,
+      minCharactersPerChunk: config.minCharactersPerChunk,
+      chunkOverlap: config.chunkOverlap,
       baseUrl: config.baseUrl,
     })
 

apps/sim/stores/knowledge/store.ts

@@ -3,9 +3,20 @@ import { createLogger } from '@/lib/logs/console/logger'
 
 const logger = createLogger('KnowledgeStore')
 
+/**
+ * Configuration for document chunking in knowledge bases
+ *
+ * Units:
+ * - maxSize: Maximum chunk size in TOKENS (1 token ≈ 4 characters)
+ * - minSize: Minimum chunk size in CHARACTERS (floor to avoid tiny fragments)
+ * - overlap: Overlap between chunks in TOKENS (1 token ≈ 4 characters)
+ */
 export interface ChunkingConfig {
+  /** Maximum chunk size in tokens (default: 1024, range: 100-4000) */
   maxSize: number
+  /** Minimum chunk size in characters (default: 100, range: 1-2000) */
   minSize: number
+  /** Overlap between chunks in tokens (default: 200, range: 0-500) */
   overlap: number
   chunkSize?: number // Legacy support
   minCharactersPerChunk?: number // Legacy support