diff --git a/sdk/packages/core/src/extensions/context/basic-compaction.ts b/sdk/packages/core/src/extensions/context/basic-compaction.ts index 229c6518b3..0269322ba4 100644 --- a/sdk/packages/core/src/extensions/context/basic-compaction.ts +++ b/sdk/packages/core/src/extensions/context/basic-compaction.ts @@ -1,9 +1,14 @@ -import type { BasicLogger, MessageWithMetadata } from "@cline/shared"; +import { + type BasicLogger, + CHARS_PER_TOKEN, + type MessageWithMetadata, +} from "@cline/shared"; import type { CoreCompactionContext, CoreCompactionResult, } from "../../types/config"; import { + DEFAULT_TARGET_RATIO, type EstimateMessageTokens, findFirstUserMessageIndex, findLastAssistantIndex, @@ -11,7 +16,6 @@ import { isCompactionSummaryMessage, isTurnStartMessage, MIN_TRUNCATED_MESSAGE_TOKENS, - truncateText, truncateToolResultContentForCompaction, } from "./compaction-shared"; @@ -19,11 +23,17 @@ interface BasicCompactionCandidate { index: number; message: MessageWithMetadata; estimatedTokens: number; + toolPairIds: string[]; isFirstUser: boolean; isLastUser: boolean; isLastAssistant: boolean; } +interface BasicCompactionCandidateSet { + candidates: BasicCompactionCandidate[]; + totalTokens: number; +} + function sanitizeMessageForBasic( message: MessageWithMetadata, ): MessageWithMetadata | undefined { @@ -67,15 +77,24 @@ function getTotalTokens( ); } +function truncateMessageText(text: string, limit: number): string { + if (text.length <= limit) { + return text; + } + const suffix = "\n..."; + const sliceLength = Math.max(1, limit - suffix.length); + return `${text.slice(0, sliceLength)}${suffix}`; +} + function truncateMessageToTokens( message: MessageWithMetadata, maxTokens: number, ): MessageWithMetadata { - const safeMaxTokens = Math.max(1, maxTokens); - const targetChars = Math.max(16, safeMaxTokens * 4); + const safeMaxTokens = Number.isFinite(maxTokens) ? Math.max(1, maxTokens) : 1; + const targetChars = Math.max(16, safeMaxTokens * CHARS_PER_TOKEN); if (typeof message.content === "string") { - const truncated = truncateText(message.content, targetChars).trim(); + const truncated = truncateMessageText(message.content, targetChars).trim(); return { ...message, content: truncated || "..." }; } @@ -85,7 +104,7 @@ function truncateMessageToTokens( if (block.type !== "text" || remaining <= 0) { return block; } - const truncated = truncateText(block.text, remaining).trim(); + const truncated = truncateMessageText(block.text, remaining).trim(); remaining -= truncated.length; return { ...block, text: truncated || "..." }; }); @@ -95,26 +114,30 @@ function truncateMessageToTokens( function buildBasicCandidates( messages: MessageWithMetadata[], estimateMessageTokens: EstimateMessageTokens, -): BasicCompactionCandidate[] { +): BasicCompactionCandidateSet { const firstUserIndex = findFirstUserMessageIndex(messages); const lastUserIndex = findLastTurnStartIndex(messages); const lastAssistantIndex = findLastAssistantIndex(messages); const candidates: BasicCompactionCandidate[] = []; + let totalTokens = 0; for (let index = 0; index < messages.length; index += 1) { const sanitized = sanitizeMessageForBasic(messages[index]); if (!sanitized) { continue; } + const estimatedTokens = estimateMessageTokens(sanitized); + totalTokens += estimatedTokens; candidates.push({ index, message: sanitized, - estimatedTokens: estimateMessageTokens(sanitized), + estimatedTokens, + toolPairIds: collectToolPairIds(sanitized), isFirstUser: index === firstUserIndex, isLastUser: index === lastUserIndex, isLastAssistant: index === lastAssistantIndex, }); } - return candidates; + return { candidates, totalTokens }; } function updateCandidate( @@ -122,98 +145,102 @@ function updateCandidate( index: number, message: MessageWithMetadata, estimateMessageTokens: EstimateMessageTokens, -): void { +): number { const candidate = candidates[index]; + const previousTokens = candidate.estimatedTokens; candidate.message = message; candidate.estimatedTokens = estimateMessageTokens(message); + candidate.toolPairIds = collectToolPairIds(message); + return candidate.estimatedTokens - previousTokens; } -function collectToolUseIds(message: MessageWithMetadata): Set { - const ids = new Set(); +function collectToolPairIds(message: MessageWithMetadata): string[] { if (!Array.isArray(message.content)) { - return ids; + return []; } + const ids: string[] = []; for (const block of message.content) { if (block.type === "tool_use") { - ids.add(block.id); + ids.push(block.id); + } else if (block.type === "tool_result") { + ids.push(block.tool_use_id); } } return ids; } -function collectToolResultIds(message: MessageWithMetadata): Set { - const ids = new Set(); - if (!Array.isArray(message.content)) { - return ids; - } - for (const block of message.content) { - if (block.type === "tool_result") { - ids.add(block.tool_use_id); - } - } - return ids; -} - -function collectToolPairIds(candidate: BasicCompactionCandidate): Set { - return new Set([ - ...collectToolUseIds(candidate.message), - ...collectToolResultIds(candidate.message), - ]); -} - function buildToolPairCandidateIndex( candidates: BasicCompactionCandidate[], -): Map> { - const indexByToolUseId = new Map>(); - for (let index = 0; index < candidates.length; index += 1) { - for (const id of collectToolPairIds(candidates[index])) { +): Map> { + const indexByToolUseId = new Map>(); + for (const candidate of candidates) { + for (const id of candidate.toolPairIds) { const existing = indexByToolUseId.get(id); if (existing) { - existing.add(index); + existing.add(candidate); } else { - indexByToolUseId.set(id, new Set([index])); + indexByToolUseId.set(id, new Set([candidate])); } } } return indexByToolUseId; } -function collectAtomicRemovalIndexes( - candidates: BasicCompactionCandidate[], - startIndex: number, -): Set { - const pairIndex = buildToolPairCandidateIndex(candidates); - const removalIndexes = new Set(); - const queue = [startIndex]; +function collectAtomicRemovalCandidates( + activeCandidates: Set, + pairIndex: Map>, + startCandidate: BasicCompactionCandidate, +): Set { + const removalCandidates = new Set(); + const queue = [startCandidate]; + let queueIndex = 0; - while (queue.length > 0) { - const index = queue.shift(); - if (index === undefined || removalIndexes.has(index)) { + while (queueIndex < queue.length) { + const candidate = queue[queueIndex]; + queueIndex += 1; + if ( + candidate === undefined || + !activeCandidates.has(candidate) || + removalCandidates.has(candidate) + ) { continue; } - removalIndexes.add(index); - for (const id of collectToolPairIds(candidates[index])) { - for (const linkedIndex of pairIndex.get(id) ?? []) { - if (!removalIndexes.has(linkedIndex)) { - queue.push(linkedIndex); + removalCandidates.add(candidate); + for (const id of candidate.toolPairIds) { + for (const linkedCandidate of pairIndex.get(id) ?? []) { + if (!removalCandidates.has(linkedCandidate)) { + queue.push(linkedCandidate); } } } } - return removalIndexes; + return removalCandidates; +} + +function removeCandidateSetInPlace( + candidates: BasicCompactionCandidate[], + removalCandidates: Set, +): void { + let writeIndex = 0; + for (const candidate of candidates) { + if (removalCandidates.has(candidate)) { + continue; + } + candidates[writeIndex] = candidate; + writeIndex += 1; + } + candidates.length = writeIndex; } function removeCandidatesByPredicate( candidates: BasicCompactionCandidate[], predicate: (candidate: BasicCompactionCandidate) => boolean, targetTokens: number, - estimateMessageTokens: EstimateMessageTokens, -): void { - let totalTokens = getTotalTokens( - candidates.map((candidate) => candidate.message), - estimateMessageTokens, - ); + totalTokens: number, +): number { + const activeCandidates = new Set(candidates); + const pairIndex = buildToolPairCandidateIndex(candidates); for ( let index = 0; index < candidates.length && totalTokens > targetTokens; @@ -222,30 +249,34 @@ function removeCandidatesByPredicate( index += 1; continue; } - const removalIndexes = collectAtomicRemovalIndexes(candidates, index); - totalTokens -= Array.from(removalIndexes).reduce( - (total, removalIndex) => total + candidates[removalIndex].estimatedTokens, - 0, + const removalCandidates = collectAtomicRemovalCandidates( + activeCandidates, + pairIndex, + candidates[index], ); - for (const removalIndex of Array.from(removalIndexes).sort( - (left, right) => right - left, - )) { - candidates.splice(removalIndex, 1); + if (removalCandidates.size === 0) { + index += 1; + continue; } + let removedTokens = 0; + for (const candidate of removalCandidates) { + removedTokens += candidate.estimatedTokens; + activeCandidates.delete(candidate); + } + totalTokens -= removedTokens; + removeCandidateSetInPlace(candidates, removalCandidates); } + return totalTokens; } function trimCandidatesToBudget( candidates: BasicCompactionCandidate[], targetTokens: number, + totalTokens: number, estimateMessageTokens: EstimateMessageTokens, -): void { - let totalTokens = getTotalTokens( - candidates.map((candidate) => candidate.message), - estimateMessageTokens, - ); +): number { if (totalTokens <= targetTokens) { - return; + return totalTokens; } for ( @@ -264,40 +295,44 @@ function trimCandidatesToBudget( if (desiredTokens >= candidate.estimatedTokens) { continue; } - updateCandidate( + totalTokens += updateCandidate( candidates, index, truncateMessageToTokens(candidate.message, desiredTokens), estimateMessageTokens, ); - totalTokens = getTotalTokens( - candidates.map((item) => item.message), - estimateMessageTokens, - ); } if (totalTokens <= targetTokens) { - return; + return totalTokens; } const firstUserIndex = candidates.findIndex( (candidate) => candidate.isFirstUser, ); if (firstUserIndex >= 0) { - const desiredTokens = Math.max( - 1, - candidates[firstUserIndex].estimatedTokens - (totalTokens - targetTokens), - ); - updateCandidate( - candidates, - firstUserIndex, - truncateMessageToTokens( - candidates[firstUserIndex].message, - desiredTokens, - ), - estimateMessageTokens, - ); + while (totalTokens > targetTokens) { + const candidate = candidates[firstUserIndex]; + const desiredTokens = Math.max( + 1, + candidate.estimatedTokens - (totalTokens - targetTokens), + ); + if (desiredTokens >= candidate.estimatedTokens) { + break; + } + const previousTokens = candidate.estimatedTokens; + totalTokens += updateCandidate( + candidates, + firstUserIndex, + truncateMessageToTokens(candidate.message, desiredTokens), + estimateMessageTokens, + ); + if (candidate.estimatedTokens >= previousTokens) { + break; + } + } } + return totalTokens; } function haveMessagesChanged( @@ -329,9 +364,13 @@ export function runBasicCompaction(options: { estimateMessageTokens: EstimateMessageTokens; logger?: BasicLogger; }): CoreCompactionResult | undefined { - const targetTokens = Math.max( + const totalTargetTokens = Math.max( 1, - Math.min(options.context.triggerTokens, options.context.maxInputTokens), + Math.min( + options.context.targetTokens ?? + Math.floor(options.context.triggerTokens * DEFAULT_TARGET_RATIO), + options.context.maxInputTokens, + ), ); const { compactable, protectedTail } = splitLatestTurn( options.context.messages, @@ -339,50 +378,59 @@ export function runBasicCompaction(options: { if (compactable.length === 0) { return undefined; } - const candidates = buildBasicCandidates( + const protectedTailTokens = getTotalTokens( + protectedTail, + options.estimateMessageTokens, + ); + const targetTokens = Math.max(1, totalTargetTokens - protectedTailTokens); + const candidateSet = buildBasicCandidates( compactable, options.estimateMessageTokens, ); + const { candidates } = candidateSet; if (candidates.length === 0) { return undefined; } + const beforeCompactableTokens = candidateSet.totalTokens; + let totalTokens = beforeCompactableTokens; - removeCandidatesByPredicate( + totalTokens = removeCandidatesByPredicate( candidates, (candidate) => candidate.message.role === "assistant" && !candidate.isLastAssistant, targetTokens, - options.estimateMessageTokens, + totalTokens, ); - removeCandidatesByPredicate( + totalTokens = removeCandidatesByPredicate( candidates, (candidate) => candidate.message.role === "user" && !candidate.isFirstUser && !candidate.isLastUser, targetTokens, - options.estimateMessageTokens, + totalTokens, ); - removeCandidatesByPredicate( + totalTokens = removeCandidatesByPredicate( candidates, (candidate) => candidate.message.role === "assistant" && candidate.isLastAssistant, targetTokens, - options.estimateMessageTokens, + totalTokens, ); - removeCandidatesByPredicate( + totalTokens = removeCandidatesByPredicate( candidates, (candidate) => candidate.message.role === "user" && candidate.isLastUser && !candidate.isFirstUser, targetTokens, - options.estimateMessageTokens, + totalTokens, ); - trimCandidatesToBudget( + totalTokens = trimCandidatesToBudget( candidates, targetTokens, + totalTokens, options.estimateMessageTokens, ); @@ -394,24 +442,15 @@ export function runBasicCompaction(options: { return undefined; } - const beforeTokens = getTotalTokens( - [ - ...compactable.map((m) => sanitizeMessageForBasic(m) ?? m), - ...protectedTail, - ], - options.estimateMessageTokens, - ); - const afterTokens = getTotalTokens( - nextMessages, - options.estimateMessageTokens, - ); + const beforeTokens = beforeCompactableTokens + protectedTailTokens; + const afterTokens = totalTokens + protectedTailTokens; options.logger?.debug("Performed basic compaction", { messagesBefore: options.context.messages.length, messagesAfter: nextMessages.length, messagesRemoved: options.context.messages.length - nextMessages.length, tokensBefore: beforeTokens, tokensAfter: afterTokens, - targetTokens, + targetTokens: totalTargetTokens, maxInputTokens: options.context.maxInputTokens, }); diff --git a/sdk/packages/core/src/extensions/context/compaction-shared.ts b/sdk/packages/core/src/extensions/context/compaction-shared.ts index bea7669e81..d0b25f370d 100644 --- a/sdk/packages/core/src/extensions/context/compaction-shared.ts +++ b/sdk/packages/core/src/extensions/context/compaction-shared.ts @@ -1,7 +1,11 @@ import type { ToolResultContent } from "@cline/llms"; -import { estimateTokens, type MessageWithMetadata } from "@cline/shared"; +import { + CHARS_PER_TOKEN, + estimateTokens, + type MessageWithMetadata, +} from "@cline/shared"; -export { estimateTokens }; +export { CHARS_PER_TOKEN, estimateTokens }; import type { CoreCompactionContext, @@ -11,6 +15,7 @@ import type { ProviderConfig } from "../../types/provider-settings"; export const DEFAULT_MAX_INPUT_TOKENS = 200_000; export const DEFAULT_THRESHOLD_RATIO = 0.9; +export const DEFAULT_TARGET_RATIO = 0.7; export const DEFAULT_RESERVE_TOKENS = 16_384; export const DEFAULT_PRESERVE_RECENT_TOKENS = 20_000; export const DEFAULT_SUMMARY_MAX_OUTPUT_TOKENS = 1_024; diff --git a/sdk/packages/core/src/extensions/context/compaction.test.ts b/sdk/packages/core/src/extensions/context/compaction.test.ts index b3307138a1..5c896a257e 100644 --- a/sdk/packages/core/src/extensions/context/compaction.test.ts +++ b/sdk/packages/core/src/extensions/context/compaction.test.ts @@ -1,4 +1,5 @@ import type * as LlmsProviders from "@cline/llms"; +import type { MessageWithMetadata } from "@cline/shared"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { CoreCompactionContext } from "../../types/config"; import { runBasicCompaction } from "./basic-compaction"; @@ -34,6 +35,39 @@ function totalJsonTokens(messages: LlmsProviders.Message[]): number { ); } +describe("createTokenEstimator", () => { + it("does not treat assistant request metrics as per-message token counts", () => { + const estimateMessageTokens = createTokenEstimator(); + const message: MessageWithMetadata = { + role: "assistant", + content: "short", + metrics: { + inputTokens: 12, + outputTokens: 7, + }, + }; + + expect(estimateMessageTokens(message)).toBe( + Math.ceil(JSON.stringify(message).length / 3), + ); + }); + + it("falls back to serialized character estimation when metrics are incomplete", () => { + const estimateMessageTokens = createTokenEstimator(); + const message: MessageWithMetadata = { + role: "assistant", + content: "short", + metrics: { + inputTokens: 12, + }, + }; + + expect(estimateMessageTokens(message)).toBe( + Math.ceil(JSON.stringify(message).length / 3), + ); + }); +}); + function runForcedBasicCompaction( messages: LlmsProviders.Message[], targetTokens: number, @@ -1076,6 +1110,254 @@ describe("createContextCompactionPrepareTurn", () => { ]); }); + it("does not subtract model max output tokens from an explicit input budget", async () => { + const compact = vi.fn((_context: CoreCompactionContext) => ({ + messages: [ + { role: "user" as const, content: "Compacted by input budget" }, + ], + })); + const prepareTurn = createContextCompactionPrepareTurn({ + providerId: "openai-codex", + modelId: "gpt-5.4-mini", + providerConfig: { + providerId: "openai-codex", + modelId: "gpt-5.4-mini", + } as LlmsProviders.ProviderConfig, + compaction: { enabled: true, maxInputTokens: 200_000, compact }, + logger: undefined, + }); + const messages: MessageWithMetadata[] = [ + { + role: "user", + content: "large prompt ".repeat(20_000), + }, + ]; + + const result = await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 1, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages, + apiMessages: messages, + model: { + id: "gpt-5.4-mini", + provider: "openai-codex", + info: { + id: "gpt-5.4-mini", + maxInputTokens: 200_000, + maxTokens: 128_000, + }, + }, + }); + + expect(compact).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); + }); + + it("targets basic compaction below the model output-reserved input budget", async () => { + const compact = vi.fn((_context: CoreCompactionContext) => ({ + messages: [ + { role: "user" as const, content: "Compacted by target budget" }, + ], + })); + const prepareTurn = createContextCompactionPrepareTurn({ + providerId: "openai-codex", + modelId: "gpt-5.5", + providerConfig: { + providerId: "openai-codex", + modelId: "gpt-5.5", + } as LlmsProviders.ProviderConfig, + compaction: { enabled: true, strategy: "basic", compact }, + logger: undefined, + }); + const messages: MessageWithMetadata[] = [ + { + role: "user", + content: "large prompt ".repeat(70_000), + }, + ]; + + await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 1, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages, + apiMessages: messages, + model: { + id: "gpt-5.5", + provider: "openai-codex", + info: { + id: "gpt-5.5", + maxInputTokens: 272_000, + maxTokens: 128_000, + }, + }, + }); + + expect(compact).toHaveBeenCalledTimes(1); + const context = compact.mock.calls[0]?.[0]; + expect(context?.triggerTokens).toBe(244_800); + expect(context?.targetTokens).toBe(100_800); + }); + + it("derives input budget by reserving model max output tokens from context window", async () => { + const compact = vi.fn((_context: CoreCompactionContext) => ({ + messages: [ + { role: "user" as const, content: "Compacted by derived input budget" }, + ], + })); + const prepareTurn = createContextCompactionPrepareTurn({ + providerId: "openai-codex", + modelId: "gpt-5.5", + providerConfig: { + providerId: "openai-codex", + modelId: "gpt-5.5", + } as LlmsProviders.ProviderConfig, + compaction: { enabled: true, compact }, + logger: undefined, + }); + const messages: MessageWithMetadata[] = [ + { + role: "user", + content: "large prompt ".repeat(60_000), + }, + ]; + + const result = await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 1, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages, + apiMessages: messages, + model: { + id: "gpt-5.5", + provider: "openai-codex", + info: { + id: "gpt-5.5", + contextWindow: 400_000, + maxTokens: 128_000, + }, + }, + }); + + expect(compact).toHaveBeenCalledTimes(1); + const context = compact.mock.calls[0]?.[0]; + expect(context?.maxInputTokens).toBe(272_000); + expect(context?.triggerTokens).toBe(244_800); + expect(context?.thresholdRatio).toBe(0.9); + expect(result?.messages).toEqual([ + { role: "user", content: "Compacted by derived input budget" }, + ]); + }); + + it("uses the lower split input budget when it is below context-derived input budget", async () => { + const compact = vi.fn((_context: CoreCompactionContext) => ({ + messages: [ + { role: "user" as const, content: "Compacted by split input" }, + ], + })); + const prepareTurn = createContextCompactionPrepareTurn({ + providerId: "openai-codex", + modelId: "gpt-5.5", + providerConfig: { + providerId: "openai-codex", + modelId: "gpt-5.5", + } as LlmsProviders.ProviderConfig, + compaction: { enabled: true, compact }, + logger: undefined, + }); + const messages: MessageWithMetadata[] = [ + { + role: "user", + content: "large prompt ".repeat(60_000), + }, + ]; + + const result = await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 1, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages, + apiMessages: messages, + model: { + id: "gpt-5.5", + provider: "openai-codex", + info: { + id: "gpt-5.5", + contextWindow: 400_000, + maxInputTokens: 200_000, + maxTokens: 128_000, + }, + }, + }); + + expect(compact).toHaveBeenCalledTimes(1); + const context = compact.mock.calls[0]?.[0]; + expect(context?.maxInputTokens).toBe(200_000); + expect(context?.triggerTokens).toBe(180_000); + expect(context?.thresholdRatio).toBe(0.9); + expect(result?.messages).toEqual([ + { role: "user", content: "Compacted by split input" }, + ]); + }); + + it("falls back to the default trigger when model max output cannot derive a lower input budget", async () => { + const compact = vi.fn((_context: CoreCompactionContext) => ({ + messages: [{ role: "user" as const, content: "Compacted by fallback" }], + })); + const prepareTurn = createContextCompactionPrepareTurn({ + providerId: "openai-codex", + modelId: "large-output-model", + providerConfig: { + providerId: "openai-codex", + modelId: "large-output-model", + } as LlmsProviders.ProviderConfig, + compaction: { enabled: true, maxInputTokens: 200_000, compact }, + logger: undefined, + }); + + const result = await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 1, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages: [{ role: "user", content: "small prompt" }], + apiMessages: [{ role: "user", content: "small prompt" }], + model: { + id: "large-output-model", + provider: "openai-codex", + info: { + id: "large-output-model", + maxInputTokens: 200_000, + contextWindow: 200_000, + maxTokens: 200_000, + }, + }, + }); + + expect(compact).not.toHaveBeenCalled(); + expect(result).toBeUndefined(); + }); + it("triggers compaction from provider-sized tool result payloads", async () => { const compact = vi.fn((_context: CoreCompactionContext) => ({ messages: [ @@ -1726,4 +2008,107 @@ describe("createContextCompactionPrepareTurn", () => { "manual", ); }); + + it("does not immediately re-trigger basic compaction on the next turn after accounting for the protected tail", async () => { + const prepareTurn = createContextCompactionPrepareTurn({ + providerId: "openai-codex", + modelId: "mock-model", + providerConfig: { + providerId: "openai-codex", + modelId: "mock-model", + } as LlmsProviders.ProviderConfig, + compaction: { enabled: true, strategy: "basic", thresholdRatio: 0.5 }, + logger: undefined, + }); + const estimateMessageTokens = createTokenEstimator(); + const model = { + id: "mock-model", + provider: "openai-codex", + info: { id: "mock-model", maxInputTokens: 300 }, + }; + const messages: LlmsProviders.Message[] = [ + { role: "user", content: "old user context ".repeat(80) }, + { role: "assistant", content: "old assistant context ".repeat(80) }, + { role: "user", content: "current request" }, + ]; + const triggerTokens = 150; + const initialTokens = messages.reduce( + (total, message) => total + estimateMessageTokens(message), + 0, + ); + console.info("[basic compaction loop] round=1 before", { + triggerTokens, + messageCount: messages.length, + tokens: initialTokens, + }); + + const firstResult = await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 1, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages, + apiMessages: messages, + model, + }); + + expect(firstResult?.messages).toBeDefined(); + const firstAfterTokens = firstResult?.messages.reduce( + (total, message) => total + estimateMessageTokens(message), + 0, + ); + console.info("[basic compaction loop] round=1 after", { + compacted: Boolean(firstResult?.messages), + messageCount: firstResult?.messages.length, + tokens: firstAfterTokens, + overTrigger: (firstAfterTokens ?? 0) > triggerTokens, + }); + expect(firstAfterTokens).toBeLessThanOrEqual(triggerTokens); + + const nextTurnMessages: LlmsProviders.Message[] = [ + ...(firstResult?.messages ?? []), + { role: "assistant", content: "short answer" }, + { role: "user", content: "next request" }, + ]; + const nextTurnTokens = nextTurnMessages.reduce( + (total, message) => total + estimateMessageTokens(message), + 0, + ); + console.info("[basic compaction loop] round=2 before", { + triggerTokens, + messageCount: nextTurnMessages.length, + tokens: nextTurnTokens, + overTrigger: nextTurnTokens > triggerTokens, + }); + const secondResult = await prepareTurn?.({ + agentId: "agent-1", + conversationId: "conv-1", + parentAgentId: null, + iteration: 2, + abortSignal: new AbortController().signal, + systemPrompt: "You are helpful.", + tools: [], + messages: nextTurnMessages, + apiMessages: nextTurnMessages, + model, + }); + + const secondAfterTokens = secondResult?.messages.reduce( + (total, message) => total + estimateMessageTokens(message), + 0, + ); + console.info("[basic compaction loop] round=2 after", { + compacted: Boolean(secondResult?.messages), + messageCount: secondResult?.messages.length, + tokens: secondAfterTokens, + overTrigger: + typeof secondAfterTokens === "number" + ? secondAfterTokens > triggerTokens + : undefined, + }); + expect(secondResult).toBeUndefined(); + }); }); diff --git a/sdk/packages/core/src/extensions/context/compaction.ts b/sdk/packages/core/src/extensions/context/compaction.ts index 5657661f54..3e07288307 100644 --- a/sdk/packages/core/src/extensions/context/compaction.ts +++ b/sdk/packages/core/src/extensions/context/compaction.ts @@ -18,6 +18,7 @@ import { DEFAULT_MAX_INPUT_TOKENS, DEFAULT_PRESERVE_RECENT_TOKENS, DEFAULT_RESERVE_TOKENS, + DEFAULT_TARGET_RATIO, DEFAULT_THRESHOLD_RATIO, } from "./compaction-shared"; @@ -76,6 +77,37 @@ function safeJsonSize(value: unknown): number { } } +function isPositiveFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value > 0; +} + +function resolveMaxInputTokens(input: { + configMaxInputTokens?: number; + modelMaxInputTokens?: number; + contextWindow?: number; + modelMaxTokens?: number; +}): number { + const candidates: number[] = []; + if (isPositiveFiniteNumber(input.configMaxInputTokens)) { + candidates.push(input.configMaxInputTokens); + } + if (isPositiveFiniteNumber(input.modelMaxInputTokens)) { + candidates.push(input.modelMaxInputTokens); + } + if (isPositiveFiniteNumber(input.contextWindow)) { + candidates.push(input.contextWindow); + if ( + isPositiveFiniteNumber(input.modelMaxTokens) && + input.modelMaxTokens < input.contextWindow + ) { + candidates.push(input.contextWindow - input.modelMaxTokens); + } + } + return candidates.length > 0 + ? Math.min(...candidates) + : DEFAULT_MAX_INPUT_TOKENS; +} + function summarizeToolResults(messages: CoreCompactionContext["messages"]): { toolResultCount: number; toolResultSerializedChars: number; @@ -157,28 +189,28 @@ function resolveTriggerState(input: { }; } - if (typeof input.config.thresholdRatio !== "number") { - const triggerTokens = Math.max( - 0, - Math.min( - input.maxInputTokens - DEFAULT_RESERVE_TOKENS, - input.maxInputTokens * DEFAULT_THRESHOLD_RATIO, - ), - ); + if (typeof input.config.thresholdRatio === "number") { + const thresholdRatio = input.config.thresholdRatio; + const triggerTokens = input.maxInputTokens * thresholdRatio; return { shouldCompact: input.inputTokens > triggerTokens, triggerTokens, - thresholdRatio: - input.maxInputTokens > 0 ? triggerTokens / input.maxInputTokens : 0, + thresholdRatio, }; } - const thresholdRatio = input.config.thresholdRatio ?? DEFAULT_THRESHOLD_RATIO; - const triggerTokens = input.maxInputTokens * thresholdRatio; + const triggerTokens = Math.max( + 0, + Math.min( + input.maxInputTokens - DEFAULT_RESERVE_TOKENS, + input.maxInputTokens * DEFAULT_THRESHOLD_RATIO, + ), + ); return { shouldCompact: input.inputTokens > triggerTokens, triggerTokens, - thresholdRatio, + thresholdRatio: + input.maxInputTokens > 0 ? triggerTokens / input.maxInputTokens : 0, }; } @@ -209,6 +241,26 @@ function resolveManualTargetState(input: { }; } +function resolveBasicTargetTokens(input: { + maxInputTokens: number; + modelMaxTokens?: number; + triggerTokens: number; +}): number { + const targetBaseTokens = + typeof input.modelMaxTokens === "number" && + Number.isFinite(input.modelMaxTokens) && + input.modelMaxTokens < input.maxInputTokens + ? input.maxInputTokens - input.modelMaxTokens + : input.triggerTokens; + return Math.max( + 1, + Math.min( + Math.floor(targetBaseTokens * DEFAULT_TARGET_RATIO), + input.maxInputTokens, + ), + ); +} + /** * Build the `prepareTurn` callback used by the agent runtime to compact the * transcript before each model request. @@ -264,23 +316,18 @@ export function createContextCompactionPrepareTurn( (total: number, message) => total + estimateMessageTokens(message), 0, ); - const maxInputTokens = - userCompaction?.maxInputTokens ?? - context.model.info?.maxInputTokens ?? - context.model.info?.contextWindow ?? - DEFAULT_MAX_INPUT_TOKENS; - if ( - typeof maxInputTokens !== "number" || - !Number.isFinite(maxInputTokens) || - maxInputTokens <= 0 - ) { - return undefined; - } + const maxInputTokens = resolveMaxInputTokens({ + configMaxInputTokens: userCompaction?.maxInputTokens, + modelMaxInputTokens: context.model.info?.maxInputTokens, + contextWindow: context.model.info?.contextWindow, + modelMaxTokens: context.model.info?.maxTokens, + }); const triggerState = resolveTriggerState({ inputTokens, maxInputTokens, config: { + maxInputTokens: userCompaction?.maxInputTokens, reserveTokens: userCompaction?.reserveTokens, thresholdRatio: userCompaction?.thresholdRatio, }, @@ -304,28 +351,37 @@ export function createContextCompactionPrepareTurn( if (mode === "auto" && !triggerState.shouldCompact) { return undefined; } - const targetState = - mode === "manual" - ? resolveManualTargetState({ - inputTokens, - maxInputTokens, + const targetState = + mode === "manual" + ? resolveManualTargetState({ + inputTokens, + maxInputTokens, autoTriggerTokens: triggerState.triggerTokens, manualTargetRatio: options.manualTargetRatio, }) - : triggerState; + : triggerState; + const targetTokens = + mode === "auto" + ? resolveBasicTargetTokens({ + maxInputTokens, + modelMaxTokens: context.model.info?.maxTokens, + triggerTokens: targetState.triggerTokens, + }) + : undefined; - const compactionContext = { - agentId: context.agentId, - conversationId: context.conversationId, + const compactionContext = { + agentId: context.agentId, + conversationId: context.conversationId, parentAgentId: context.parentAgentId, iteration: context.iteration, messages: context.messages, - model: context.model, - maxInputTokens, - triggerTokens: targetState.triggerTokens, - thresholdRatio: targetState.thresholdRatio, - utilizationRatio: maxInputTokens > 0 ? inputTokens / maxInputTokens : 0, - }; + model: context.model, + maxInputTokens, + triggerTokens: targetState.triggerTokens, + targetTokens, + thresholdRatio: targetState.thresholdRatio, + utilizationRatio: maxInputTokens > 0 ? inputTokens / maxInputTokens : 0, + }; const statusReason = mode === "manual" ? "manual_compaction" : "auto_compaction"; diff --git a/sdk/packages/core/src/types/config.ts b/sdk/packages/core/src/types/config.ts index 4915b41d11..360e0382e6 100644 --- a/sdk/packages/core/src/types/config.ts +++ b/sdk/packages/core/src/types/config.ts @@ -64,6 +64,7 @@ export interface CoreCompactionContext { }; maxInputTokens: number; triggerTokens: number; + targetTokens?: number; thresholdRatio: number; utilizationRatio: number; } diff --git a/sdk/packages/shared/src/index.browser.ts b/sdk/packages/shared/src/index.browser.ts index 4493aa96b6..84631be754 100644 --- a/sdk/packages/shared/src/index.browser.ts +++ b/sdk/packages/shared/src/index.browser.ts @@ -194,7 +194,7 @@ export { resolveReasoningEffortRatio, } from "./llms/reasoning-effort"; export { DEFAULT_REQUEST_HEADERS, serializeAbortReason } from "./llms/requests"; -export { estimateTokens } from "./llms/tokens"; +export { CHARS_PER_TOKEN, estimateTokens } from "./llms/tokens"; export type { ToolApprovalRequest, ToolApprovalResult, diff --git a/sdk/packages/shared/src/index.ts b/sdk/packages/shared/src/index.ts index 6c804c67b2..a9b5ec5c76 100644 --- a/sdk/packages/shared/src/index.ts +++ b/sdk/packages/shared/src/index.ts @@ -208,7 +208,7 @@ export { resolveReasoningEffortRatio, } from "./llms/reasoning-effort"; export { DEFAULT_REQUEST_HEADERS, serializeAbortReason } from "./llms/requests"; -export { estimateTokens } from "./llms/tokens"; +export { CHARS_PER_TOKEN, estimateTokens } from "./llms/tokens"; export type { ToolApprovalRequest, ToolApprovalResult, diff --git a/sdk/packages/shared/src/llms/tokens.ts b/sdk/packages/shared/src/llms/tokens.ts index f43c759df9..49e232a167 100644 --- a/sdk/packages/shared/src/llms/tokens.ts +++ b/sdk/packages/shared/src/llms/tokens.ts @@ -5,7 +5,7 @@ * rather than after. */ -const CHARS_PER_TOKEN = 3; +export const CHARS_PER_TOKEN = 3; export function estimateTokens(chars: number): number { return Math.max(1, Math.ceil(chars / CHARS_PER_TOKEN));