Compare commits

...
5 changed files with 309 additions and 93 deletions
@@ -138,7 +138,7 @@ describe("compactInteractiveMessages", () => {
}));
const config = createConfig();
const compact = vi.fn((context: CoreCompactionContext) => {
expect(context.maxInputTokens).toBe(400_000);
expect(context.maxInputTokens).toBe(383_616);
return { messages: [messages[0]] };
});
config.knownModels = {
@@ -16,7 +16,12 @@ import type { ProviderConfig } from "../../types/provider-settings";
export const DEFAULT_MAX_INPUT_TOKENS = 128_000;
export const DEFAULT_THRESHOLD_RATIO = 0.9;
export const DEFAULT_TARGET_RATIO = 0.7;
export const DEFAULT_RESERVE_TOKENS = 16_384;
/**
* Estimated output reserve for shared-context models that do not declare
* `maxTokens`. Only consulted in that fallback; explicit input ceilings
* (config or true input limits) never reserve output.
*/
export const FALLBACK_OUTPUT_RESERVE_TOKENS = 16_384;
export const DEFAULT_PRESERVE_RECENT_TOKENS = 20_000;
export const DEFAULT_SUMMARY_MAX_OUTPUT_TOKENS = 1_024;
export const TOOL_RESULT_CHAR_LIMIT = 2_000;
@@ -1105,7 +1105,7 @@ describe("createContextCompactionPrepareTurn", () => {
}
});
it("uses the default reserve when no trigger is configured", async () => {
it("uses the true input budget when no context window is available", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [{ role: "user" as const, content: "Compacted by reserve" }],
}));
@@ -1146,7 +1146,7 @@ describe("createContextCompactionPrepareTurn", () => {
expect(createHandlerMock).not.toHaveBeenCalled();
expect(compact).toHaveBeenCalledTimes(1);
const context = compact.mock.calls[0]?.[0];
expect(context?.triggerTokens).toBe(0);
expect(context?.triggerTokens).toBe(180);
expect(result?.messages).toEqual([
{ role: "user", content: "Compacted by reserve" },
]);
@@ -1367,7 +1367,7 @@ describe("createContextCompactionPrepareTurn", () => {
expect(context?.targetTokens).toBe(39);
});
it("derives input budget by reserving model max output tokens from context window", async () => {
it("reserves output once for a shared context window", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [
{ role: "user" as const, content: "Compacted by derived input budget" },
@@ -1421,6 +1421,191 @@ describe("createContextCompactionPrepareTurn", () => {
]);
});
it("does not double-reserve output for mirrored Qwen context metadata", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [{ role: "user" as const, content: "Compacted Qwen" }],
}));
const prepareTurn = createContextCompactionPrepareTurn({
providerId: "openrouter",
modelId: "qwen/qwen3-32b",
providerConfig: {
providerId: "openrouter",
modelId: "qwen/qwen3-32b",
} as LlmsProviders.ProviderConfig,
compaction: { enabled: true, compact },
logger: undefined,
});
const messages: MessageWithMetadata[] = [
{ role: "user", content: "large prompt ".repeat(6_000) },
];
await prepareTurn?.({
agentId: "agent-1",
conversationId: "conv-1",
parentAgentId: null,
iteration: 1,
abortSignal: new AbortController().signal,
systemPrompt: "You are helpful.",
tools: [],
messages,
apiMessages: messages,
model: {
id: "qwen/qwen3-32b",
provider: "openrouter",
info: {
id: "qwen/qwen3-32b",
contextWindow: 40_960,
maxInputTokens: 40_960,
maxTokens: 16_384,
},
},
});
expect(compact).toHaveBeenCalledTimes(1);
const context = compact.mock.calls[0]?.[0];
expect(context?.maxInputTokens).toBe(24_576);
expect(context?.triggerTokens).toBe(22_118);
expect(context?.thresholdRatio).toBe(22_118 / 24_576);
});
it("caps shared-context output reserve at half the context window", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [{ role: "user" as const, content: "Compacted capped output" }],
}));
const prepareTurn = createContextCompactionPrepareTurn({
providerId: "kilo",
modelId: "qwen/qwen3-32b",
providerConfig: {
providerId: "kilo",
modelId: "qwen/qwen3-32b",
} as LlmsProviders.ProviderConfig,
compaction: { enabled: true, compact },
logger: undefined,
});
const messages: MessageWithMetadata[] = [
{ role: "user", content: "large prompt ".repeat(5_000) },
];
await prepareTurn?.({
agentId: "agent-1",
conversationId: "conv-1",
parentAgentId: null,
iteration: 1,
abortSignal: new AbortController().signal,
systemPrompt: "You are helpful.",
tools: [],
messages,
apiMessages: messages,
model: {
id: "qwen/qwen3-32b",
provider: "kilo",
info: {
id: "qwen/qwen3-32b",
contextWindow: 40_960,
maxInputTokens: 40_960,
maxTokens: 40_960,
},
},
});
expect(compact).toHaveBeenCalledTimes(1);
const context = compact.mock.calls[0]?.[0];
expect(context?.maxInputTokens).toBe(20_480);
expect(context?.triggerTokens).toBe(18_432);
});
it("applies explicit threshold ratio to the usable budget", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [{ role: "user" as const, content: "Compacted threshold" }],
}));
const prepareTurn = createContextCompactionPrepareTurn({
providerId: "openrouter",
modelId: "qwen/qwen3-32b",
providerConfig: {
providerId: "openrouter",
modelId: "qwen/qwen3-32b",
} as LlmsProviders.ProviderConfig,
compaction: { enabled: true, thresholdRatio: 0.8, compact },
logger: undefined,
});
const messages: MessageWithMetadata[] = [
{ role: "user", content: "large prompt ".repeat(6_000) },
];
await prepareTurn?.({
agentId: "agent-1",
conversationId: "conv-1",
parentAgentId: null,
iteration: 1,
abortSignal: new AbortController().signal,
systemPrompt: "You are helpful.",
tools: [],
messages,
apiMessages: messages,
model: {
id: "qwen/qwen3-32b",
provider: "openrouter",
info: {
id: "qwen/qwen3-32b",
contextWindow: 40_960,
maxInputTokens: 40_960,
maxTokens: 16_384,
},
},
});
expect(compact).toHaveBeenCalledTimes(1);
const context = compact.mock.calls[0]?.[0];
expect(context?.triggerTokens).toBe(19_660);
expect(context?.thresholdRatio).toBe(19_660 / 24_576);
});
it("applies explicit reserve tokens to the usable budget", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [{ role: "user" as const, content: "Compacted reserve" }],
}));
const prepareTurn = createContextCompactionPrepareTurn({
providerId: "openrouter",
modelId: "qwen/qwen3-32b",
providerConfig: {
providerId: "openrouter",
modelId: "qwen/qwen3-32b",
} as LlmsProviders.ProviderConfig,
compaction: { enabled: true, reserveTokens: 4_096, compact },
logger: undefined,
});
const messages: MessageWithMetadata[] = [
{ role: "user", content: "large prompt ".repeat(6_000) },
];
await prepareTurn?.({
agentId: "agent-1",
conversationId: "conv-1",
parentAgentId: null,
iteration: 1,
abortSignal: new AbortController().signal,
systemPrompt: "You are helpful.",
tools: [],
messages,
apiMessages: messages,
model: {
id: "qwen/qwen3-32b",
provider: "openrouter",
info: {
id: "qwen/qwen3-32b",
contextWindow: 40_960,
maxInputTokens: 40_960,
maxTokens: 16_384,
},
},
});
expect(compact).toHaveBeenCalledTimes(1);
const context = compact.mock.calls[0]?.[0];
expect(context?.triggerTokens).toBe(20_480);
expect(context?.thresholdRatio).toBe(20_480 / 24_576);
});
it("uses the lower split input budget when it is below context-derived input budget", async () => {
const compact = vi.fn((_context: CoreCompactionContext) => ({
messages: [
@@ -22,9 +22,9 @@ import {
createTokenEstimator,
DEFAULT_MAX_INPUT_TOKENS,
DEFAULT_PRESERVE_RECENT_TOKENS,
DEFAULT_RESERVE_TOKENS,
DEFAULT_TARGET_RATIO,
DEFAULT_THRESHOLD_RATIO,
FALLBACK_OUTPUT_RESERVE_TOKENS,
} from "./compaction-shared";
export interface ContextPipelinePrepareTurnInput {
@@ -78,7 +78,6 @@ export interface ContextCompactionPrepareTurnOptions {
manualTargetRatio?: number;
}
const MIN_CONTEXT_DERIVED_INPUT_RATIO = 0.5;
const LONG_CONVERSATION_TARGET_RATIO = 0.5;
function safeJsonSize(value: unknown): number {
@@ -93,35 +92,86 @@ function isPositiveFiniteNumber(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value) && value > 0;
}
function resolveMaxInputTokens(input: {
configMaxInputTokens?: number;
type CompactionBudgetSource = "config" | "true_input" | "context" | "default";
interface CompactionBudget {
ceilingTokens: number;
usableBudgetTokens: number;
outputReserveTokens: number;
triggerTokens: number;
thresholdRatio: number;
ceilingSource: CompactionBudgetSource;
reserveCapped: boolean;
}
function resolveCompactionBudget(input: {
config: CoreCompactionConfig;
modelMaxInputTokens?: number;
contextWindow?: number;
modelMaxTokens?: number;
}): number {
const candidates: number[] = [];
if (isPositiveFiniteNumber(input.configMaxInputTokens)) {
candidates.push(input.configMaxInputTokens);
}): CompactionBudget {
let ceilingTokens = DEFAULT_MAX_INPUT_TOKENS;
let ceilingSource: CompactionBudgetSource = "default";
if (isPositiveFiniteNumber(input.config.maxInputTokens)) {
ceilingTokens = input.config.maxInputTokens;
ceilingSource = "config";
} else if (
isPositiveFiniteNumber(input.modelMaxInputTokens) &&
(!isPositiveFiniteNumber(input.contextWindow) ||
input.modelMaxInputTokens < input.contextWindow)
) {
ceilingTokens = input.modelMaxInputTokens;
ceilingSource = "true_input";
} else if (isPositiveFiniteNumber(input.contextWindow)) {
ceilingTokens = input.contextWindow;
ceilingSource = "context";
}
if (isPositiveFiniteNumber(input.modelMaxInputTokens)) {
candidates.push(input.modelMaxInputTokens);
// Output space is reserved once, and only when the ceiling is a shared
// context window. Explicit config and true input limits already describe
// input-only budgets, so reserving there would double-count output.
const maxReserveTokens = Math.floor(ceilingTokens / 2);
const outputReserveTokens =
ceilingSource === "config" ||
ceilingSource === "true_input" ||
ceilingSource === "default"
? 0
: isPositiveFiniteNumber(input.modelMaxTokens)
? Math.min(input.modelMaxTokens, maxReserveTokens)
: Math.min(
FALLBACK_OUTPUT_RESERVE_TOKENS,
Math.floor(ceilingTokens * 0.25),
);
const usableBudgetTokens = Math.max(1, ceilingTokens - outputReserveTokens);
let triggerTokens: number;
if (typeof input.config.reserveTokens === "number") {
triggerTokens = Math.max(
0,
usableBudgetTokens - Math.max(0, input.config.reserveTokens),
);
} else if (typeof input.config.thresholdRatio === "number") {
triggerTokens = Math.floor(
usableBudgetTokens * input.config.thresholdRatio,
);
} else {
triggerTokens = Math.floor(usableBudgetTokens * DEFAULT_THRESHOLD_RATIO);
}
if (isPositiveFiniteNumber(input.contextWindow)) {
candidates.push(input.contextWindow);
const derivedInputTokens = isPositiveFiniteNumber(input.modelMaxTokens)
? input.contextWindow - input.modelMaxTokens
: undefined;
if (
isPositiveFiniteNumber(derivedInputTokens) &&
derivedInputTokens >=
input.contextWindow * MIN_CONTEXT_DERIVED_INPUT_RATIO
) {
candidates.push(derivedInputTokens);
}
}
return candidates.length > 0
? Math.min(...candidates)
: DEFAULT_MAX_INPUT_TOKENS;
return {
ceilingTokens,
usableBudgetTokens,
outputReserveTokens,
triggerTokens,
thresholdRatio:
usableBudgetTokens > 0 ? triggerTokens / usableBudgetTokens : 0,
ceilingSource,
reserveCapped:
ceilingSource === "context" &&
isPositiveFiniteNumber(input.modelMaxTokens) &&
input.modelMaxTokens > maxReserveTokens,
};
}
function summarizeToolResults(messages: CoreCompactionContext["messages"]): {
@@ -189,47 +239,6 @@ const BUILTIN_COMPACTION_STRATEGIES = {
}),
} satisfies Record<CoreCompactionStrategy, BuiltinCompactionStrategyRunner>;
function resolveTriggerState(input: {
inputTokens: number;
maxInputTokens: number;
config: CoreCompactionConfig;
}): { shouldCompact: boolean; triggerTokens: number; thresholdRatio: number } {
if (typeof input.config.reserveTokens === "number") {
const reserveTokens = Math.max(0, input.config.reserveTokens);
const triggerTokens = Math.max(0, input.maxInputTokens - reserveTokens);
return {
shouldCompact: input.inputTokens > triggerTokens,
triggerTokens,
thresholdRatio:
input.maxInputTokens > 0 ? triggerTokens / input.maxInputTokens : 0,
};
}
if (typeof input.config.thresholdRatio === "number") {
const thresholdRatio = input.config.thresholdRatio;
const triggerTokens = input.maxInputTokens * thresholdRatio;
return {
shouldCompact: input.inputTokens > triggerTokens,
triggerTokens,
thresholdRatio,
};
}
const triggerTokens = Math.max(
0,
Math.min(
input.maxInputTokens - DEFAULT_RESERVE_TOKENS,
input.maxInputTokens * DEFAULT_THRESHOLD_RATIO,
),
);
return {
shouldCompact: input.inputTokens > triggerTokens,
triggerTokens,
thresholdRatio:
input.maxInputTokens > 0 ? triggerTokens / input.maxInputTokens : 0,
};
}
function resolveManualTargetState(input: {
inputTokens: number;
maxInputTokens: number;
@@ -258,7 +267,8 @@ function resolveManualTargetState(input: {
}
function resolveBasicTargetTokens(input: {
maxInputTokens: number;
ceilingTokens: number;
usableBudgetTokens: number;
modelMaxTokens?: number;
triggerTokens: number;
messagePairCount: number;
@@ -267,13 +277,13 @@ function resolveBasicTargetTokens(input: {
input.messagePairCount >= 5 &&
typeof input.modelMaxTokens === "number" &&
Number.isFinite(input.modelMaxTokens) &&
input.modelMaxTokens < input.maxInputTokens
? Math.floor(input.maxInputTokens * LONG_CONVERSATION_TARGET_RATIO)
input.modelMaxTokens < input.ceilingTokens
? Math.floor(input.usableBudgetTokens * LONG_CONVERSATION_TARGET_RATIO)
: Math.floor(input.triggerTokens * DEFAULT_TARGET_RATIO);
const triggerCeiling = Math.max(1, input.triggerTokens - 1);
return Math.max(
1,
Math.min(targetTokens, input.maxInputTokens, triggerCeiling),
Math.min(targetTokens, input.usableBudgetTokens, triggerCeiling),
);
}
@@ -348,22 +358,18 @@ export function createContextCompactionPrepareTurn(
(total: number, message) => total + estimateMessageTokens(message),
0,
);
const maxInputTokens = resolveMaxInputTokens({
configMaxInputTokens: userCompaction?.maxInputTokens,
modelMaxInputTokens: context.model.info?.maxInputTokens,
contextWindow: context.model.info?.contextWindow,
modelMaxTokens: context.model.info?.maxTokens,
});
const triggerState = resolveTriggerState({
inputTokens,
maxInputTokens,
const compactionBudget = resolveCompactionBudget({
config: {
maxInputTokens: userCompaction?.maxInputTokens,
reserveTokens: userCompaction?.reserveTokens,
thresholdRatio: userCompaction?.thresholdRatio,
},
modelMaxInputTokens: context.model.info?.maxInputTokens,
contextWindow: context.model.info?.contextWindow,
modelMaxTokens: context.model.info?.maxTokens,
});
const maxInputTokens = compactionBudget.usableBudgetTokens;
const shouldCompact = inputTokens > compactionBudget.triggerTokens;
config.logger?.debug("Context compaction diagnostics", {
mode,
strategy,
@@ -372,15 +378,19 @@ export function createContextCompactionPrepareTurn(
modelId: config.modelId,
inputTokens,
maxInputTokens,
triggerTokens: triggerState.triggerTokens,
thresholdRatio: triggerState.thresholdRatio,
shouldCompact: triggerState.shouldCompact,
ceilingTokens: compactionBudget.ceilingTokens,
outputReserveTokens: compactionBudget.outputReserveTokens,
ceilingSource: compactionBudget.ceilingSource,
reserveCapped: compactionBudget.reserveCapped,
triggerTokens: compactionBudget.triggerTokens,
thresholdRatio: compactionBudget.thresholdRatio,
shouldCompact,
messageCount: context.messages.length,
apiMessageCount: context.apiMessages.length,
apiMessagesJsonChars: safeJsonSize(context.apiMessages),
...summarizeToolResults(context.apiMessages),
});
if (mode === "auto" && !triggerState.shouldCompact) {
if (mode === "auto" && !shouldCompact) {
return undefined;
}
const targetState =
@@ -388,14 +398,15 @@ export function createContextCompactionPrepareTurn(
? resolveManualTargetState({
inputTokens,
maxInputTokens,
autoTriggerTokens: triggerState.triggerTokens,
autoTriggerTokens: compactionBudget.triggerTokens,
manualTargetRatio: options.manualTargetRatio,
})
: triggerState;
: compactionBudget;
const targetTokens =
mode === "auto"
? resolveBasicTargetTokens({
maxInputTokens,
ceilingTokens: compactionBudget.ceilingTokens,
usableBudgetTokens: compactionBudget.usableBudgetTokens,
modelMaxTokens: context.model.info?.maxTokens,
triggerTokens: targetState.triggerTokens,
messagePairCount: countUserAssistantPairs(context.messages),
@@ -426,6 +437,9 @@ export function createContextCompactionPrepareTurn(
iteration: context.iteration,
triggerTokens: targetState.triggerTokens,
maxInputTokens,
ceilingTokens: compactionBudget.ceilingTokens,
outputReserveTokens: compactionBudget.outputReserveTokens,
ceilingSource: compactionBudget.ceilingSource,
},
);
@@ -467,6 +481,10 @@ export function createContextCompactionPrepareTurn(
severity: "info",
strategy: strategy,
maxInputTokens,
ceilingTokens: compactionBudget.ceilingTokens,
outputReserveTokens: compactionBudget.outputReserveTokens,
ceilingSource: compactionBudget.ceilingSource,
reserveCapped: compactionBudget.reserveCapped,
inputTokens,
afterTokens,
tokensSaved: inputTokens - afterTokens,
+8
View File
@@ -66,9 +66,17 @@ export interface CoreCompactionContext {
provider: string;
info?: ModelInfo;
};
/**
* Usable input budget after reserving any shared context window space needed
* for model output.
*/
maxInputTokens: number;
triggerTokens: number;
targetTokens?: number;
/**
* Effective trigger point as a fraction of the usable input budget
* (`triggerTokens / maxInputTokens`).
*/
thresholdRatio: number;
utilizationRatio: number;
}