mirror of
https://github.com/musistudio/claude-code-router.git
synced 2026-08-28 19:01:32 +08:00
Add Claude compact archive support
This commit is contained in:
musi
@@ -0,0 +1,491 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { defaultTaskCaseId, findTaskCase, taskCaseIds } from "./context-archive-task-cases.mjs";
|
||||
|
||||
const defaultTargetInputTokens = 180000;
|
||||
const defaultMinInputTokens = 170000;
|
||||
const defaultMaxInputTokens = 190000;
|
||||
const defaultMaxBudgetUsd = "5";
|
||||
|
||||
async function main(argv = process.argv.slice(2)) {
|
||||
const options = parseArgs(argv);
|
||||
if (options.listTaskCases) {
|
||||
console.log(taskCaseIds().join("\n"));
|
||||
return;
|
||||
}
|
||||
const taskCase = findTaskCase(options.taskCase);
|
||||
const sessionId = options.sessionId || randomUUID();
|
||||
const steps = [];
|
||||
|
||||
let lineCursor = 0;
|
||||
let currentInputTokens = 0;
|
||||
let loadLineCount = options.lineCount ?? estimateLineCount(options.targetInputTokens);
|
||||
let loadPrompt = buildContextPrompt({
|
||||
lineCount: loadLineCount,
|
||||
lineOffset: lineCursor,
|
||||
taskCase,
|
||||
title: "initial",
|
||||
totalPlannedLines: loadLineCount
|
||||
});
|
||||
lineCursor += loadLineCount;
|
||||
|
||||
const load = await runClaude({
|
||||
input: loadPrompt,
|
||||
label: "load-context",
|
||||
options,
|
||||
sessionId
|
||||
});
|
||||
steps.push(load);
|
||||
currentInputTokens = totalInputTokens(load.result);
|
||||
|
||||
for (let attempt = 0; attempt < options.maxAppendAttempts && currentInputTokens < options.minInputTokens; attempt += 1) {
|
||||
const observedContextTokens = Math.max(1, currentInputTokens - basePromptOverhead(load.result));
|
||||
const tokensPerLine = Math.max(4, observedContextTokens / Math.max(1, lineCursor));
|
||||
const missingTokens = options.targetInputTokens - currentInputTokens;
|
||||
const appendLineCount = Math.max(200, Math.ceil((missingTokens / tokensPerLine) * 1.05));
|
||||
const appendPrompt = buildContextPrompt({
|
||||
lineCount: appendLineCount,
|
||||
lineOffset: lineCursor,
|
||||
taskCase,
|
||||
title: `append-${attempt + 1}`,
|
||||
totalPlannedLines: lineCursor + appendLineCount
|
||||
});
|
||||
lineCursor += appendLineCount;
|
||||
|
||||
const append = await runClaude({
|
||||
input: appendPrompt,
|
||||
label: `append-context-${attempt + 1}`,
|
||||
options,
|
||||
resume: sessionId
|
||||
});
|
||||
steps.push(append);
|
||||
currentInputTokens = totalInputTokens(append.result);
|
||||
}
|
||||
|
||||
const anchor = await runClaude({
|
||||
input: [
|
||||
"Read this benchmark anchor and current task state for the current conversation only. Do not write memory, do not use tools, and do not persist anything outside this chat.",
|
||||
"This packet is the latest task progress immediately before /compact.",
|
||||
`The loaded synthetic context contains ${lineCursor} generated lines.`,
|
||||
"CURRENT_TASK_STATE_PACKET_BEGIN",
|
||||
`objective=${factMarker(taskCase, "objective")}`,
|
||||
`completed=${factMarker(taskCase, "completed")}`,
|
||||
`current_focus=${factMarker(taskCase, "currentFocus")}`,
|
||||
`next_step=${factMarker(taskCase, "nextStep")}`,
|
||||
`validation_command=${factMarker(taskCase, "validationCommand")}`,
|
||||
`risk=${factMarker(taskCase, "risk")}`,
|
||||
"CURRENT_TASK_STATE_PACKET_END",
|
||||
"Reply exactly ACK_READY_TO_COMPACT."
|
||||
].join("\n"),
|
||||
label: "anchor-turn",
|
||||
options,
|
||||
resume: sessionId
|
||||
});
|
||||
steps.push(anchor);
|
||||
currentInputTokens = totalInputTokens(anchor.result);
|
||||
|
||||
const compact = await runClaude({
|
||||
input: "/compact",
|
||||
label: "slash-compact",
|
||||
options,
|
||||
resume: sessionId
|
||||
});
|
||||
steps.push(compact);
|
||||
|
||||
const continuity = await runClaude({
|
||||
input: continuityProbePrompt(taskCase),
|
||||
label: "continuity-probe",
|
||||
options,
|
||||
resume: sessionId
|
||||
});
|
||||
steps.push(continuity);
|
||||
const continuityEval = evaluateContinuity(taskCase, continuity.result);
|
||||
|
||||
const report = {
|
||||
compact: summarizeResult(compact.result),
|
||||
constructedContext: {
|
||||
inputTokensBeforeCompact: currentInputTokens,
|
||||
lineCount: lineCursor,
|
||||
maxInputTokens: options.maxInputTokens,
|
||||
minInputTokens: options.minInputTokens,
|
||||
nearTarget: currentInputTokens >= options.minInputTokens && currentInputTokens <= options.maxInputTokens,
|
||||
targetInputTokens: options.targetInputTokens
|
||||
},
|
||||
continuity: continuityEval,
|
||||
options: publicOptions(options),
|
||||
sessionId,
|
||||
taskCase: {
|
||||
id: taskCase.id,
|
||||
title: taskCase.title
|
||||
},
|
||||
steps: steps.map((step) => ({
|
||||
elapsedMs: Math.round(step.elapsedMs),
|
||||
label: step.label,
|
||||
stderr: step.stderr.trim(),
|
||||
summary: summarizeResult(step.result)
|
||||
}))
|
||||
};
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
printReport(report);
|
||||
}
|
||||
|
||||
if (
|
||||
compact.result.duration_api_ms === 0 ||
|
||||
/not enough messages to compact/i.test(String(compact.result.result ?? "")) ||
|
||||
continuityEval.recall < options.minContinuityRecall
|
||||
) {
|
||||
process.exitCode = 2;
|
||||
}
|
||||
}
|
||||
|
||||
function buildContextPrompt(input) {
|
||||
const lines = [
|
||||
`You are loading synthetic benchmark context block ${input.title}.`,
|
||||
`Task case: ${input.taskCase.id} - ${input.taskCase.title}.`,
|
||||
"Do not summarize the block. Do not call tools. Reply exactly ACK_CONTEXT_BLOCK_LOADED.",
|
||||
"The following lines are synthetic and intentionally verbose to fill the Claude Code context window.",
|
||||
input.lineOffset === 0 ? "EARLY_DESIGN_DECISION_BEGIN" : undefined,
|
||||
input.lineOffset === 0 ? `early_decision=${factMarker(input.taskCase, "earlyDecision")}` : undefined,
|
||||
input.lineOffset === 0 ? "EARLY_DESIGN_DECISION_END" : undefined,
|
||||
"BEGIN_SYNTHETIC_CONTEXT"
|
||||
].filter(Boolean);
|
||||
for (let index = 0; index < input.lineCount; index += 1) {
|
||||
const id = input.lineOffset + index + 1;
|
||||
const padded = String(id).padStart(6, "0");
|
||||
lines.push([
|
||||
`段落${padded}`,
|
||||
`ctx_marker_${padded}`,
|
||||
input.taskCase.filler[index % input.taskCase.filler.length],
|
||||
`事实编号${padded}要求压缩摘要能够通过历史检索恢复精确细节。`,
|
||||
`unique_alpha_${padded}_unique_beta_${padded}_unique_gamma_${padded}.`
|
||||
].join(" "));
|
||||
if (id === Math.floor(input.totalPlannedLines / 2)) {
|
||||
lines.push("MID_SESSION_PROGRESS_BEGIN");
|
||||
lines.push(`mid_progress=${factMarker(input.taskCase, "midProgress")}`);
|
||||
lines.push("MID_SESSION_PROGRESS_END");
|
||||
}
|
||||
}
|
||||
lines.push("END_SYNTHETIC_CONTEXT");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function continuityProbePrompt(taskCase) {
|
||||
return [
|
||||
"This is a post-/compact continuity probe.",
|
||||
"Do not use tools. Use only the compressed conversation state.",
|
||||
"Return the exact marker values you remember. If a value is absent, write UNKNOWN.",
|
||||
"Use this exact line format, one key per line:",
|
||||
...taskCase.facts.map((fact) => `${fact.key}=<${fact.prompt}>`)
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function evaluateContinuity(taskCase, result) {
|
||||
const text = String(result.result ?? "");
|
||||
const items = taskCase.facts.map((fact) => ({
|
||||
key: fact.key,
|
||||
marker: fact.marker,
|
||||
found: text.includes(fact.marker)
|
||||
}));
|
||||
const found = items.filter((item) => item.found).length;
|
||||
return {
|
||||
found,
|
||||
items,
|
||||
outputPreview: text.slice(0, 1200),
|
||||
recall: items.length > 0 ? found / items.length : 0,
|
||||
total: items.length
|
||||
};
|
||||
}
|
||||
|
||||
function factMarker(taskCase, key) {
|
||||
const fact = taskCase.facts.find((item) => item.key === key);
|
||||
if (!fact) {
|
||||
throw new Error(`Unknown continuity fact in ${taskCase.id}: ${key}`);
|
||||
}
|
||||
return fact.marker;
|
||||
}
|
||||
|
||||
async function runClaude(input) {
|
||||
const started = performance.now();
|
||||
const args = [
|
||||
"-p",
|
||||
"--output-format",
|
||||
"json",
|
||||
"--tools",
|
||||
"",
|
||||
"--max-budget-usd",
|
||||
input.options.maxBudgetUsd
|
||||
];
|
||||
if (input.options.model) {
|
||||
args.push("--model", input.options.model);
|
||||
}
|
||||
if (input.options.debugFile) {
|
||||
args.push("--debug-file", input.options.debugFile);
|
||||
}
|
||||
if (input.resume) {
|
||||
args.push("--resume", input.resume);
|
||||
} else {
|
||||
args.push("--session-id", input.sessionId);
|
||||
}
|
||||
|
||||
const child = spawn(input.options.claudeBin, args, {
|
||||
cwd: input.options.cwd,
|
||||
env: process.env,
|
||||
stdio: ["pipe", "pipe", "pipe"]
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.stdin.end(input.input);
|
||||
|
||||
const code = await new Promise((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("exit", (exitCode) => resolve(exitCode ?? 1));
|
||||
});
|
||||
const elapsedMs = performance.now() - started;
|
||||
const result = parseClaudeJson(stdout);
|
||||
if (code !== 0 && !result) {
|
||||
throw new Error([
|
||||
`claude exited with code ${code} during ${input.label}.`,
|
||||
stderr.trim(),
|
||||
stdout.trim()
|
||||
].filter(Boolean).join("\n"));
|
||||
}
|
||||
return {
|
||||
elapsedMs,
|
||||
label: input.label,
|
||||
result: result ?? {
|
||||
is_error: true,
|
||||
raw_stdout: stdout,
|
||||
session_id: input.resume || input.sessionId,
|
||||
terminal_reason: `exit_${code}`
|
||||
},
|
||||
stderr
|
||||
};
|
||||
}
|
||||
|
||||
function parseClaudeJson(stdout) {
|
||||
const trimmed = stdout.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(trimmed);
|
||||
} catch {
|
||||
const firstBrace = trimmed.indexOf("{");
|
||||
const lastBrace = trimmed.lastIndexOf("}");
|
||||
if (firstBrace >= 0 && lastBrace > firstBrace) {
|
||||
return JSON.parse(trimmed.slice(firstBrace, lastBrace + 1));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeResult(result) {
|
||||
const modelUsage = aggregateModelUsage(result.modelUsage);
|
||||
return {
|
||||
apiMs: result.duration_api_ms ?? 0,
|
||||
cacheCreationInputTokens: result.usage?.cache_creation_input_tokens ?? 0,
|
||||
cacheReadInputTokens: result.usage?.cache_read_input_tokens ?? 0,
|
||||
costUsd: result.total_cost_usd ?? 0,
|
||||
inputTokens: result.usage?.input_tokens ?? 0,
|
||||
isError: Boolean(result.is_error),
|
||||
modelInputTokens: modelUsage.inputTokens,
|
||||
modelOutputTokens: modelUsage.outputTokens,
|
||||
modelUsage: result.modelUsage ?? {},
|
||||
outputPreview: typeof result.result === "string" ? result.result.slice(0, 300) : undefined,
|
||||
outputTokens: result.usage?.output_tokens ?? 0,
|
||||
sessionId: result.session_id,
|
||||
stopReason: result.stop_reason ?? null,
|
||||
terminalReason: result.terminal_reason ?? null,
|
||||
totalInputTokens: totalInputTokens(result),
|
||||
totalOutputTokens: totalOutputTokens(result)
|
||||
};
|
||||
}
|
||||
|
||||
function totalInputTokens(result) {
|
||||
const topLevel = Number(result.usage?.input_tokens ?? 0) +
|
||||
Number(result.usage?.cache_creation_input_tokens ?? 0) +
|
||||
Number(result.usage?.cache_read_input_tokens ?? 0);
|
||||
return topLevel > 0 ? topLevel : aggregateModelUsage(result.modelUsage).inputTokens;
|
||||
}
|
||||
|
||||
function totalOutputTokens(result) {
|
||||
const topLevel = Number(result.usage?.output_tokens ?? 0);
|
||||
return topLevel > 0 ? topLevel : aggregateModelUsage(result.modelUsage).outputTokens;
|
||||
}
|
||||
|
||||
function aggregateModelUsage(modelUsage) {
|
||||
const totals = {
|
||||
contextWindow: 0,
|
||||
inputTokens: 0,
|
||||
outputTokens: 0
|
||||
};
|
||||
if (!modelUsage || typeof modelUsage !== "object") {
|
||||
return totals;
|
||||
}
|
||||
for (const value of Object.values(modelUsage)) {
|
||||
if (!value || typeof value !== "object") {
|
||||
continue;
|
||||
}
|
||||
totals.contextWindow = Math.max(totals.contextWindow, Number(value.contextWindow ?? 0));
|
||||
totals.inputTokens += Number(value.inputTokens ?? 0) +
|
||||
Number(value.cacheCreationInputTokens ?? 0) +
|
||||
Number(value.cacheReadInputTokens ?? 0);
|
||||
totals.outputTokens += Number(value.outputTokens ?? 0);
|
||||
}
|
||||
return totals;
|
||||
}
|
||||
|
||||
function basePromptOverhead(result) {
|
||||
return Math.min(3000, Math.max(0, totalInputTokens(result) - Number(result.usage?.output_tokens ?? 0)));
|
||||
}
|
||||
|
||||
function estimateLineCount(targetInputTokens) {
|
||||
return Math.max(500, Math.round(targetInputTokens / 65));
|
||||
}
|
||||
|
||||
function printReport(report) {
|
||||
console.log(`Claude Code real /compact benchmark session: ${report.sessionId}`);
|
||||
console.log(`task case: ${report.taskCase.id} - ${report.taskCase.title}`);
|
||||
console.log(`constructed input tokens before /compact: ${report.constructedContext.inputTokensBeforeCompact}`);
|
||||
console.log(`target range: ${report.constructedContext.minInputTokens}-${report.constructedContext.maxInputTokens}`);
|
||||
console.log(`near target: ${report.constructedContext.nearTarget ? "yes" : "no"}`);
|
||||
console.log(`continuity recall: ${report.continuity.found}/${report.continuity.total} (${(report.continuity.recall * 100).toFixed(0)}%)`);
|
||||
console.log("");
|
||||
for (const step of report.steps) {
|
||||
console.log([
|
||||
step.label.padEnd(18),
|
||||
`input=${step.summary.totalInputTokens}`,
|
||||
`output=${step.summary.totalOutputTokens}`,
|
||||
`apiMs=${step.summary.apiMs}`,
|
||||
`cost=$${Number(step.summary.costUsd).toFixed(4)}`,
|
||||
`error=${step.summary.isError ? "yes" : "no"}`,
|
||||
step.summary.outputPreview ? `result=${JSON.stringify(step.summary.outputPreview)}` : undefined
|
||||
].filter(Boolean).join(" "));
|
||||
}
|
||||
console.log("");
|
||||
console.log("continuity facts:");
|
||||
for (const item of report.continuity.items) {
|
||||
console.log(`${item.found ? "PASS" : "MISS"} ${item.key}=${item.marker}`);
|
||||
}
|
||||
}
|
||||
|
||||
function publicOptions(options) {
|
||||
return {
|
||||
claudeBin: options.claudeBin,
|
||||
cwd: options.cwd,
|
||||
debugFile: options.debugFile,
|
||||
json: options.json,
|
||||
lineCount: options.lineCount,
|
||||
listTaskCases: options.listTaskCases,
|
||||
maxAppendAttempts: options.maxAppendAttempts,
|
||||
maxBudgetUsd: options.maxBudgetUsd,
|
||||
maxInputTokens: options.maxInputTokens,
|
||||
minContinuityRecall: options.minContinuityRecall,
|
||||
minInputTokens: options.minInputTokens,
|
||||
model: options.model,
|
||||
taskCase: options.taskCase,
|
||||
targetInputTokens: options.targetInputTokens
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
claudeBin: process.env.CLAUDE_BIN || "claude",
|
||||
cwd: process.cwd(),
|
||||
debugFile: undefined,
|
||||
json: false,
|
||||
lineCount: undefined,
|
||||
listTaskCases: false,
|
||||
maxAppendAttempts: 2,
|
||||
maxBudgetUsd: defaultMaxBudgetUsd,
|
||||
maxInputTokens: defaultMaxInputTokens,
|
||||
minContinuityRecall: 0,
|
||||
minInputTokens: defaultMinInputTokens,
|
||||
model: undefined,
|
||||
sessionId: undefined,
|
||||
taskCase: defaultTaskCaseId,
|
||||
targetInputTokens: defaultTargetInputTokens
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const [name, inlineValue] = arg.includes("=") ? arg.split(/=(.*)/s, 2) : [arg, undefined];
|
||||
const readValue = () => inlineValue ?? argv[++index];
|
||||
if (name === "--claude-bin") {
|
||||
options.claudeBin = requireValue(readValue(), name);
|
||||
} else if (name === "--cwd") {
|
||||
options.cwd = requireValue(readValue(), name);
|
||||
} else if (name === "--debug-file") {
|
||||
options.debugFile = requireValue(readValue(), name);
|
||||
} else if (name === "--json") {
|
||||
options.json = true;
|
||||
} else if (name === "--list-task-cases") {
|
||||
options.listTaskCases = true;
|
||||
} else if (name === "--line-count") {
|
||||
options.lineCount = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--max-append-attempts") {
|
||||
options.maxAppendAttempts = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--max-budget-usd") {
|
||||
options.maxBudgetUsd = requireValue(readValue(), name);
|
||||
} else if (name === "--max-input-tokens") {
|
||||
options.maxInputTokens = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--min-continuity-recall") {
|
||||
options.minContinuityRecall = readUnitNumber(readValue(), name);
|
||||
} else if (name === "--min-input-tokens") {
|
||||
options.minInputTokens = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--model") {
|
||||
options.model = requireValue(readValue(), name);
|
||||
} else if (name === "--session-id") {
|
||||
options.sessionId = requireValue(readValue(), name);
|
||||
} else if (name === "--task-case") {
|
||||
options.taskCase = requireValue(readValue(), name);
|
||||
} else if (name.startsWith("--task-case=")) {
|
||||
options.taskCase = requireValue(name.slice("--task-case=".length), "--task-case");
|
||||
} else if (name === "--target-input-tokens") {
|
||||
options.targetInputTokens = readPositiveInteger(readValue(), name);
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function readUnitNumber(value, name) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number < 0 || number > 1) {
|
||||
throw new Error(`${name} must be a number from 0 to 1.`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function requireValue(value, name) {
|
||||
if (!value) {
|
||||
throw new Error(`${name} requires a value.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function readPositiveInteger(value, name) {
|
||||
const number = Number(value);
|
||||
if (!Number.isInteger(number) || number <= 0) {
|
||||
throw new Error(`${name} must be a positive integer.`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack || error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,577 @@
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { createDefaultAppConfig } from "../packages/core/src/config/default-config";
|
||||
import type { AppConfig, GatewayProviderProtocol } from "../packages/core/src/contracts/app";
|
||||
import {
|
||||
contextArchiveService,
|
||||
prepareContextArchiveRequest
|
||||
} from "../packages/core/src/gateway/context-archive";
|
||||
import { selectTaskCases } from "./context-archive-task-cases.mjs";
|
||||
|
||||
type BenchmarkOptions = {
|
||||
cases: string;
|
||||
iterations: number;
|
||||
json: boolean;
|
||||
turns: number;
|
||||
};
|
||||
|
||||
type Fact = {
|
||||
detail: string;
|
||||
expected: string;
|
||||
index: (turns: number) => number;
|
||||
label: string;
|
||||
query: string;
|
||||
};
|
||||
|
||||
type Corpus = {
|
||||
caseId: string;
|
||||
facts: Fact[];
|
||||
messages: Array<{ content: string; role: "assistant" | "user" }>;
|
||||
};
|
||||
|
||||
type StrategyId =
|
||||
| "archive-only"
|
||||
| "auto-prune-handoff"
|
||||
| "claude-ccr-compact"
|
||||
| "claude-summary-adapter"
|
||||
| "codex-summary-adapter"
|
||||
| "false-positive-guard";
|
||||
|
||||
type Strategy = {
|
||||
description: string;
|
||||
id: StrategyId;
|
||||
};
|
||||
|
||||
type Scenario = {
|
||||
body: Record<string, unknown>;
|
||||
config: AppConfig;
|
||||
headers: Record<string, string>;
|
||||
path: string;
|
||||
protocol: GatewayProviderProtocol;
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
type RunMetric = {
|
||||
archiveRecall: number;
|
||||
bodyRecall: number;
|
||||
caseId: string;
|
||||
compressionRatio: number;
|
||||
diagnostic: string;
|
||||
falsePositive: boolean;
|
||||
forwardedBytes: number;
|
||||
historyAccessInjected: boolean;
|
||||
originalBytes: number;
|
||||
prepareMs: number;
|
||||
searchMs: number[];
|
||||
strategy: StrategyId;
|
||||
};
|
||||
|
||||
type SummaryMetric = {
|
||||
archiveRecall: number;
|
||||
bodyRecall: number;
|
||||
caseId?: string;
|
||||
compressionRatio: number;
|
||||
diagnosticModes: string[];
|
||||
falsePositiveRate: number;
|
||||
forwardedBytes: number;
|
||||
historyAccessRate: number;
|
||||
originalBytes: number;
|
||||
prepareP50Ms: number;
|
||||
prepareP95Ms: number;
|
||||
score: number;
|
||||
searchP50Ms: number;
|
||||
searchP95Ms: number;
|
||||
strategy: StrategyId;
|
||||
};
|
||||
|
||||
const strategies: Strategy[] = [
|
||||
{
|
||||
description: "Gateway-side pruning plus CCR handoff and archive search.",
|
||||
id: "auto-prune-handoff"
|
||||
},
|
||||
{
|
||||
description: "Codex client summary request; preserve full payload and inject archive access.",
|
||||
id: "codex-summary-adapter"
|
||||
},
|
||||
{
|
||||
description: "Claude Code ordinary summary request; preserve full payload and inject archive access.",
|
||||
id: "claude-summary-adapter"
|
||||
},
|
||||
{
|
||||
description: "Claude Code compact request with CCR replacement enabled; prune to CCR handoff plus recent context.",
|
||||
id: "claude-ccr-compact"
|
||||
},
|
||||
{
|
||||
description: "Archive the request only; no gateway compaction and no handoff injection.",
|
||||
id: "archive-only"
|
||||
},
|
||||
{
|
||||
description: "Claude Code request with unrelated 'compact' wording; should not trigger compaction.",
|
||||
id: "false-positive-guard"
|
||||
}
|
||||
];
|
||||
|
||||
export async function main(argv: string[] = process.argv.slice(2)): Promise<void> {
|
||||
const options = parseArgs(argv);
|
||||
const runs: RunMetric[] = [];
|
||||
const taskCases = selectTaskCases(options.cases);
|
||||
|
||||
for (let iteration = 0; iteration < options.iterations; iteration += 1) {
|
||||
for (const taskCase of taskCases) {
|
||||
const corpus = buildCorpus(taskCase, options.turns, iteration);
|
||||
for (const strategy of strategies) {
|
||||
runs.push(await runStrategy(strategy.id, corpus, iteration));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const summaries = strategies.map((strategy) => summarizeStrategy(strategy.id, runs));
|
||||
const caseSummaries = taskCases.map((taskCase) => summarizeStrategy("auto-prune-handoff", runs, taskCase.id));
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({
|
||||
caseSummaries,
|
||||
notes: benchmarkNotes(),
|
||||
options,
|
||||
selectedCases: taskCases.map((taskCase) => taskCase.id),
|
||||
strategies: summaries
|
||||
}, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
printSummary(options, summaries, caseSummaries);
|
||||
}
|
||||
|
||||
async function runStrategy(strategy: StrategyId, corpus: Corpus, iteration: number): Promise<RunMetric> {
|
||||
contextArchiveService.clear();
|
||||
const scenario = buildScenario(strategy, corpus, iteration);
|
||||
const original = Buffer.from(JSON.stringify(scenario.body), "utf8");
|
||||
const started = performance.now();
|
||||
const result = await prepareContextArchiveRequest({
|
||||
body: original,
|
||||
config: scenario.config,
|
||||
headers: scenario.headers,
|
||||
method: "POST",
|
||||
path: scenario.path,
|
||||
protocol: scenario.protocol,
|
||||
requestId: `${strategy}-${iteration}`
|
||||
});
|
||||
const prepareMs = performance.now() - started;
|
||||
const forwarded = result?.body ?? original;
|
||||
const forwardedText = forwarded.toString("utf8");
|
||||
const diagnostic = result?.diagnostic ?? "none";
|
||||
if (strategy === "false-positive-guard") {
|
||||
const passed = diagnostic.startsWith("archived:");
|
||||
return {
|
||||
archiveRecall: passed ? 1 : 0,
|
||||
bodyRecall: passed ? 1 : 0,
|
||||
caseId: corpus.caseId,
|
||||
compressionRatio: forwarded.byteLength / original.byteLength,
|
||||
diagnostic,
|
||||
falsePositive: !passed,
|
||||
forwardedBytes: forwarded.byteLength,
|
||||
historyAccessInjected: forwardedText.includes("Archived history access") || forwardedText.includes("CCR CONTEXT HANDOFF"),
|
||||
originalBytes: original.byteLength,
|
||||
prepareMs,
|
||||
searchMs: [],
|
||||
strategy
|
||||
};
|
||||
}
|
||||
const searchMs: number[] = [];
|
||||
let archiveHits = 0;
|
||||
|
||||
for (const fact of corpus.facts) {
|
||||
const searchStarted = performance.now();
|
||||
const search = await contextArchiveService.search({
|
||||
prompt: fact.query,
|
||||
sessionId: scenario.sessionId
|
||||
}, scenario.config.contextArchive);
|
||||
searchMs.push(performance.now() - searchStarted);
|
||||
if (searchOutputContains(search, fact.expected)) {
|
||||
archiveHits += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const bodyHits = corpus.facts.filter((fact) => forwardedText.includes(fact.expected)).length;
|
||||
return {
|
||||
archiveRecall: archiveHits / corpus.facts.length,
|
||||
bodyRecall: bodyHits / corpus.facts.length,
|
||||
caseId: corpus.caseId,
|
||||
compressionRatio: forwarded.byteLength / original.byteLength,
|
||||
diagnostic,
|
||||
falsePositive: strategy === "false-positive-guard" && !diagnostic.startsWith("archived:"),
|
||||
forwardedBytes: forwarded.byteLength,
|
||||
historyAccessInjected: forwardedText.includes("Archived history access") || forwardedText.includes("CCR CONTEXT HANDOFF"),
|
||||
originalBytes: original.byteLength,
|
||||
prepareMs,
|
||||
searchMs,
|
||||
strategy
|
||||
};
|
||||
}
|
||||
|
||||
function buildScenario(strategy: StrategyId, corpus: Corpus, iteration: number): Scenario {
|
||||
const sessionId = `${strategy}-${iteration}`;
|
||||
switch (strategy) {
|
||||
case "auto-prune-handoff":
|
||||
return {
|
||||
body: openAiChatBody(corpus),
|
||||
config: benchmarkConfig(1),
|
||||
headers: { "user-agent": "generic-openai-client/1.0", "x-session-id": sessionId },
|
||||
path: "/v1/chat/completions",
|
||||
protocol: "openai_chat_completions",
|
||||
sessionId
|
||||
};
|
||||
case "codex-summary-adapter":
|
||||
return {
|
||||
body: openAiResponsesBody(corpus, "Please summarize the conversation so far for context compaction. Include decisions, constraints, commands, and next steps."),
|
||||
config: benchmarkConfig(999999),
|
||||
headers: { "user-agent": "codex-cli/1.0", "x-codex-session-id": sessionId },
|
||||
path: "/v1/responses",
|
||||
protocol: "openai_responses",
|
||||
sessionId
|
||||
};
|
||||
case "claude-summary-adapter":
|
||||
return {
|
||||
body: anthropicMessagesBody(corpus, "Summarize the conversation so far for handoff into a new context window."),
|
||||
config: benchmarkConfig(999999),
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": sessionId },
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
sessionId
|
||||
};
|
||||
case "claude-ccr-compact":
|
||||
return {
|
||||
body: anthropicMessagesBody(corpus, "Summarize the conversation so far for handoff into a new context window."),
|
||||
config: benchmarkConfig(999999, { claudeCodeCompact: true }),
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": sessionId },
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
sessionId
|
||||
};
|
||||
case "archive-only":
|
||||
return {
|
||||
body: openAiChatBody(corpus),
|
||||
config: benchmarkConfig(999999),
|
||||
headers: { "user-agent": "generic-openai-client/1.0", "x-session-id": sessionId },
|
||||
path: "/v1/chat/completions",
|
||||
protocol: "openai_chat_completions",
|
||||
sessionId
|
||||
};
|
||||
case "false-positive-guard":
|
||||
return {
|
||||
body: {
|
||||
messages: [
|
||||
{ content: "We are editing a product preferences panel.", role: "assistant" },
|
||||
{ content: "Please set the UI density option to compact.", role: "user" }
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code."
|
||||
},
|
||||
config: benchmarkConfig(999999),
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": sessionId },
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
sessionId
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function benchmarkConfig(triggerTokenLimit: number, contextArchiveOverrides: Partial<AppConfig["contextArchive"]> = {}): AppConfig {
|
||||
const config = createDefaultAppConfig({
|
||||
generatedConfigFile: "/tmp/ccr-context-archive-benchmark-gateway.json"
|
||||
});
|
||||
return {
|
||||
...config,
|
||||
APIKEY: "benchmark-key",
|
||||
APIKEYS: [{ id: "benchmark", key: "benchmark-key", name: "Benchmark" }],
|
||||
contextArchive: {
|
||||
...config.contextArchive,
|
||||
enabled: true,
|
||||
handoffMaxCharacters: 16000,
|
||||
maxEntries: 20000,
|
||||
maxSearchResults: 8,
|
||||
retainRecentItems: 8,
|
||||
triggerTokenLimit,
|
||||
...contextArchiveOverrides
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildCorpus(taskCase: {
|
||||
facts: Array<{ detail: string; key: string; marker: string; placement: string; prompt: string }>;
|
||||
filler: string[];
|
||||
id: string;
|
||||
title: string;
|
||||
}, turns: number, iteration: number): Corpus {
|
||||
const safeTurns = Math.max(20, turns);
|
||||
const facts: Fact[] = taskCase.facts.map((fact) => ({
|
||||
detail: fact.detail,
|
||||
expected: iteration === 0 ? fact.marker : `${fact.marker}_ITER_${iteration}`,
|
||||
index: placementIndex(fact.placement, fact.key),
|
||||
label: fact.key,
|
||||
query: `What marker captures this task fact: ${fact.detail}`
|
||||
}));
|
||||
const factsByIndex = new Map(facts.map((fact) => [fact.index(safeTurns), fact]));
|
||||
const messages: Corpus["messages"] = [];
|
||||
for (let index = 0; index < safeTurns; index += 1) {
|
||||
const fact = factsByIndex.get(index);
|
||||
const role = index % 2 === 0 ? "user" : "assistant";
|
||||
messages.push({
|
||||
content: [
|
||||
`Turn ${index}: ${role} works on task case ${taskCase.id}: ${taskCase.title}.`,
|
||||
filler(taskCase, index),
|
||||
fact ? `FACT ${fact.label}: ${fact.expected}. ${fact.detail}` : undefined
|
||||
].filter(Boolean).join("\n"),
|
||||
role
|
||||
});
|
||||
}
|
||||
return { caseId: taskCase.id, facts, messages };
|
||||
}
|
||||
|
||||
function placementIndex(placement: string, key: string): (turns: number) => number {
|
||||
switch (placement) {
|
||||
case "early":
|
||||
return (count) => Math.max(2, Math.floor(count * 0.08));
|
||||
case "middle":
|
||||
return (count) => Math.max(3, Math.floor(count * 0.50));
|
||||
case "recent":
|
||||
return (count) => Math.max(4, count - recentOffset(key));
|
||||
default:
|
||||
return (count) => Math.max(1, count - 5);
|
||||
}
|
||||
}
|
||||
|
||||
function recentOffset(key: string): number {
|
||||
if (key === "objective") return 8;
|
||||
if (key === "completed") return 6;
|
||||
if (key === "currentFocus") return 5;
|
||||
if (key === "nextStep") return 4;
|
||||
if (key === "validationCommand") return 3;
|
||||
if (key === "risk") return 2;
|
||||
return 7;
|
||||
}
|
||||
|
||||
function filler(taskCase: { filler: string[] }, index: number): string {
|
||||
const fragment = taskCase.filler[index % taskCase.filler.length] || "The task context contains implementation details and verification notes.";
|
||||
return `${fragment} Repeated context marker ${index.toString().padStart(3, "0")}.`;
|
||||
}
|
||||
|
||||
function openAiChatBody(corpus: Corpus): Record<string, unknown> {
|
||||
return {
|
||||
messages: [
|
||||
{ content: "You are a coding agent.", role: "system" },
|
||||
...corpus.messages
|
||||
],
|
||||
model: "benchmark-model"
|
||||
};
|
||||
}
|
||||
|
||||
function openAiResponsesBody(corpus: Corpus, finalPrompt: string): Record<string, unknown> {
|
||||
return {
|
||||
input: [
|
||||
...corpus.messages.map((message) => ({
|
||||
content: [{ text: message.content, type: "input_text" }],
|
||||
role: message.role,
|
||||
type: "message"
|
||||
})),
|
||||
{
|
||||
content: [{ text: finalPrompt, type: "input_text" }],
|
||||
role: "user",
|
||||
type: "message"
|
||||
}
|
||||
],
|
||||
instructions: "You are Codex.",
|
||||
model: "gpt-5-codex"
|
||||
};
|
||||
}
|
||||
|
||||
function anthropicMessagesBody(corpus: Corpus, finalPrompt: string): Record<string, unknown> {
|
||||
return {
|
||||
messages: [
|
||||
...corpus.messages,
|
||||
{ content: finalPrompt, role: "user" }
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code."
|
||||
};
|
||||
}
|
||||
|
||||
function searchOutputContains(value: unknown, expected: string): boolean {
|
||||
return JSON.stringify(value).includes(expected);
|
||||
}
|
||||
|
||||
function summarizeStrategy(strategy: StrategyId, runs: RunMetric[], caseId?: string): SummaryMetric {
|
||||
const selected = runs.filter((run) => run.strategy === strategy && (!caseId || run.caseId === caseId));
|
||||
const summary = {
|
||||
archiveRecall: average(selected.map((run) => run.archiveRecall)),
|
||||
bodyRecall: average(selected.map((run) => run.bodyRecall)),
|
||||
compressionRatio: average(selected.map((run) => run.compressionRatio)),
|
||||
diagnosticModes: unique(selected.map((run) => run.diagnostic.split(":")[0] || "none")),
|
||||
falsePositiveRate: average(selected.map((run) => run.falsePositive ? 1 : 0)),
|
||||
forwardedBytes: average(selected.map((run) => run.forwardedBytes)),
|
||||
historyAccessRate: average(selected.map((run) => run.historyAccessInjected ? 1 : 0)),
|
||||
originalBytes: average(selected.map((run) => run.originalBytes)),
|
||||
prepareP50Ms: percentile(selected.map((run) => run.prepareMs), 50),
|
||||
prepareP95Ms: percentile(selected.map((run) => run.prepareMs), 95),
|
||||
searchP50Ms: percentile(selected.flatMap((run) => run.searchMs), 50),
|
||||
searchP95Ms: percentile(selected.flatMap((run) => run.searchMs), 95),
|
||||
strategy
|
||||
};
|
||||
return {
|
||||
...summary,
|
||||
...(caseId ? { caseId } : {}),
|
||||
score: scoreSummary(summary)
|
||||
};
|
||||
}
|
||||
|
||||
function scoreSummary(summary: Omit<SummaryMetric, "score">): number {
|
||||
const quality = clamp01(summary.archiveRecall);
|
||||
const efficiency = 1 - Math.min(1, Math.max(0, summary.compressionRatio));
|
||||
const safety = 1 - clamp01(summary.falsePositiveRate);
|
||||
return quality * 0.5 + efficiency * 0.3 + safety * 0.2;
|
||||
}
|
||||
|
||||
function printSummary(options: BenchmarkOptions, summaries: SummaryMetric[], caseSummaries: SummaryMetric[]): void {
|
||||
console.log(`Context archive benchmark: cases=${options.cases} iterations=${options.iterations} turns=${options.turns}`);
|
||||
console.log(benchmarkNotes());
|
||||
console.log("");
|
||||
console.log("Strategy summary:");
|
||||
const headers = [
|
||||
"strategy",
|
||||
"diag",
|
||||
"ratio",
|
||||
"body_recall",
|
||||
"archive_recall",
|
||||
"handoff",
|
||||
"false_pos",
|
||||
"prep_p50",
|
||||
"search_p95",
|
||||
"score"
|
||||
];
|
||||
const rows = summaries.map((summary) => [
|
||||
summary.strategy,
|
||||
summary.diagnosticModes.join(","),
|
||||
formatNumber(summary.compressionRatio),
|
||||
formatPercent(summary.bodyRecall),
|
||||
formatPercent(summary.archiveRecall),
|
||||
formatPercent(summary.historyAccessRate),
|
||||
formatPercent(summary.falsePositiveRate),
|
||||
`${formatNumber(summary.prepareP50Ms)}ms`,
|
||||
`${formatNumber(summary.searchP95Ms)}ms`,
|
||||
formatNumber(summary.score)
|
||||
]);
|
||||
printTable(headers, rows);
|
||||
console.log("");
|
||||
console.log("Auto-prune-handoff by task case:");
|
||||
printTable([
|
||||
"case",
|
||||
"ratio",
|
||||
"body_recall",
|
||||
"archive_recall",
|
||||
"search_p95",
|
||||
"score"
|
||||
], caseSummaries.map((summary) => [
|
||||
summary.caseId ?? "all",
|
||||
formatNumber(summary.compressionRatio),
|
||||
formatPercent(summary.bodyRecall),
|
||||
formatPercent(summary.archiveRecall),
|
||||
`${formatNumber(summary.searchP95Ms)}ms`,
|
||||
formatNumber(summary.score)
|
||||
]));
|
||||
}
|
||||
|
||||
function benchmarkNotes(): string {
|
||||
return [
|
||||
"Notes:",
|
||||
"ratio=forwarded request bytes/original request bytes; lower is better for gateway-side compression.",
|
||||
"body_recall=continuity markers still visible in the forwarded compact body without retrieval.",
|
||||
"archive_recall=continuity markers recovered through ccr_history_search.",
|
||||
"score=0.5*archive_recall + 0.3*(1-min(ratio,1)) + 0.2*(1-false_positive_rate).",
|
||||
"This benchmark does not judge external LLM summary quality."
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function printTable(headers: string[], rows: string[][]): void {
|
||||
const widths = headers.map((header, index) =>
|
||||
Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0))
|
||||
);
|
||||
console.log(headers.map((header, index) => header.padEnd(widths[index])).join(" "));
|
||||
console.log(widths.map((width) => "-".repeat(width)).join(" "));
|
||||
for (const row of rows) {
|
||||
console.log(row.map((cell, index) => cell.padEnd(widths[index])).join(" "));
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): BenchmarkOptions {
|
||||
const options: BenchmarkOptions = {
|
||||
iterations: 5,
|
||||
json: false,
|
||||
cases: "all",
|
||||
turns: 120
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--json") {
|
||||
options.json = true;
|
||||
} else if (arg === "--case" || arg === "--cases") {
|
||||
options.cases = readString(argv[++index], arg);
|
||||
} else if (arg.startsWith("--case=")) {
|
||||
options.cases = readString(arg.slice("--case=".length), "--case");
|
||||
} else if (arg.startsWith("--cases=")) {
|
||||
options.cases = readString(arg.slice("--cases=".length), "--cases");
|
||||
} else if (arg === "--iterations") {
|
||||
options.iterations = readPositiveInteger(argv[++index], "--iterations");
|
||||
} else if (arg.startsWith("--iterations=")) {
|
||||
options.iterations = readPositiveInteger(arg.slice("--iterations=".length), "--iterations");
|
||||
} else if (arg === "--turns") {
|
||||
options.turns = readPositiveInteger(argv[++index], "--turns");
|
||||
} else if (arg.startsWith("--turns=")) {
|
||||
options.turns = readPositiveInteger(arg.slice("--turns=".length), "--turns");
|
||||
} else {
|
||||
throw new Error(`Unknown benchmark argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function readString(value: string | undefined, name: string): string {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`${name} requires a value.`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function readPositiveInteger(value: string | undefined, name: string): number {
|
||||
const number = Number(value);
|
||||
if (!Number.isInteger(number) || number <= 0) {
|
||||
throw new Error(`${name} must be a positive integer.`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function average(values: number[]): number {
|
||||
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
|
||||
}
|
||||
|
||||
function percentile(values: number[], percentileValue: number): number {
|
||||
if (values.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1));
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
function unique(values: string[]): string[] {
|
||||
return [...new Set(values)].sort();
|
||||
}
|
||||
|
||||
function clamp01(value: number): number {
|
||||
return Math.max(0, Math.min(1, value));
|
||||
}
|
||||
|
||||
function formatNumber(value: number): string {
|
||||
return value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 3);
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return `${(value * 100).toFixed(0)}%`;
|
||||
}
|
||||
@@ -0,0 +1,665 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { createDefaultAppConfig } from "../packages/core/src/config/default-config";
|
||||
import type { AppConfig } from "../packages/core/src/contracts/app";
|
||||
import {
|
||||
contextArchiveService,
|
||||
prepareContextArchiveRequest
|
||||
} from "../packages/core/src/gateway/context-archive";
|
||||
import { defaultTaskCaseId, findTaskCase, selectTaskCases, taskCaseIds } from "./context-archive-task-cases.mjs";
|
||||
|
||||
type BenchmarkOptions = {
|
||||
caseSelector: string;
|
||||
claudeBin: string;
|
||||
json: boolean;
|
||||
listTaskCases: boolean;
|
||||
maxBudgetUsd: string;
|
||||
maxEstimatedTokens: number;
|
||||
minEstimatedTokens: number;
|
||||
targetEstimatedTokens: number;
|
||||
turns?: number;
|
||||
};
|
||||
|
||||
type TaskCase = {
|
||||
facts: Array<{ detail: string; key: string; marker: string; placement: string; prompt: string }>;
|
||||
filler: string[];
|
||||
id: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
type CorpusFact = {
|
||||
detail: string;
|
||||
expected: string;
|
||||
key: string;
|
||||
query: string;
|
||||
};
|
||||
|
||||
type Corpus = {
|
||||
facts: CorpusFact[];
|
||||
messages: Array<{ content: string; role: "assistant" | "user" }>;
|
||||
taskCase: TaskCase;
|
||||
turns: number;
|
||||
};
|
||||
|
||||
type ClaudeRun = {
|
||||
apiMs: number;
|
||||
costUsd: number;
|
||||
elapsedMs: number;
|
||||
inputTokens: number;
|
||||
isError: boolean;
|
||||
output: string;
|
||||
outputTokens: number;
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
type ToolRequest = {
|
||||
deep?: boolean;
|
||||
max_chunks?: number;
|
||||
prompt?: string;
|
||||
session_id?: string;
|
||||
};
|
||||
|
||||
type CaseReport = {
|
||||
bodyRecall: number;
|
||||
caseId: string;
|
||||
compactedBytes: number;
|
||||
diagnostic: string;
|
||||
estimatedTokens: number;
|
||||
finalOutput: string;
|
||||
found: number;
|
||||
initialAgent: ClaudeRun;
|
||||
misses: string[];
|
||||
nearTarget: boolean;
|
||||
originalBytes: number;
|
||||
ratio: number;
|
||||
sessionId: string;
|
||||
synthesisAgent: ClaudeRun;
|
||||
title: string;
|
||||
toolCalls: number;
|
||||
total: number;
|
||||
totalCostUsd: number;
|
||||
turns: number;
|
||||
};
|
||||
|
||||
export async function main(argv: string[] = process.argv.slice(2)): Promise<void> {
|
||||
const options = parseArgs(argv);
|
||||
if (options.listTaskCases) {
|
||||
console.log(taskCaseIds().join("\n"));
|
||||
return;
|
||||
}
|
||||
|
||||
const taskCases = options.caseSelector === "all"
|
||||
? selectTaskCases("all")
|
||||
: [findTaskCase(options.caseSelector)];
|
||||
const reports: CaseReport[] = [];
|
||||
for (const taskCase of taskCases) {
|
||||
reports.push(await runCase(taskCase, options));
|
||||
}
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify({ reports, summary: summarizeReports(reports) }, null, 2));
|
||||
return;
|
||||
}
|
||||
printReports(reports);
|
||||
}
|
||||
|
||||
async function runCase(taskCase: TaskCase, options: BenchmarkOptions): Promise<CaseReport> {
|
||||
contextArchiveService.clear();
|
||||
const corpus = buildTargetCorpus(taskCase, options);
|
||||
const sessionId = `real-ccr-${taskCase.id}-${randomUUID()}`;
|
||||
const config = benchmarkConfig();
|
||||
const body = anthropicMessagesBody(corpus);
|
||||
const original = Buffer.from(JSON.stringify(body), "utf8");
|
||||
const prepared = await prepareContextArchiveRequest({
|
||||
body: original,
|
||||
config,
|
||||
headers: { "user-agent": "generic-agent/1.0", "x-session-id": sessionId },
|
||||
method: "POST",
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
requestId: `real-agent-${taskCase.id}`
|
||||
});
|
||||
if (!prepared) {
|
||||
throw new Error(`CCR did not prepare context archive request for ${taskCase.id}.`);
|
||||
}
|
||||
|
||||
const compactedBody = JSON.parse(prepared.body.toString("utf8")) as Record<string, unknown>;
|
||||
const compactedText = renderCompactedBody(compactedBody);
|
||||
const initialAgent = await runClaude({
|
||||
claudeBin: options.claudeBin,
|
||||
input: initialAgentPrompt(corpus, compactedText, sessionId),
|
||||
maxBudgetUsd: options.maxBudgetUsd,
|
||||
sessionId: randomUUID()
|
||||
});
|
||||
const toolRequests = parseToolRequests(initialAgent.output, corpus, sessionId);
|
||||
const toolResults = [];
|
||||
for (const request of toolRequests) {
|
||||
const prompt = request.prompt?.trim();
|
||||
if (!prompt) {
|
||||
continue;
|
||||
}
|
||||
const result = await contextArchiveService.search({
|
||||
deep: request.deep !== false,
|
||||
maxChunks: request.max_chunks,
|
||||
prompt,
|
||||
sessionId: request.session_id || sessionId
|
||||
}, config.contextArchive);
|
||||
toolResults.push({ request, result });
|
||||
}
|
||||
|
||||
const synthesisAgent = await runClaude({
|
||||
claudeBin: options.claudeBin,
|
||||
input: synthesisPrompt(corpus, compactedText, sessionId, toolResults),
|
||||
maxBudgetUsd: options.maxBudgetUsd,
|
||||
resume: initialAgent.sessionId
|
||||
});
|
||||
const finalOutput = parseFinalText(synthesisAgent.output);
|
||||
const foundFacts = corpus.facts.filter((fact) => finalOutput.includes(fact.expected));
|
||||
const bodyFacts = corpus.facts.filter((fact) => compactedText.includes(fact.expected));
|
||||
const misses = corpus.facts.filter((fact) => !finalOutput.includes(fact.expected)).map((fact) => fact.key);
|
||||
const estimatedTokens = estimateBodyTokens(body);
|
||||
|
||||
return {
|
||||
bodyRecall: bodyFacts.length / corpus.facts.length,
|
||||
caseId: taskCase.id,
|
||||
compactedBytes: prepared.body.byteLength,
|
||||
diagnostic: prepared.diagnostic,
|
||||
estimatedTokens,
|
||||
finalOutput,
|
||||
found: foundFacts.length,
|
||||
initialAgent,
|
||||
misses,
|
||||
nearTarget: estimatedTokens >= options.minEstimatedTokens && estimatedTokens <= options.maxEstimatedTokens,
|
||||
originalBytes: original.byteLength,
|
||||
ratio: prepared.body.byteLength / original.byteLength,
|
||||
sessionId,
|
||||
synthesisAgent,
|
||||
title: taskCase.title,
|
||||
toolCalls: toolResults.length,
|
||||
total: corpus.facts.length,
|
||||
totalCostUsd: initialAgent.costUsd + synthesisAgent.costUsd,
|
||||
turns: corpus.turns
|
||||
};
|
||||
}
|
||||
|
||||
function benchmarkConfig(): AppConfig {
|
||||
const config = createDefaultAppConfig({
|
||||
generatedConfigFile: "/tmp/ccr-context-archive-real-agent-benchmark.json"
|
||||
});
|
||||
return {
|
||||
...config,
|
||||
APIKEY: "benchmark-key",
|
||||
APIKEYS: [{ id: "benchmark", key: "benchmark-key", name: "Benchmark" }],
|
||||
contextArchive: {
|
||||
...config.contextArchive,
|
||||
enabled: true,
|
||||
handoffMaxCharacters: 16000,
|
||||
maxEntries: 50000,
|
||||
maxSearchResults: 8,
|
||||
retainRecentItems: 8,
|
||||
triggerTokenLimit: 1
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildTargetCorpus(taskCase: TaskCase, options: BenchmarkOptions): Corpus {
|
||||
if (options.turns) {
|
||||
return buildCorpus(taskCase, options.turns);
|
||||
}
|
||||
let turns = 1200;
|
||||
let corpus = buildCorpus(taskCase, turns);
|
||||
for (let attempt = 0; attempt < 8; attempt += 1) {
|
||||
const estimate = estimateBodyTokens(anthropicMessagesBody(corpus));
|
||||
if (estimate >= options.minEstimatedTokens && estimate <= options.maxEstimatedTokens) {
|
||||
return corpus;
|
||||
}
|
||||
const nextTurns = Math.max(20, Math.round(turns * (options.targetEstimatedTokens / Math.max(1, estimate))));
|
||||
if (Math.abs(nextTurns - turns) <= 2) {
|
||||
return corpus;
|
||||
}
|
||||
turns = nextTurns;
|
||||
corpus = buildCorpus(taskCase, turns);
|
||||
}
|
||||
return corpus;
|
||||
}
|
||||
|
||||
function buildCorpus(taskCase: TaskCase, turns: number): Corpus {
|
||||
const safeTurns = Math.max(20, turns);
|
||||
const facts = taskCase.facts.map((fact) => ({
|
||||
detail: fact.detail,
|
||||
expected: fact.marker,
|
||||
key: fact.key,
|
||||
query: `What exact marker records this task fact: ${fact.detail}`
|
||||
}));
|
||||
const factsByIndex = new Map(taskCase.facts.map((fact) => [
|
||||
placementIndex(fact.placement, fact.key)(safeTurns),
|
||||
fact
|
||||
]));
|
||||
const messages: Corpus["messages"] = [];
|
||||
for (let index = 0; index < safeTurns; index += 1) {
|
||||
const fact = factsByIndex.get(index);
|
||||
const role = index % 2 === 0 ? "user" : "assistant";
|
||||
messages.push({
|
||||
content: [
|
||||
`Turn ${index}: ${role} works on realistic task case ${taskCase.id}: ${taskCase.title}.`,
|
||||
filler(taskCase, index),
|
||||
"The conversation includes file paths, command output, partial implementation notes, and review constraints.",
|
||||
fact ? `FACT ${fact.key}: ${fact.marker}. ${fact.detail}` : undefined
|
||||
].filter(Boolean).join("\n"),
|
||||
role
|
||||
});
|
||||
}
|
||||
return { facts, messages, taskCase, turns: safeTurns };
|
||||
}
|
||||
|
||||
function placementIndex(placement: string, key: string): (turns: number) => number {
|
||||
switch (placement) {
|
||||
case "early":
|
||||
return (count) => Math.max(2, Math.floor(count * 0.08));
|
||||
case "middle":
|
||||
return (count) => Math.max(3, Math.floor(count * 0.50));
|
||||
case "recent":
|
||||
return (count) => Math.max(4, count - recentOffset(key));
|
||||
default:
|
||||
return (count) => Math.max(1, count - 5);
|
||||
}
|
||||
}
|
||||
|
||||
function recentOffset(key: string): number {
|
||||
if (key === "objective") return 8;
|
||||
if (key === "completed") return 6;
|
||||
if (key === "currentFocus") return 5;
|
||||
if (key === "nextStep") return 4;
|
||||
if (key === "validationCommand") return 3;
|
||||
if (key === "risk") return 2;
|
||||
return 7;
|
||||
}
|
||||
|
||||
function filler(taskCase: TaskCase, index: number): string {
|
||||
const fragment = taskCase.filler[index % taskCase.filler.length] || "The task context contains implementation details and verification notes.";
|
||||
const padded = String(index).padStart(5, "0");
|
||||
return [
|
||||
fragment,
|
||||
`Repeated realistic worklog marker ${padded}.`,
|
||||
`The agent records constraints, tests, and pending decisions for continuation quality ${padded}.`
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function anthropicMessagesBody(corpus: Corpus): Record<string, unknown> {
|
||||
return {
|
||||
messages: [
|
||||
...corpus.messages,
|
||||
{
|
||||
content: [
|
||||
"Continue the coding task after context management.",
|
||||
"Preserve exact decisions, completed work, current focus, next step, validation command, and known risk.",
|
||||
"Do not summarize unless asked."
|
||||
].join(" "),
|
||||
role: "user"
|
||||
}
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code working inside a repository."
|
||||
};
|
||||
}
|
||||
|
||||
function renderCompactedBody(body: Record<string, unknown>): string {
|
||||
const system = contentText(body.system);
|
||||
const messages = Array.isArray(body.messages) ? body.messages : [];
|
||||
return [
|
||||
"SYSTEM:",
|
||||
system,
|
||||
"",
|
||||
"MESSAGES:",
|
||||
...messages.map((message, index) => {
|
||||
const record = isRecord(message) ? message : {};
|
||||
return [
|
||||
`--- message ${index + 1} role=${String(record.role ?? "unknown")} ---`,
|
||||
contentText(record.content)
|
||||
].join("\n");
|
||||
})
|
||||
].join("\n").slice(0, 120000);
|
||||
}
|
||||
|
||||
function initialAgentPrompt(corpus: Corpus, compactedText: string, sessionId: string): string {
|
||||
return [
|
||||
"You are a real post-compaction coding agent evaluating CCR context continuity.",
|
||||
"You received only the CCR-compressed context below. Exact older details may require ccr_history_search.",
|
||||
"Do not guess marker strings. Request retrieval for every marker that is not directly visible.",
|
||||
"Return ONLY JSON in this shape:",
|
||||
'{"tool_calls":[{"prompt":"specific retrieval question","deep":true,"session_id":"archive session id"}]}',
|
||||
"",
|
||||
`Archive session id: ${sessionId}`,
|
||||
"Continuity facts to recover:",
|
||||
...corpus.facts.map((fact) => `- ${fact.key}: ${fact.detail}`),
|
||||
"",
|
||||
"CCR_COMPRESSED_CONTEXT_BEGIN",
|
||||
compactedText,
|
||||
"CCR_COMPRESSED_CONTEXT_END"
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function synthesisPrompt(
|
||||
corpus: Corpus,
|
||||
compactedText: string,
|
||||
sessionId: string,
|
||||
toolResults: Array<{ request: ToolRequest; result: unknown }>
|
||||
): string {
|
||||
return [
|
||||
"Use the CCR-compressed context and ccr_history_search evidence to answer the continuity probe.",
|
||||
"Return ONLY JSON in this exact shape:",
|
||||
`{"final":{${corpus.facts.map((fact) => `"${fact.key}":"exact marker or UNKNOWN"`).join(",")}}}`,
|
||||
"Do not guess. If evidence is insufficient, use UNKNOWN.",
|
||||
"",
|
||||
`Archive session id: ${sessionId}`,
|
||||
"",
|
||||
"CCR_COMPRESSED_CONTEXT_BEGIN",
|
||||
compactedText,
|
||||
"CCR_COMPRESSED_CONTEXT_END",
|
||||
"",
|
||||
"CCR_HISTORY_SEARCH_RESULTS_BEGIN",
|
||||
JSON.stringify(toolResults, null, 2),
|
||||
"CCR_HISTORY_SEARCH_RESULTS_END"
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parseToolRequests(output: string, corpus: Corpus, sessionId: string): ToolRequest[] {
|
||||
const parsed = parseJsonObject(output);
|
||||
const rawCalls = Array.isArray(parsed?.tool_calls) ? parsed.tool_calls : undefined;
|
||||
if (rawCalls?.length) {
|
||||
return rawCalls
|
||||
.filter(isRecord)
|
||||
.map((call) => ({
|
||||
deep: call.deep === false ? false : true,
|
||||
max_chunks: typeof call.max_chunks === "number" ? call.max_chunks : undefined,
|
||||
prompt: typeof call.prompt === "string" ? call.prompt : undefined,
|
||||
session_id: typeof call.session_id === "string" ? call.session_id : sessionId
|
||||
}))
|
||||
.filter((call) => Boolean(call.prompt));
|
||||
}
|
||||
return corpus.facts.map((fact) => ({
|
||||
deep: true,
|
||||
prompt: fact.query,
|
||||
session_id: sessionId
|
||||
}));
|
||||
}
|
||||
|
||||
function parseFinalText(output: string): string {
|
||||
const parsed = parseJsonObject(output);
|
||||
if (parsed?.final && typeof parsed.final === "object") {
|
||||
return JSON.stringify(parsed.final);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
async function runClaude(input: {
|
||||
claudeBin: string;
|
||||
input: string;
|
||||
maxBudgetUsd: string;
|
||||
resume?: string;
|
||||
sessionId?: string;
|
||||
}): Promise<ClaudeRun> {
|
||||
const started = performance.now();
|
||||
const args = [
|
||||
"-p",
|
||||
"--output-format",
|
||||
"json",
|
||||
"--tools",
|
||||
"",
|
||||
"--max-budget-usd",
|
||||
input.maxBudgetUsd
|
||||
];
|
||||
if (input.resume) {
|
||||
args.push("--resume", input.resume);
|
||||
} else {
|
||||
args.push("--session-id", input.sessionId || randomUUID());
|
||||
}
|
||||
|
||||
const child = spawn(input.claudeBin, args, {
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdio: ["pipe", "pipe", "pipe"]
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let stdinError = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.stdin.on("error", (error) => {
|
||||
stdinError = error instanceof Error ? error.message : String(error);
|
||||
});
|
||||
child.stdin.end(input.input);
|
||||
|
||||
const code = await new Promise<number>((resolve, reject) => {
|
||||
child.on("error", reject);
|
||||
child.on("exit", (exitCode) => resolve(exitCode ?? 1));
|
||||
});
|
||||
const elapsedMs = performance.now() - started;
|
||||
const result = parseClaudeJson(stdout);
|
||||
if (code !== 0 && !result) {
|
||||
throw new Error([
|
||||
`claude exited with code ${code}.`,
|
||||
stdinError ? `stdin error: ${stdinError}` : undefined,
|
||||
stderr.trim(),
|
||||
stdout.trim()
|
||||
].filter(Boolean).join("\n"));
|
||||
}
|
||||
return {
|
||||
apiMs: Number(result?.duration_api_ms ?? 0),
|
||||
costUsd: Number(result?.total_cost_usd ?? 0),
|
||||
elapsedMs: Math.round(elapsedMs),
|
||||
inputTokens: totalInputTokens(result),
|
||||
isError: Boolean(result?.is_error),
|
||||
output: typeof result?.result === "string" ? result.result : stdout,
|
||||
outputTokens: totalOutputTokens(result),
|
||||
sessionId: String(result?.session_id || input.resume || input.sessionId || "")
|
||||
};
|
||||
}
|
||||
|
||||
function parseClaudeJson(stdout: string): Record<string, unknown> | undefined {
|
||||
const trimmed = stdout.trim();
|
||||
if (!trimmed) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(trimmed) as Record<string, unknown>;
|
||||
} catch {
|
||||
return parseJsonObject(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string): Record<string, unknown> | undefined {
|
||||
const fence = /```(?:json)?\s*([\s\S]*?)```/i.exec(text);
|
||||
const candidate = fence?.[1] ?? text;
|
||||
const firstBrace = candidate.indexOf("{");
|
||||
const lastBrace = candidate.lastIndexOf("}");
|
||||
if (firstBrace < 0 || lastBrace <= firstBrace) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(candidate.slice(firstBrace, lastBrace + 1)) as Record<string, unknown>;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function totalInputTokens(result: Record<string, unknown> | undefined): number {
|
||||
const usage = isRecord(result?.usage) ? result.usage : {};
|
||||
const topLevel =
|
||||
Number(usage.input_tokens ?? 0) +
|
||||
Number(usage.cache_creation_input_tokens ?? 0) +
|
||||
Number(usage.cache_read_input_tokens ?? 0);
|
||||
return topLevel;
|
||||
}
|
||||
|
||||
function totalOutputTokens(result: Record<string, unknown> | undefined): number {
|
||||
const usage = isRecord(result?.usage) ? result.usage : {};
|
||||
return Number(usage.output_tokens ?? 0);
|
||||
}
|
||||
|
||||
function contentText(value: unknown): string {
|
||||
if (value === undefined || value === null) {
|
||||
return "";
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(contentText).filter(Boolean).join("\n");
|
||||
}
|
||||
if (!isRecord(value)) {
|
||||
return "";
|
||||
}
|
||||
const direct = stringValue(value.text) || stringValue(value.input_text) || stringValue(value.output_text);
|
||||
if (direct) {
|
||||
return direct;
|
||||
}
|
||||
if (value.content !== undefined) {
|
||||
return contentText(value.content);
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function estimateBodyTokens(body: Record<string, unknown>): number {
|
||||
return Math.ceil(JSON.stringify(body).length / 4);
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function stringValue(value: unknown): string | undefined {
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function summarizeReports(reports: CaseReport[]): Record<string, unknown> {
|
||||
const found = reports.reduce((sum, report) => sum + report.found, 0);
|
||||
const total = reports.reduce((sum, report) => sum + report.total, 0);
|
||||
return {
|
||||
avgRatio: average(reports.map((report) => report.ratio)),
|
||||
bodyRecall: average(reports.map((report) => report.bodyRecall)),
|
||||
costUsd: reports.reduce((sum, report) => sum + report.totalCostUsd, 0),
|
||||
found,
|
||||
nearTargetCases: reports.filter((report) => report.nearTarget).length,
|
||||
recall: total ? found / total : 0,
|
||||
total
|
||||
};
|
||||
}
|
||||
|
||||
function printReports(reports: CaseReport[]): void {
|
||||
const summary = summarizeReports(reports);
|
||||
console.log("CCR real post-compaction agent benchmark");
|
||||
console.log(`summary: recall=${summary.found}/${summary.total} (${formatPercent(Number(summary.recall))}) avg_ratio=${formatNumber(Number(summary.avgRatio))} body_recall=${formatPercent(Number(summary.bodyRecall))} cost=$${formatNumber(Number(summary.costUsd))} near_target=${summary.nearTargetCases}/${reports.length}`);
|
||||
console.log("");
|
||||
printTable([
|
||||
"case",
|
||||
"est_tokens",
|
||||
"turns",
|
||||
"near",
|
||||
"ratio",
|
||||
"body",
|
||||
"tool_calls",
|
||||
"recall",
|
||||
"misses",
|
||||
"cost"
|
||||
], reports.map((report) => [
|
||||
report.caseId,
|
||||
String(report.estimatedTokens),
|
||||
String(report.turns),
|
||||
report.nearTarget ? "yes" : "no",
|
||||
formatNumber(report.ratio),
|
||||
formatPercent(report.bodyRecall),
|
||||
String(report.toolCalls),
|
||||
`${report.found}/${report.total}`,
|
||||
report.misses.join(",") || "-",
|
||||
`$${formatNumber(report.totalCostUsd)}`
|
||||
]));
|
||||
}
|
||||
|
||||
function printTable(headers: string[], rows: string[][]): void {
|
||||
const widths = headers.map((header, index) =>
|
||||
Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0))
|
||||
);
|
||||
console.log(headers.map((header, index) => header.padEnd(widths[index])).join(" "));
|
||||
console.log(widths.map((width) => "-".repeat(width)).join(" "));
|
||||
for (const row of rows) {
|
||||
console.log(row.map((cell, index) => cell.padEnd(widths[index])).join(" "));
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): BenchmarkOptions {
|
||||
const options: BenchmarkOptions = {
|
||||
caseSelector: "all",
|
||||
claudeBin: process.env.CLAUDE_BIN || "claude",
|
||||
json: false,
|
||||
listTaskCases: false,
|
||||
maxBudgetUsd: "20",
|
||||
maxEstimatedTokens: 190000,
|
||||
minEstimatedTokens: 170000,
|
||||
targetEstimatedTokens: 180000
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const [name, inlineValue] = arg.includes("=") ? arg.split(/=(.*)/s, 2) : [arg, undefined];
|
||||
const readValue = () => inlineValue ?? argv[++index];
|
||||
if (name === "--case" || name === "--cases") {
|
||||
options.caseSelector = readString(readValue(), name);
|
||||
} else if (name === "--claude-bin") {
|
||||
options.claudeBin = readString(readValue(), name);
|
||||
} else if (name === "--json") {
|
||||
options.json = true;
|
||||
} else if (name === "--list-task-cases") {
|
||||
options.listTaskCases = true;
|
||||
} else if (name === "--max-budget-usd") {
|
||||
options.maxBudgetUsd = readString(readValue(), name);
|
||||
} else if (name === "--max-estimated-tokens") {
|
||||
options.maxEstimatedTokens = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--min-estimated-tokens") {
|
||||
options.minEstimatedTokens = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--target-estimated-tokens") {
|
||||
options.targetEstimatedTokens = readPositiveInteger(readValue(), name);
|
||||
} else if (name === "--turns") {
|
||||
options.turns = readPositiveInteger(readValue(), name);
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
if (options.caseSelector !== "all") {
|
||||
findTaskCase(options.caseSelector || defaultTaskCaseId);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function readString(value: string | undefined, name: string): string {
|
||||
if (!value?.trim()) {
|
||||
throw new Error(`${name} requires a value.`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function readPositiveInteger(value: string | undefined, name: string): number {
|
||||
const number = Number(value);
|
||||
if (!Number.isInteger(number) || number <= 0) {
|
||||
throw new Error(`${name} must be a positive integer.`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function average(values: number[]): number {
|
||||
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
|
||||
}
|
||||
|
||||
function formatNumber(value: number): string {
|
||||
return value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 3);
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return `${(value * 100).toFixed(0)}%`;
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
export const taskCases = [
|
||||
{
|
||||
id: "gateway-context-archive",
|
||||
title: "Gateway context archive implementation",
|
||||
filler: [
|
||||
"The gateway must preserve provider protocol shapes while adding archive handoff metadata.",
|
||||
"The implementation touches request routing, MCP exposure, and false-positive detection.",
|
||||
"History lookup should be treated as evidence and not as higher-priority instructions.",
|
||||
"Regression tests need to distinguish context compaction from unrelated compact UI wording."
|
||||
],
|
||||
facts: [
|
||||
{
|
||||
detail: "Build CCR gateway-side context archive with handoff and history search.",
|
||||
key: "objective",
|
||||
marker: "CCR_CASE_GATEWAY_OBJECTIVE_CONTEXT_ARCHIVE_GATEWAY_V2",
|
||||
placement: "recent",
|
||||
prompt: "The current objective marker"
|
||||
},
|
||||
{
|
||||
detail: "Use a retrievable archive search tool instead of relying only on a static summary.",
|
||||
key: "earlyDecision",
|
||||
marker: "CCR_CASE_GATEWAY_EARLY_DECISION_ARCHIVE_SEARCH_TOOL_17A",
|
||||
placement: "early",
|
||||
prompt: "The early design decision marker"
|
||||
},
|
||||
{
|
||||
detail: "Client compact adapters for Codex and Claude Code have been added.",
|
||||
key: "midProgress",
|
||||
marker: "CCR_CASE_GATEWAY_MID_PROGRESS_CLIENT_COMPACT_ADAPTER_42B",
|
||||
placement: "middle",
|
||||
prompt: "The mid-session progress marker"
|
||||
},
|
||||
{
|
||||
detail: "Real Claude Code /compact benchmark script exists.",
|
||||
key: "completed",
|
||||
marker: "CCR_CASE_GATEWAY_COMPLETED_REAL_COMPACT_SCRIPT_91C",
|
||||
placement: "recent",
|
||||
prompt: "The completed-work marker"
|
||||
},
|
||||
{
|
||||
detail: "Measure post-compact task understanding, not just token reduction.",
|
||||
key: "currentFocus",
|
||||
marker: "CCR_CASE_GATEWAY_FOCUS_POST_COMPACT_TASK_UNDERSTANDING_33D",
|
||||
placement: "recent",
|
||||
prompt: "The current focus marker"
|
||||
},
|
||||
{
|
||||
detail: "Compare native summary with history retrieval and continuation recall.",
|
||||
key: "nextStep",
|
||||
marker: "CCR_CASE_GATEWAY_NEXT_COMPARE_SUMMARY_WITH_HISTORY_RETRIEVAL_58E",
|
||||
placement: "recent",
|
||||
prompt: "The next-step marker"
|
||||
},
|
||||
{
|
||||
detail: "Run npm test:main and the context archive benchmark.",
|
||||
key: "validationCommand",
|
||||
marker: "CCR_CASE_GATEWAY_VALIDATE_NPM_TEST_MAIN_AND_BENCHMARK_76F",
|
||||
placement: "recent",
|
||||
prompt: "The validation command marker"
|
||||
},
|
||||
{
|
||||
detail: "Avoid false positives for unrelated compact UI density requests.",
|
||||
key: "risk",
|
||||
marker: "CCR_CASE_GATEWAY_RISK_FALSE_POSITIVE_COMPACT_UI_DENSITY_24G",
|
||||
placement: "recent",
|
||||
prompt: "The remaining risk marker"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
id: "ci-cache-accounting",
|
||||
title: "CI failure in cached token accounting",
|
||||
filler: [
|
||||
"The failing CI job reports token totals that double-count cache read and creation values.",
|
||||
"The fix must keep Anthropic and OpenAI-compatible usage normalization behavior separate.",
|
||||
"Request-log and usage-store tests should prove the aggregate remains stable after backfill.",
|
||||
"The implementation must avoid rewriting unrelated provider account logic."
|
||||
],
|
||||
facts: [
|
||||
{
|
||||
detail: "Fix CI failure caused by cached input tokens being counted twice.",
|
||||
key: "objective",
|
||||
marker: "CCR_CASE_CI_OBJECTIVE_FIX_CACHE_TOKEN_ACCOUNTING_81A",
|
||||
placement: "recent",
|
||||
prompt: "The current objective marker"
|
||||
},
|
||||
{
|
||||
detail: "Treat OpenAI cache read and cache creation as already included in prompt tokens.",
|
||||
key: "earlyDecision",
|
||||
marker: "CCR_CASE_CI_EARLY_DECISION_SUBTRACT_OPENAI_CACHE_TOKENS_11B",
|
||||
placement: "early",
|
||||
prompt: "The early design decision marker"
|
||||
},
|
||||
{
|
||||
detail: "normalizeUsageInputTokens was updated but usage-store aggregation still needs verification.",
|
||||
key: "midProgress",
|
||||
marker: "CCR_CASE_CI_MID_PROGRESS_NORMALIZER_PATCHED_VERIFY_STORE_22C",
|
||||
placement: "middle",
|
||||
prompt: "The mid-session progress marker"
|
||||
},
|
||||
{
|
||||
detail: "Added regression fixture for mixed prompt/cache token payloads.",
|
||||
key: "completed",
|
||||
marker: "CCR_CASE_CI_COMPLETED_MIXED_CACHE_FIXTURE_33D",
|
||||
placement: "recent",
|
||||
prompt: "The completed-work marker"
|
||||
},
|
||||
{
|
||||
detail: "Focus is reconciling UsageStore day totals with RequestLogStore detail rows.",
|
||||
key: "currentFocus",
|
||||
marker: "CCR_CASE_CI_FOCUS_RECONCILE_USAGESTORE_REQUESTLOG_44E",
|
||||
placement: "recent",
|
||||
prompt: "The current focus marker"
|
||||
},
|
||||
{
|
||||
detail: "Next step is to run the focused usage normalization and usage-store tests.",
|
||||
key: "nextStep",
|
||||
marker: "CCR_CASE_CI_NEXT_RUN_USAGE_NORMALIZATION_TESTS_55F",
|
||||
placement: "recent",
|
||||
prompt: "The next-step marker"
|
||||
},
|
||||
{
|
||||
detail: "Validation command is npm run test:main.",
|
||||
key: "validationCommand",
|
||||
marker: "CCR_CASE_CI_VALIDATE_NPM_RUN_TEST_MAIN_66G",
|
||||
placement: "recent",
|
||||
prompt: "The validation command marker"
|
||||
},
|
||||
{
|
||||
detail: "Risk is breaking Anthropic accounting while fixing OpenAI-compatible accounting.",
|
||||
key: "risk",
|
||||
marker: "CCR_CASE_CI_RISK_ANTHROPIC_ACCOUNTING_REGRESSION_77H",
|
||||
placement: "recent",
|
||||
prompt: "The remaining risk marker"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
id: "frontend-mobile-drawer",
|
||||
title: "Frontend mobile drawer regression",
|
||||
filler: [
|
||||
"The contacts page mobile drawer should not overlap fixed toolbar controls.",
|
||||
"Responsive layout must preserve dense operational UI rather than introducing a marketing layout.",
|
||||
"Visual verification should include a 390x844 screenshot and text-overlap checks.",
|
||||
"The change should respect existing BaseUI and Tailwind conventions."
|
||||
],
|
||||
facts: [
|
||||
{
|
||||
detail: "Fix mobile contacts drawer overlap in the web UI.",
|
||||
key: "objective",
|
||||
marker: "CCR_CASE_UI_OBJECTIVE_FIX_CONTACTS_MOBILE_DRAWER_12A",
|
||||
placement: "recent",
|
||||
prompt: "The current objective marker"
|
||||
},
|
||||
{
|
||||
detail: "Keep the drawer as an operational panel, not a new landing-style card layout.",
|
||||
key: "earlyDecision",
|
||||
marker: "CCR_CASE_UI_EARLY_DECISION_KEEP_OPERATIONAL_PANEL_23B",
|
||||
placement: "early",
|
||||
prompt: "The early design decision marker"
|
||||
},
|
||||
{
|
||||
detail: "The drawer close button and filter tabs were moved into a stable toolbar grid.",
|
||||
key: "midProgress",
|
||||
marker: "CCR_CASE_UI_MID_PROGRESS_STABLE_TOOLBAR_GRID_34C",
|
||||
placement: "middle",
|
||||
prompt: "The mid-session progress marker"
|
||||
},
|
||||
{
|
||||
detail: "Updated packages/ui/src/pages/home/components/contacts.tsx.",
|
||||
key: "completed",
|
||||
marker: "CCR_CASE_UI_COMPLETED_CONTACTS_TSX_PATCH_45D",
|
||||
placement: "recent",
|
||||
prompt: "The completed-work marker"
|
||||
},
|
||||
{
|
||||
detail: "Focus is checking that labels fit in 390px viewport without overlap.",
|
||||
key: "currentFocus",
|
||||
marker: "CCR_CASE_UI_FOCUS_390PX_NO_LABEL_OVERLAP_56E",
|
||||
placement: "recent",
|
||||
prompt: "The current focus marker"
|
||||
},
|
||||
{
|
||||
detail: "Next step is to capture mobile screenshot and inspect toolbar boundaries.",
|
||||
key: "nextStep",
|
||||
marker: "CCR_CASE_UI_NEXT_CAPTURE_MOBILE_SCREENSHOT_67F",
|
||||
placement: "recent",
|
||||
prompt: "The next-step marker"
|
||||
},
|
||||
{
|
||||
detail: "Validation command is npm run test:renderer.",
|
||||
key: "validationCommand",
|
||||
marker: "CCR_CASE_UI_VALIDATE_NPM_RUN_TEST_RENDERER_78G",
|
||||
placement: "recent",
|
||||
prompt: "The validation command marker"
|
||||
},
|
||||
{
|
||||
detail: "Risk is text overlap from long localized labels.",
|
||||
key: "risk",
|
||||
marker: "CCR_CASE_UI_RISK_LOCALIZED_LABEL_OVERLAP_89H",
|
||||
placement: "recent",
|
||||
prompt: "The remaining risk marker"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
id: "database-migration",
|
||||
title: "Database migration and backfill",
|
||||
filler: [
|
||||
"The migration adds a nullable column before backfilling derived request metadata.",
|
||||
"SQLite and Postgres behavior must stay aligned for local desktop and server deployments.",
|
||||
"The backfill should be resumable and avoid locking the request log table for too long.",
|
||||
"Tests need to cover old rows, partially migrated rows, and new writes."
|
||||
],
|
||||
facts: [
|
||||
{
|
||||
detail: "Add migration for gateway request context archive metadata.",
|
||||
key: "objective",
|
||||
marker: "CCR_CASE_DB_OBJECTIVE_ADD_CONTEXT_ARCHIVE_METADATA_MIGRATION_13A",
|
||||
placement: "recent",
|
||||
prompt: "The current objective marker"
|
||||
},
|
||||
{
|
||||
detail: "Use an additive nullable column before any destructive schema changes.",
|
||||
key: "earlyDecision",
|
||||
marker: "CCR_CASE_DB_EARLY_DECISION_ADDITIVE_NULLABLE_COLUMN_24B",
|
||||
placement: "early",
|
||||
prompt: "The early design decision marker"
|
||||
},
|
||||
{
|
||||
detail: "Backfill cursor now stores the last processed request id.",
|
||||
key: "midProgress",
|
||||
marker: "CCR_CASE_DB_MID_PROGRESS_BACKFILL_CURSOR_REQUEST_ID_35C",
|
||||
placement: "middle",
|
||||
prompt: "The mid-session progress marker"
|
||||
},
|
||||
{
|
||||
detail: "Added migration test for old request_log rows.",
|
||||
key: "completed",
|
||||
marker: "CCR_CASE_DB_COMPLETED_OLD_REQUEST_LOG_MIGRATION_TEST_46D",
|
||||
placement: "recent",
|
||||
prompt: "The completed-work marker"
|
||||
},
|
||||
{
|
||||
detail: "Focus is ensuring backfill is idempotent after process restart.",
|
||||
key: "currentFocus",
|
||||
marker: "CCR_CASE_DB_FOCUS_IDEMPOTENT_BACKFILL_RESTART_57E",
|
||||
placement: "recent",
|
||||
prompt: "The current focus marker"
|
||||
},
|
||||
{
|
||||
detail: "Next step is to run migration tests against a temporary SQLite database.",
|
||||
key: "nextStep",
|
||||
marker: "CCR_CASE_DB_NEXT_RUN_TEMP_SQLITE_MIGRATION_TEST_68F",
|
||||
placement: "recent",
|
||||
prompt: "The next-step marker"
|
||||
},
|
||||
{
|
||||
detail: "Validation command is npm run test:main.",
|
||||
key: "validationCommand",
|
||||
marker: "CCR_CASE_DB_VALIDATE_NPM_RUN_TEST_MAIN_79G",
|
||||
placement: "recent",
|
||||
prompt: "The validation command marker"
|
||||
},
|
||||
{
|
||||
detail: "Risk is long-running backfill blocking interactive request logging.",
|
||||
key: "risk",
|
||||
marker: "CCR_CASE_DB_RISK_BACKFILL_BLOCKS_REQUEST_LOGGING_80H",
|
||||
placement: "recent",
|
||||
prompt: "The remaining risk marker"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
id: "github-review-fix",
|
||||
title: "GitHub PR review fix",
|
||||
filler: [
|
||||
"The review thread points to a subtle behavioral regression, not just formatting.",
|
||||
"The patch should address the review comment without broad refactoring.",
|
||||
"Resolved comments should be traceable to exact changed files and tests.",
|
||||
"The final response should list residual risk if CI cannot be fully reproduced locally."
|
||||
],
|
||||
facts: [
|
||||
{
|
||||
detail: "Address PR review feedback about provider fallback response headers.",
|
||||
key: "objective",
|
||||
marker: "CCR_CASE_REVIEW_OBJECTIVE_FIX_FALLBACK_RESPONSE_HEADERS_14A",
|
||||
placement: "recent",
|
||||
prompt: "The current objective marker"
|
||||
},
|
||||
{
|
||||
detail: "Do not collapse fallback credential-chain diagnostics into the primary attempt.",
|
||||
key: "earlyDecision",
|
||||
marker: "CCR_CASE_REVIEW_EARLY_DECISION_KEEP_ATTEMPT_DIAGNOSTICS_SEPARATE_25B",
|
||||
placement: "early",
|
||||
prompt: "The early design decision marker"
|
||||
},
|
||||
{
|
||||
detail: "fetchUpstreamWithFallback now returns the selected attempt metadata.",
|
||||
key: "midProgress",
|
||||
marker: "CCR_CASE_REVIEW_MID_PROGRESS_SELECTED_ATTEMPT_METADATA_36C",
|
||||
placement: "middle",
|
||||
prompt: "The mid-session progress marker"
|
||||
},
|
||||
{
|
||||
detail: "Updated tests/main/gateway-virtual-models.test.mjs.",
|
||||
key: "completed",
|
||||
marker: "CCR_CASE_REVIEW_COMPLETED_GATEWAY_VIRTUAL_MODELS_TEST_47D",
|
||||
placement: "recent",
|
||||
prompt: "The completed-work marker"
|
||||
},
|
||||
{
|
||||
detail: "Focus is preserving x-ccr-route-reason on fallback responses.",
|
||||
key: "currentFocus",
|
||||
marker: "CCR_CASE_REVIEW_FOCUS_PRESERVE_ROUTE_REASON_HEADER_58E",
|
||||
placement: "recent",
|
||||
prompt: "The current focus marker"
|
||||
},
|
||||
{
|
||||
detail: "Next step is rerun targeted gateway tests and inspect diff.",
|
||||
key: "nextStep",
|
||||
marker: "CCR_CASE_REVIEW_NEXT_RUN_TARGETED_GATEWAY_TESTS_69F",
|
||||
placement: "recent",
|
||||
prompt: "The next-step marker"
|
||||
},
|
||||
{
|
||||
detail: "Validation command is npm run test:main.",
|
||||
key: "validationCommand",
|
||||
marker: "CCR_CASE_REVIEW_VALIDATE_NPM_RUN_TEST_MAIN_70G",
|
||||
placement: "recent",
|
||||
prompt: "The validation command marker"
|
||||
},
|
||||
{
|
||||
detail: "Risk is masking upstream provider auth failures as routing failures.",
|
||||
key: "risk",
|
||||
marker: "CCR_CASE_REVIEW_RISK_MASKING_PROVIDER_AUTH_FAILURES_81H",
|
||||
placement: "recent",
|
||||
prompt: "The remaining risk marker"
|
||||
}
|
||||
]
|
||||
}
|
||||
];
|
||||
|
||||
export const defaultTaskCaseId = taskCases[0].id;
|
||||
|
||||
export function taskCaseIds() {
|
||||
return taskCases.map((taskCase) => taskCase.id);
|
||||
}
|
||||
|
||||
export function findTaskCase(id = defaultTaskCaseId) {
|
||||
const taskCase = taskCases.find((candidate) => candidate.id === id);
|
||||
if (!taskCase) {
|
||||
throw new Error(`Unknown task case: ${id}. Available cases: ${taskCaseIds().join(", ")}`);
|
||||
}
|
||||
return taskCase;
|
||||
}
|
||||
|
||||
export function selectTaskCases(value = "all") {
|
||||
if (value === "all") {
|
||||
return taskCases;
|
||||
}
|
||||
return value.split(",").map((id) => findTaskCase(id.trim())).filter(Boolean);
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import esbuild from "esbuild";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const projectRoot = path.resolve(__dirname, "..");
|
||||
const outDir = mkdtempSync(path.join(os.tmpdir(), "ccr-context-archive-benchmark-"));
|
||||
const outfile = path.join(outDir, "context-archive-benchmark.mjs");
|
||||
const keepBundle = process.argv.includes("--keep-bundle");
|
||||
|
||||
try {
|
||||
await esbuild.build({
|
||||
absWorkingDir: projectRoot,
|
||||
bundle: true,
|
||||
entryPoints: [path.join(projectRoot, "benchmarks", "context-archive-benchmark.ts")],
|
||||
external: ["better-sqlite3", "electron"],
|
||||
format: "esm",
|
||||
legalComments: "none",
|
||||
logLevel: "silent",
|
||||
outfile,
|
||||
platform: "node",
|
||||
target: "node22"
|
||||
});
|
||||
|
||||
const benchmark = await import(pathToFileURL(outfile).href);
|
||||
await benchmark.main(process.argv.slice(2).filter((arg) => arg !== "--keep-bundle"));
|
||||
} finally {
|
||||
if (!keepBundle) {
|
||||
rmSync(outDir, { force: true, recursive: true });
|
||||
} else {
|
||||
console.error(`Kept benchmark bundle at ${outfile}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { build } from "esbuild";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const projectRoot = process.cwd();
|
||||
const outDir = mkdtempSync(path.join(os.tmpdir(), "ccr-context-archive-real-agent-benchmark-"));
|
||||
const outfile = path.join(outDir, "context-archive-real-agent-benchmark.mjs");
|
||||
|
||||
await build({
|
||||
bundle: true,
|
||||
entryPoints: [path.join(projectRoot, "benchmarks", "context-archive-real-agent-benchmark.ts")],
|
||||
external: ["esbuild"],
|
||||
format: "esm",
|
||||
logLevel: "silent",
|
||||
outfile,
|
||||
platform: "node"
|
||||
});
|
||||
|
||||
try {
|
||||
const benchmark = await import(pathToFileURL(outfile).href);
|
||||
await benchmark.main(process.argv.slice(2).filter((arg) => arg !== "--keep-bundle"));
|
||||
} finally {
|
||||
if (process.argv.includes("--keep-bundle")) {
|
||||
console.error(`Kept benchmark bundle at ${outfile}`);
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,9 @@
|
||||
"prepack": "npm run build:assets",
|
||||
"prepublishOnly": "npm run typecheck",
|
||||
"preview": "npm run build:assets && electron .",
|
||||
"bench:context-archive": "node benchmarks/run-context-archive-benchmark.mjs",
|
||||
"bench:context-archive-real": "node benchmarks/run-context-archive-real-agent-benchmark.mjs",
|
||||
"bench:claude-compact-real": "node benchmarks/claude-code-real-compact.mjs",
|
||||
"docker:build": "docker build -t claude-code-router:local .",
|
||||
"docker:run": "docker run --rm -p 3458:8080 -v ccr-data:/data claude-code-router:local",
|
||||
"test": "node build/test.mjs && node build/run-tests.mjs",
|
||||
|
||||
@@ -703,6 +703,10 @@ function parseContextArchive(value: unknown): Partial<ContextArchiveConfig> | un
|
||||
if (typeof mcpEnabled === "boolean") {
|
||||
contextArchive.mcpEnabled = mcpEnabled;
|
||||
}
|
||||
const claudeCodeCompact = value.claudeCodeCompact ?? value.claude_code_compact ?? value.overrideClaudeCodeCompact ?? value.override_claude_code_compact;
|
||||
if (typeof claudeCodeCompact === "boolean") {
|
||||
contextArchive.claudeCodeCompact = claudeCodeCompact;
|
||||
}
|
||||
const triggerTokenLimit = readNumber(value.triggerTokenLimit ?? value.trigger_token_limit);
|
||||
if (triggerTokenLimit !== undefined) {
|
||||
contextArchive.triggerTokenLimit = clampNumber(triggerTokenLimit, 1000, 2_000_000);
|
||||
|
||||
@@ -83,6 +83,7 @@ export function createDefaultAppConfig(options: DefaultAppConfigOptions): AppCon
|
||||
tenantId: "ccr"
|
||||
},
|
||||
contextArchive: {
|
||||
claudeCodeCompact: false,
|
||||
enabled: false,
|
||||
handoffMaxCharacters: 24000,
|
||||
llm: {
|
||||
|
||||
@@ -664,6 +664,7 @@ export type ContextArchiveLlmConfig = {
|
||||
};
|
||||
|
||||
export type ContextArchiveConfig = {
|
||||
claudeCodeCompact: boolean;
|
||||
enabled: boolean;
|
||||
handoffMaxCharacters: number;
|
||||
llm: ContextArchiveLlmConfig;
|
||||
|
||||
@@ -377,11 +377,14 @@ export async function prepareContextArchiveRequest(input: {
|
||||
}
|
||||
|
||||
const retained = clampInteger(archiveConfig.retainRecentItems, 2, 200, 12);
|
||||
const replaceClientCompact = clientCompact && shouldReplaceClientCompact(client, archiveConfig);
|
||||
const prunedEntries = clientCompact
|
||||
? extractArchiveEntries(parsedBody, protocol)
|
||||
: extractPrunedEntries(parsedBody, protocol, retained);
|
||||
const reason = clientCompact
|
||||
? `${client} requested a context compaction/summary; CCR archived the full request and injected history-retrieval handoff instructions without pruning the client payload.`
|
||||
? replaceClientCompact
|
||||
? `${client} requested a context compaction/summary; CCR replaced the native compaction input with a compact handoff plus recent context while archiving the full request for history retrieval.`
|
||||
: `${client} requested a context compaction/summary; CCR archived the full request and injected history-retrieval handoff instructions without pruning the client payload.`
|
||||
: undefined;
|
||||
const handoff = await buildHandoff({
|
||||
archiveConfig,
|
||||
@@ -394,18 +397,26 @@ export async function prepareContextArchiveRequest(input: {
|
||||
toolName: archiveConfig.toolName || defaultToolName
|
||||
});
|
||||
const compactedBody = clientCompact
|
||||
? adaptClientCompactBody(parsedBody, protocol, handoff, {
|
||||
client,
|
||||
sessionId,
|
||||
toolName: archiveConfig.toolName || defaultToolName
|
||||
})
|
||||
? replaceClientCompact
|
||||
? replaceClientCompactBody(parsedBody, protocol, clientCompactInstruction(handoff, {
|
||||
client,
|
||||
sessionId,
|
||||
toolName: archiveConfig.toolName || defaultToolName
|
||||
}), retained)
|
||||
: adaptClientCompactBody(parsedBody, protocol, handoff, {
|
||||
client,
|
||||
sessionId,
|
||||
toolName: archiveConfig.toolName || defaultToolName
|
||||
})
|
||||
: compactBody(parsedBody, protocol, handoff, retained);
|
||||
contextArchiveService.recordHandoff(record, handoff, archiveConfig);
|
||||
|
||||
return {
|
||||
body: Buffer.from(`${JSON.stringify(compactedBody)}\n`, "utf8"),
|
||||
diagnostic: clientCompact
|
||||
? `client-compact:${client}:${sessionId}:${estimatedTokens}`
|
||||
? replaceClientCompact
|
||||
? `client-compact-ccr:${client}:${sessionId}:${estimatedTokens}`
|
||||
: `client-compact:${client}:${sessionId}:${estimatedTokens}`
|
||||
: `compacted:${sessionId}:${estimatedTokens}`,
|
||||
record
|
||||
};
|
||||
@@ -748,6 +759,105 @@ function compactBody(
|
||||
};
|
||||
}
|
||||
|
||||
function replaceClientCompactBody(
|
||||
body: Record<string, unknown>,
|
||||
protocol: GatewayProviderProtocol,
|
||||
instruction: string,
|
||||
retainRecentItems: number
|
||||
): Record<string, unknown> {
|
||||
const base = withoutToolAccess(body);
|
||||
const prompt =
|
||||
"Return the compacted summary as plain assistant message text for the next context window. Do not create, edit, or write files. Do not call tools.";
|
||||
if (protocol === "openai_chat_completions") {
|
||||
const messages = Array.isArray(body.messages) ? body.messages : [];
|
||||
const leading = leadingOpenAiInstructionMessages(messages);
|
||||
const recent = trimClientCompactTailMessages(messages.slice(Math.max(leading.length, messages.length - retainRecentItems)));
|
||||
return {
|
||||
...base,
|
||||
messages: [
|
||||
...leading,
|
||||
{ content: instruction, role: "system" },
|
||||
...recent,
|
||||
{ content: prompt, role: "user" }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
if (protocol === "openai_responses") {
|
||||
return {
|
||||
...base,
|
||||
input: prompt,
|
||||
instructions: appendTextBlock(body.instructions, instruction)
|
||||
};
|
||||
}
|
||||
|
||||
const messages = Array.isArray(body.messages) ? body.messages : [];
|
||||
return {
|
||||
...base,
|
||||
messages: [
|
||||
...trimClientCompactTailMessages(messages.slice(-retainRecentItems)),
|
||||
{ content: prompt, role: "user" }
|
||||
],
|
||||
system: appendAnthropicSystem(body.system, instruction)
|
||||
};
|
||||
}
|
||||
|
||||
function withoutToolAccess(body: Record<string, unknown>): Record<string, unknown> {
|
||||
const next = { ...body };
|
||||
delete next.tools;
|
||||
delete next.tool_choice;
|
||||
delete next.parallel_tool_calls;
|
||||
delete next.mcp_servers;
|
||||
return next;
|
||||
}
|
||||
|
||||
function trimClientCompactTailMessages(messages: unknown[]): unknown[] {
|
||||
let next = [...messages];
|
||||
if (isCompactPromptMessage(next.at(-1))) {
|
||||
next = next.slice(0, -1);
|
||||
}
|
||||
|
||||
while (next.length > 0) {
|
||||
const tail = next.at(-1);
|
||||
if (isToolResultOnlyMessage(tail)) {
|
||||
next = next.slice(0, -1);
|
||||
if (isAssistantToolUseMessage(next.at(-1))) {
|
||||
next = next.slice(0, -1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (isAssistantToolUseMessage(tail)) {
|
||||
next = next.slice(0, -1);
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
function isCompactPromptMessage(message: unknown): boolean {
|
||||
if (!isRecord(message)) {
|
||||
return false;
|
||||
}
|
||||
const role = stringValue(message.role);
|
||||
return role === "user" && matchesClientCompactPrompt(contentText(message.content));
|
||||
}
|
||||
|
||||
function isToolResultOnlyMessage(message: unknown): boolean {
|
||||
if (!isRecord(message) || !Array.isArray(message.content) || message.content.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return message.content.every((block) => isRecord(block) && block.type === "tool_result");
|
||||
}
|
||||
|
||||
function isAssistantToolUseMessage(message: unknown): boolean {
|
||||
if (!isRecord(message) || stringValue(message.role) !== "assistant" || !Array.isArray(message.content)) {
|
||||
return false;
|
||||
}
|
||||
return message.content.some((block) => isRecord(block) && block.type === "tool_use");
|
||||
}
|
||||
|
||||
function adaptClientCompactBody(
|
||||
body: Record<string, unknown>,
|
||||
protocol: GatewayProviderProtocol,
|
||||
@@ -796,7 +906,7 @@ function clientCompactInstruction(
|
||||
const clientName = input.client === "codex" ? "Codex" : input.client === "claude-code" ? "Claude Code" : "the client";
|
||||
return [
|
||||
`CCR detected this as a ${clientName} context compaction request.`,
|
||||
"When you write the compacted summary for the next context window, include a preserved 'Archived history access' section. Keep the archive session id and tool call shape exact so the next agent can retrieve details that are not in the summary.",
|
||||
"When you produce the compacted summary for the next context window, include a preserved 'Archived history access' section. Return the summary as assistant message text only; do not create or modify files or call tools. Keep the archive session id and tool call shape exact so the next agent can retrieve details that are not in the summary.",
|
||||
"",
|
||||
"Archived history access:",
|
||||
`- Archive session id: ${input.sessionId}`,
|
||||
@@ -906,11 +1016,11 @@ function isClientCompactRequest(input: {
|
||||
if (hasCompactHeader(input.headers) || hasStructuralCompactMarker(input.body)) {
|
||||
return true;
|
||||
}
|
||||
const text = extractArchiveEntries(input.body, input.protocol)
|
||||
.map((entry) => entry.text)
|
||||
.join("\n")
|
||||
.slice(-200000);
|
||||
return matchesClientCompactPrompt(text);
|
||||
return matchesClientCompactPrompt(clientCompactPromptCandidate(input.body, input.protocol));
|
||||
}
|
||||
|
||||
function shouldReplaceClientCompact(client: ContextArchiveClient, config: ContextArchiveConfig): boolean {
|
||||
return client === "claude-code" && config.claudeCodeCompact;
|
||||
}
|
||||
|
||||
function hasCompactHeader(headers: IncomingHttpHeaders | Record<string, string | string[] | undefined>): boolean {
|
||||
@@ -924,13 +1034,15 @@ function hasCompactHeader(headers: IncomingHttpHeaders | Record<string, string |
|
||||
}
|
||||
|
||||
function hasStructuralCompactMarker(body: Record<string, unknown>): boolean {
|
||||
if (recordHasCompactMarker(body)) {
|
||||
return true;
|
||||
}
|
||||
return [
|
||||
body,
|
||||
isRecord(body.metadata) ? body.metadata : undefined,
|
||||
isRecord(body.context_management) ? body.context_management : undefined,
|
||||
isRecord(body.contextManagement) ? body.contextManagement : undefined,
|
||||
isRecord(body.experimental) ? body.experimental : undefined
|
||||
].some((record) => Boolean(record && recordHasCompactMarker(record)));
|
||||
body.metadata,
|
||||
body.context_management,
|
||||
body.contextManagement,
|
||||
body.experimental
|
||||
].some((record) => structuralValueHasCompactMarker(record));
|
||||
}
|
||||
|
||||
function recordHasCompactMarker(record: Record<string, unknown>): boolean {
|
||||
@@ -951,6 +1063,20 @@ function recordHasCompactMarker(record: Record<string, unknown>): boolean {
|
||||
].some((value) => isCompactMarkerValue(stringValue(value)));
|
||||
}
|
||||
|
||||
function structuralValueHasCompactMarker(value: unknown, depth = 0): boolean {
|
||||
if (depth > 5) {
|
||||
return false;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((item) => structuralValueHasCompactMarker(item, depth + 1));
|
||||
}
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
return recordHasCompactMarker(value) ||
|
||||
Object.values(value).some((item) => structuralValueHasCompactMarker(item, depth + 1));
|
||||
}
|
||||
|
||||
function isCompactMarkerKey(key: string): boolean {
|
||||
return key === "compact" ||
|
||||
key === "context_compact" ||
|
||||
@@ -966,16 +1092,74 @@ function isCompactMarkerValue(value: string | undefined): boolean {
|
||||
normalized === "compact_20260112";
|
||||
}
|
||||
|
||||
function clientCompactPromptCandidate(body: Record<string, unknown>, protocol: GatewayProviderProtocol): string {
|
||||
if (protocol === "openai_responses") {
|
||||
if (Array.isArray(body.input)) {
|
||||
return latestUserPromptText(body.input);
|
||||
}
|
||||
return terminalPromptText(body.input);
|
||||
}
|
||||
return latestUserPromptText(Array.isArray(body.messages) ? body.messages : []);
|
||||
}
|
||||
|
||||
function latestUserPromptText(items: unknown[]): string {
|
||||
for (let index = items.length - 1; index >= 0; index -= 1) {
|
||||
const item = items[index];
|
||||
const role = isRecord(item) ? stringValue(item.role) : undefined;
|
||||
if (role && role !== "user") {
|
||||
continue;
|
||||
}
|
||||
if (isToolResultOnlyMessage(item)) {
|
||||
continue;
|
||||
}
|
||||
const text = terminalPromptText(isRecord(item) && item.content !== undefined ? item.content : item);
|
||||
if (text) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function terminalPromptText(value: unknown): string {
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (let index = value.length - 1; index >= 0; index -= 1) {
|
||||
const text = terminalPromptText(value[index]);
|
||||
if (text) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
if (!isRecord(value)) {
|
||||
return "";
|
||||
}
|
||||
if (value.type === "tool_result" || value.type === "function_call_output") {
|
||||
return "";
|
||||
}
|
||||
return stringValue(value.text) || stringValue(value.input) || stringValue(value.content) || "";
|
||||
}
|
||||
|
||||
function matchesClientCompactPrompt(text: string): boolean {
|
||||
const normalized = normalizeWhitespace(text).toLowerCase();
|
||||
if (!normalized) {
|
||||
return false;
|
||||
}
|
||||
const englishCompact = "\\b(?:compact(?:ion)?|condense|compress|summari[sz]e|summary)\\b";
|
||||
const englishContext = "\\b(?:conversation|session|history|context|transcript|handoff|messages|work so far|state)\\b";
|
||||
const summaryTerm = "\\b(?:summari[sz]e|summary)\\b";
|
||||
const compactTerm = "\\b(?:compact(?:ion)?|condense|compress)\\b";
|
||||
const historyScope = [
|
||||
"\\b(?:conversation|session|history|transcript|handoff|messages)\\b",
|
||||
"\\bwork so far\\b",
|
||||
"\\b(?:new|next|fresh)\\s+context(?:\\s+window)?\\b",
|
||||
"\\bcontext\\s+(?:window|summary|compaction)\\b"
|
||||
].join("|");
|
||||
return [
|
||||
new RegExp(`${englishCompact}[\\s\\S]{0,240}${englishContext}`, "i"),
|
||||
new RegExp(`${englishContext}[\\s\\S]{0,240}${englishCompact}`, "i"),
|
||||
new RegExp(`${summaryTerm}[\\s\\S]{0,240}(?:${historyScope})`, "i"),
|
||||
new RegExp(`(?:${historyScope})[\\s\\S]{0,240}${summaryTerm}`, "i"),
|
||||
new RegExp(`${compactTerm}[\\s\\S]{0,240}(?:${historyScope})`, "i"),
|
||||
/\bcontext\s+compaction\b/i,
|
||||
/\bcontinue\b[\s\S]{0,240}\b(?:new|next|fresh)\s+context(?:\s+window)?\b/i,
|
||||
/(?:总结|摘要|压缩|交接)[\s\S]{0,160}(?:会话|上下文|历史|窗口|新上下文|前文)/,
|
||||
/(?:会话|上下文|历史|窗口|前文)[\s\S]{0,160}(?:总结|摘要|压缩|交接)/
|
||||
|
||||
@@ -16,6 +16,7 @@ import { codexCliMiddlewareRuntimeScript } from "@ccr/core/agents/codex/cli-midd
|
||||
import { codexModelCatalogJson } from "@ccr/core/agents/codex/model-catalog";
|
||||
import { CONFIGDIR } from "@ccr/core/config/constants";
|
||||
import { resolveZcodeConfigFile, writeZcodeGatewayConfig, zcodeHomeFromConfigFile } from "@ccr/core/agents/zcode/profile-config";
|
||||
import { CONTEXT_ARCHIVE_MCP_SERVER_NAME, contextArchiveMcpServer } from "@ccr/core/gateway/context-archive";
|
||||
import { normalizeRouteSelector } from "@ccr/core/gateway/claude-code-router-plugin";
|
||||
import {
|
||||
TOOL_HUB_MCP_RUNTIME_FILE_NAME,
|
||||
@@ -23,6 +24,7 @@ import {
|
||||
bundledToolHubMcpEntryPathCandidates,
|
||||
toolHubClaudeCodeMcpConfig,
|
||||
toolHubMcpRuntimeConfig,
|
||||
type ClaudeCodeMcpServerConfig,
|
||||
type ToolHubMcpRuntimeConfig
|
||||
} from "@ccr/core/mcp/toolhub-config";
|
||||
|
||||
@@ -511,7 +513,7 @@ function claudeCodeToolHubMcpConfigFile(profile: ProfileConfig): string {
|
||||
function writeClaudeCodeToolHubMcpConfig(config: AppConfig, profile: ProfileConfig, token: string): { changed: boolean; file?: string } {
|
||||
const file = claudeCodeToolHubMcpConfigFile(profile);
|
||||
const entryPath = path.join(CONFIGDIR, "bin", TOOL_HUB_MCP_RUNTIME_FILE_NAME);
|
||||
const mcpConfig = toolHubClaudeCodeMcpConfig(config, {
|
||||
const toolHubMcpConfig = toolHubClaudeCodeMcpConfig(config, {
|
||||
entryPath,
|
||||
resolver: {
|
||||
apiKey: token,
|
||||
@@ -519,6 +521,12 @@ function writeClaudeCodeToolHubMcpConfig(config: AppConfig, profile: ProfileConf
|
||||
model: toolHubResolverModel(config)
|
||||
}
|
||||
});
|
||||
const contextArchiveMcpConfig = claudeCodeContextArchiveMcpConfig(config, token);
|
||||
const mcpServers = {
|
||||
...(toolHubMcpConfig?.mcpServers ?? {}),
|
||||
...(contextArchiveMcpConfig ? { [CONTEXT_ARCHIVE_MCP_SERVER_NAME]: contextArchiveMcpConfig } : {})
|
||||
};
|
||||
const mcpConfig = Object.keys(mcpServers).length > 0 ? { mcpServers } : undefined;
|
||||
if (!mcpConfig) {
|
||||
if (existsSync(file)) {
|
||||
rmSync(file, { force: true });
|
||||
@@ -527,11 +535,27 @@ function writeClaudeCodeToolHubMcpConfig(config: AppConfig, profile: ProfileConf
|
||||
return { changed: false };
|
||||
}
|
||||
|
||||
const runtimeResult = ensureToolHubMcpRuntimeFile(entryPath);
|
||||
const runtimeResult = toolHubMcpConfig ? ensureToolHubMcpRuntimeFile(entryPath) : { changed: false };
|
||||
const writeResult = writeGeneratedFileIfChanged(file, `${JSON.stringify(mcpConfig, null, 2)}\n`, { mode: privateFileMode });
|
||||
return { changed: runtimeResult.changed || writeResult.changed, file };
|
||||
}
|
||||
|
||||
function claudeCodeContextArchiveMcpConfig(config: AppConfig, token: string): ClaudeCodeMcpServerConfig | undefined {
|
||||
const server = contextArchiveMcpServer(config, gatewayEndpoint(config), token);
|
||||
if (!server || !("url" in server)) {
|
||||
return undefined;
|
||||
}
|
||||
const headers = {
|
||||
...(server.headers ?? {}),
|
||||
...(token ? { Authorization: `Bearer ${token}` } : {})
|
||||
};
|
||||
return {
|
||||
...(Object.keys(headers).length > 0 ? { headers } : {}),
|
||||
type: "http",
|
||||
url: server.url
|
||||
};
|
||||
}
|
||||
|
||||
function writeCodexToolHubMcpRuntimeConfig(config: AppConfig, token: string): { changed: boolean; file?: string; runtime?: ToolHubMcpRuntimeConfig } {
|
||||
const entryPath = path.join(CONFIGDIR, "bin", TOOL_HUB_MCP_RUNTIME_FILE_NAME);
|
||||
const runtime = toolHubMcpRuntimeConfig(config, undefined, {
|
||||
|
||||
@@ -19,7 +19,7 @@ import {
|
||||
isRoutingRewriteDraftRowValid,
|
||||
LayoutGroup, mergeModelDisplayNames, mergeProviderCapabilities, mergeProviderModelLists, modelDescriptionsForModels, modelDisplayNamesForModels,
|
||||
navigation, NavigationId, normalizeApiKeys, normalizeBotGatewaySavedConfigs, normalizeConfig, normalizeLanguagePreference, normalizeObservabilityConfig, normalizeOverviewWidgets,
|
||||
normalizeProfileItem, normalizeProfileScope, normalizeProviderBaseUrl, normalizeRouterBuiltInRules, normalizeRouterFallbackConfig, normalizeThemePreference, normalizeToolHubConfig, normalizeTrayBalanceProgressConfig, normalizeTrayIconPreference,
|
||||
normalizeContextArchiveConfig, normalizeProfileItem, normalizeProfileScope, normalizeProviderBaseUrl, normalizeRouterBuiltInRules, normalizeRouterFallbackConfig, normalizeThemePreference, normalizeToolHubConfig, normalizeTrayBalanceProgressConfig, normalizeTrayIconPreference,
|
||||
normalizeTrayWidgets, normalizeTrayWindowModules, normalizeVirtualModelDraftPatch, numberValue, OnboardingReadinessOptions, OnboardingStepId, onboardingStepOrder,
|
||||
OverviewWidgetConfig, parsePluginAppsSettingsText, parsePluginConfigSettingsText, parseProviderAccountDraft,
|
||||
providerCredentialsFromDraft,
|
||||
@@ -188,6 +188,7 @@ function App() {
|
||||
const [profileDraft, setProfileDraft] = useState<AddProfileDraft>(() => createProfileDraft());
|
||||
const [profileEditDraft, setProfileEditDraft] = useState<AddProfileDraft>(() => createProfileDraft());
|
||||
const [profileEditIndex, setProfileEditIndex] = useState<number>();
|
||||
const [profileContextArchiveDraft, setProfileContextArchiveDraft] = useState<AppConfig["contextArchive"]>(() => fallbackConfig.contextArchive);
|
||||
const [profileOpenDialog, setProfileOpenDialog] = useState<ProfileOpenDialogState>();
|
||||
const [profileActionBusy, setProfileActionBusy] = useState<ProfileActionBusy>();
|
||||
const [profileRuntimeStatus, setProfileRuntimeStatus] = useState<ProfileRuntimeStatus>({ profiles: [] });
|
||||
@@ -2405,6 +2406,7 @@ function App() {
|
||||
function openAddProfileDialog(agent: ProfileConfig["agent"] = profileAgentTab) {
|
||||
setProfileAgentTab(agent);
|
||||
setProfileDraft(createProfileDraft(agent));
|
||||
setProfileContextArchiveDraft(draftConfig.contextArchive);
|
||||
setProfileActionError("");
|
||||
setProfileAddOpen(true);
|
||||
}
|
||||
@@ -2416,6 +2418,7 @@ function App() {
|
||||
}
|
||||
setProfileEditIndex(index);
|
||||
setProfileEditDraft(createProfileDraftFromProfile(profile, draftConfig.botConfigs));
|
||||
setProfileContextArchiveDraft(draftConfig.contextArchive);
|
||||
setProfileActionError("");
|
||||
}
|
||||
|
||||
@@ -2662,6 +2665,17 @@ function App() {
|
||||
setProfileActionError("");
|
||||
}
|
||||
|
||||
function updateProfileContextArchiveDraft(patch: Partial<AppConfig["contextArchive"]>) {
|
||||
setProfileContextArchiveDraft((current) => normalizeContextArchiveConfig({
|
||||
...current,
|
||||
...patch,
|
||||
llm: {
|
||||
...current.llm,
|
||||
...(patch.llm ?? {})
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
async function submitProfileDraft(): Promise<boolean> {
|
||||
if (profileSubmitBusy) {
|
||||
return false;
|
||||
@@ -2677,8 +2691,10 @@ function App() {
|
||||
const existingProfile = onboardingProfileIndex >= 0 ? draftConfig.profile.profiles[onboardingProfileIndex] : undefined;
|
||||
const profile = profileConfigFromDraft(profileDraft, draftConfig.profile.profiles, existingProfile, draftConfig.botConfigs);
|
||||
setProfileAgentTab(profile.agent);
|
||||
const applyContextArchiveDraft = profileAddOpen && profile.agent === "claude-code";
|
||||
const next = buildConfigUpdate((config) => ({
|
||||
...config,
|
||||
contextArchive: applyContextArchiveDraft ? profileContextArchiveDraft : config.contextArchive,
|
||||
profile: {
|
||||
...config.profile,
|
||||
enabled: true,
|
||||
@@ -2735,6 +2751,7 @@ function App() {
|
||||
profiles[profileEditIndex] = nextProfile;
|
||||
return {
|
||||
...config,
|
||||
contextArchive: nextProfile.agent === "claude-code" ? profileContextArchiveDraft : config.contextArchive,
|
||||
profile: {
|
||||
...config.profile,
|
||||
profiles: enforceSingleEnabledGlobalProfilePerAgent(profiles, profileEditIndex)
|
||||
@@ -3047,10 +3064,12 @@ function App() {
|
||||
profileAdd={profileAddOpen ? {
|
||||
botConfigs: draftConfig.botConfigs,
|
||||
canSubmit: canSubmitProfile,
|
||||
contextArchive: profileContextArchiveDraft,
|
||||
draft: profileDraft,
|
||||
error: profileActionError,
|
||||
mode: "add",
|
||||
onChange: updateProfileDraft,
|
||||
onChangeContextArchive: updateProfileContextArchiveDraft,
|
||||
onCreateBot: openBotSettingsWithAddDialog,
|
||||
onClose: () => setProfileAddOpen(false),
|
||||
providers: draftConfig.Providers,
|
||||
@@ -3061,10 +3080,12 @@ function App() {
|
||||
profileEdit={profileEditIndex !== undefined ? {
|
||||
botConfigs: draftConfig.botConfigs,
|
||||
canSubmit: canSubmitProfileEdit,
|
||||
contextArchive: profileContextArchiveDraft,
|
||||
draft: profileEditDraft,
|
||||
error: profileActionError,
|
||||
mode: "edit",
|
||||
onChange: updateProfileEditDraft,
|
||||
onChangeContextArchive: updateProfileContextArchiveDraft,
|
||||
onCreateBot: openBotSettingsWithAddDialog,
|
||||
onClose: () => {
|
||||
setProfileEditIndex(undefined);
|
||||
|
||||
@@ -68,7 +68,7 @@ export function ProfileView({
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="min-h-0 flex-1 space-y-4 overflow-auto">
|
||||
<CardContent className="min-h-0 flex-1 space-y-3 overflow-auto">
|
||||
<div className="space-y-2">
|
||||
{profiles.length === 0 ? (
|
||||
<div className="flex h-32 items-center justify-center rounded-md border border-dashed border-border bg-muted/20 text-[12px] text-muted-foreground">
|
||||
@@ -176,6 +176,55 @@ export function ProfileView({
|
||||
);
|
||||
}
|
||||
|
||||
function ClaudeCodeContextArchiveCompactSetting({
|
||||
contextArchive,
|
||||
onChange
|
||||
}: {
|
||||
contextArchive: AppConfig["contextArchive"];
|
||||
onChange: (patch: Partial<AppConfig["contextArchive"]>) => void;
|
||||
}) {
|
||||
const t = useAppText();
|
||||
const checked = Boolean(contextArchive.claudeCodeCompact);
|
||||
const archiveReady = Boolean(contextArchive.enabled && contextArchive.mcpEnabled !== false);
|
||||
|
||||
return (
|
||||
<div className="rounded-md border border-border bg-muted/20 px-3 py-3">
|
||||
<div className="flex min-w-0 flex-wrap items-start justify-between gap-3">
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<AgentLogo agent="claude-code" className="h-6 w-6 rounded-[5px]" />
|
||||
<div className="min-w-0">
|
||||
<div className="truncate text-[13px] font-semibold">{t("CCR compact for Claude Code")}</div>
|
||||
<div className="mt-0.5 text-[12px] leading-5 text-muted-foreground">
|
||||
{t("Use CCR context archive when Claude Code runs /compact.")}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{checked && !archiveReady ? (
|
||||
<div className="mt-2 flex min-w-0 items-start gap-2 rounded-md border border-amber-500/30 bg-amber-500/10 px-2 py-1.5 text-[12px] leading-5 text-amber-700 dark:text-amber-300">
|
||||
<Info className="mt-0.5 h-3.5 w-3.5 shrink-0" />
|
||||
<span className="min-w-0">{t("Context archive and MCP access will be enabled for this compact mode.")}</span>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
<Toggle
|
||||
checked={checked}
|
||||
title={t("CCR compact for Claude Code")}
|
||||
onChange={(enabled) => onChange(enabled
|
||||
? {
|
||||
claudeCodeCompact: true,
|
||||
enabled: true,
|
||||
mcpEnabled: true
|
||||
}
|
||||
: {
|
||||
claudeCodeCompact: false
|
||||
})}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ProfileOpenDialog({
|
||||
appRunning = false,
|
||||
busy,
|
||||
@@ -758,17 +807,21 @@ function ProfileModelSelector({
|
||||
|
||||
export function AddProfileForm({
|
||||
botConfigs,
|
||||
contextArchive,
|
||||
draft,
|
||||
error,
|
||||
onChange,
|
||||
onChangeContextArchive,
|
||||
onCreateBot,
|
||||
providers,
|
||||
virtualModelProfiles = []
|
||||
}: {
|
||||
botConfigs: BotGatewaySavedConfig[];
|
||||
contextArchive?: AppConfig["contextArchive"];
|
||||
draft: AddProfileDraft;
|
||||
error: string;
|
||||
onChange: (patch: Partial<AddProfileDraft>) => void;
|
||||
onChangeContextArchive?: (patch: Partial<AppConfig["contextArchive"]>) => void;
|
||||
onCreateBot: () => void;
|
||||
providers: GatewayProviderConfig[];
|
||||
virtualModelProfiles?: VirtualModelProfileConfig[];
|
||||
@@ -862,6 +915,14 @@ export function AddProfileForm({
|
||||
</Field>
|
||||
</>
|
||||
)}
|
||||
{draft.agent === "claude-code" && contextArchive && onChangeContextArchive ? (
|
||||
<div className="sm:col-span-2">
|
||||
<ClaudeCodeContextArchiveCompactSetting
|
||||
contextArchive={contextArchive}
|
||||
onChange={onChangeContextArchive}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
{draft.surface !== "cli" ? (
|
||||
<div className="sm:col-span-2">
|
||||
<BotGatewaySelectForm botConfigs={botConfigs} draft={draft} onChange={onChange} onCreateBot={onCreateBot} />
|
||||
@@ -1186,10 +1247,12 @@ function handoffTargetMatchesSavedValue(target: BotHandoffScanTarget, savedValue
|
||||
export function AddProfileDialog({
|
||||
botConfigs,
|
||||
canSubmit,
|
||||
contextArchive,
|
||||
draft,
|
||||
error,
|
||||
mode = "add",
|
||||
onChange,
|
||||
onChangeContextArchive,
|
||||
onCreateBot,
|
||||
onClose,
|
||||
providers,
|
||||
@@ -1199,10 +1262,12 @@ export function AddProfileDialog({
|
||||
}: {
|
||||
botConfigs: BotGatewaySavedConfig[];
|
||||
canSubmit: boolean;
|
||||
contextArchive: AppConfig["contextArchive"];
|
||||
draft: AddProfileDraft;
|
||||
error: string;
|
||||
mode?: "add" | "edit";
|
||||
onChange: (patch: Partial<AddProfileDraft>) => void;
|
||||
onChangeContextArchive: (patch: Partial<AppConfig["contextArchive"]>) => void;
|
||||
onCreateBot: () => void;
|
||||
onClose: () => void;
|
||||
providers: GatewayProviderConfig[];
|
||||
@@ -1221,7 +1286,17 @@ export function AddProfileDialog({
|
||||
</div>
|
||||
</DialogHeader>
|
||||
<DialogBody>
|
||||
<AddProfileForm botConfigs={botConfigs} draft={draft} error={error} onChange={onChange} onCreateBot={onCreateBot} providers={providers} virtualModelProfiles={virtualModelProfiles} />
|
||||
<AddProfileForm
|
||||
botConfigs={botConfigs}
|
||||
contextArchive={contextArchive}
|
||||
draft={draft}
|
||||
error={error}
|
||||
onChange={onChange}
|
||||
onChangeContextArchive={onChangeContextArchive}
|
||||
onCreateBot={onCreateBot}
|
||||
providers={providers}
|
||||
virtualModelProfiles={virtualModelProfiles}
|
||||
/>
|
||||
</DialogBody>
|
||||
<DialogFooter>
|
||||
<div className="flex justify-end gap-2">
|
||||
|
||||
@@ -402,6 +402,7 @@ export function normalizeConfig(config: AppConfig): AppConfig {
|
||||
},
|
||||
botConfigs: normalizeBotGatewaySavedConfigs(config.botConfigs),
|
||||
botGateway: normalizeBotGatewayRuntimeConfig(config.botGateway) ?? fallbackConfig.botGateway,
|
||||
contextArchive: normalizeContextArchiveConfig(config.contextArchive),
|
||||
gateway: {
|
||||
...fallbackConfig.gateway,
|
||||
...(config.gateway || {}),
|
||||
@@ -445,6 +446,28 @@ export function normalizeConfig(config: AppConfig): AppConfig {
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeContextArchiveConfig(config: Partial<AppConfig["contextArchive"]> | undefined): AppConfig["contextArchive"] {
|
||||
return {
|
||||
...fallbackConfig.contextArchive,
|
||||
...(config || {}),
|
||||
claudeCodeCompact: Boolean(config?.claudeCodeCompact),
|
||||
enabled: Boolean(config?.enabled),
|
||||
mcpEnabled: config?.mcpEnabled !== false,
|
||||
llm: {
|
||||
...fallbackConfig.contextArchive.llm,
|
||||
...(config?.llm || {}),
|
||||
apiKey: typeof config?.llm?.apiKey === "string" ? config.llm.apiKey : "",
|
||||
baseUrl: typeof config?.llm?.baseUrl === "string" && config.llm.baseUrl.trim()
|
||||
? config.llm.baseUrl.trim()
|
||||
: fallbackConfig.contextArchive.llm.baseUrl,
|
||||
model: typeof config?.llm?.model === "string" ? config.llm.model.trim() : "",
|
||||
timeoutMs: typeof config?.llm?.timeoutMs === "number" && Number.isFinite(config.llm.timeoutMs)
|
||||
? Math.min(Math.max(Math.floor(config.llm.timeoutMs), 8000), 600000)
|
||||
: fallbackConfig.contextArchive.llm.timeoutMs
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeObservabilityConfig(config: Partial<AppConfig["observability"]> | undefined): AppConfig["observability"] {
|
||||
return {
|
||||
...fallbackConfig.observability,
|
||||
|
||||
@@ -217,6 +217,9 @@ export const appCopy: Record<ResolvedLanguage, AppCopy> = {
|
||||
"时间": "Time",
|
||||
"状态": "Status",
|
||||
"模型": "Model",
|
||||
"CCR compact for Claude Code": "CCR compact for Claude Code",
|
||||
"Use CCR context archive when Claude Code runs /compact.": "Use CCR context archive when Claude Code runs /compact.",
|
||||
"Context archive and MCP access will be enabled for this compact mode.": "Context archive and MCP access will be enabled for this compact mode.",
|
||||
"Stream": "Stream",
|
||||
"Streaming": "Streaming",
|
||||
"Non-streaming": "Non-streaming",
|
||||
@@ -573,6 +576,9 @@ export const appCopy: Record<ResolvedLanguage, AppCopy> = {
|
||||
"Agent": "Agent",
|
||||
"A provider is required before profiles can route traffic.": "需要先配置供应商，配置档案才能路由请求。",
|
||||
"Add or verify a model provider.": "添加或确认模型供应商。",
|
||||
"CCR compact for Claude Code": "Claude Code 使用 CCR 压缩",
|
||||
"Use CCR context archive when Claude Code runs /compact.": "Claude Code 执行 /compact 时改用 CCR 上下文归档。",
|
||||
"Context archive and MCP access will be enabled for this compact mode.": "将为此压缩模式启用上下文归档和 MCP 访问。",
|
||||
"Agent Analysis": "Agent 分析",
|
||||
"Agent access": "Agent 接入",
|
||||
"Agent Mix": "Agent 分布",
|
||||
|
||||
@@ -139,7 +139,11 @@ test("context archive adapts Claude Code compact requests without pruning messag
|
||||
}
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code."
|
||||
mcp_servers: [{ name: "filesystem" }],
|
||||
parallel_tool_calls: true,
|
||||
system: "You are Claude Code.",
|
||||
tool_choice: { type: "auto" },
|
||||
tools: [{ input_schema: { type: "object" }, name: "Write", type: "custom" }]
|
||||
};
|
||||
|
||||
const result = await prepareContextArchiveRequest({
|
||||
@@ -156,6 +160,10 @@ test("context archive adapts Claude Code compact requests without pruning messag
|
||||
assert.match(result.diagnostic, /^client-compact:claude-code:claude-s1:/);
|
||||
const prepared = JSON.parse(result.body.toString("utf8"));
|
||||
assert.equal(prepared.messages.length, body.messages.length);
|
||||
assert.equal(prepared.tools.length, body.tools.length);
|
||||
assert.deepEqual(prepared.tool_choice, body.tool_choice);
|
||||
assert.deepEqual(prepared.mcp_servers, body.mcp_servers);
|
||||
assert.equal(prepared.parallel_tool_calls, true);
|
||||
assert.match(prepared.system, /Archived history access/);
|
||||
assert.match(prepared.system, /ccr_history_search/);
|
||||
assert.match(prepared.system, /claude-s1/);
|
||||
@@ -167,6 +175,227 @@ test("context archive adapts Claude Code compact requests without pruning messag
|
||||
assert.match(search.answer, /npm run test:main/);
|
||||
});
|
||||
|
||||
test("context archive can replace Claude Code compact with CCR handoff and history search", async () => {
|
||||
contextArchiveService.clear();
|
||||
const config = testConfig({ claudeCodeCompact: true, retainRecentItems: 2, triggerTokenLimit: 999999 });
|
||||
const body = {
|
||||
messages: [
|
||||
{
|
||||
content: "Deep historical decision: use PostgreSQL for durable context archive storage.",
|
||||
role: "user"
|
||||
},
|
||||
{
|
||||
content: "Recent progress: added the Claude Code compact switch.",
|
||||
role: "assistant"
|
||||
},
|
||||
{
|
||||
content: "Summarize the conversation so far for handoff into a new context window.",
|
||||
role: "user"
|
||||
}
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
mcp_servers: [{ name: "filesystem" }],
|
||||
parallel_tool_calls: true,
|
||||
system: "You are Claude Code.",
|
||||
tool_choice: { type: "auto" },
|
||||
tools: [{ input_schema: { type: "object" }, name: "Write", type: "custom" }]
|
||||
};
|
||||
|
||||
const result = await prepareContextArchiveRequest({
|
||||
body: Buffer.from(JSON.stringify(body)),
|
||||
config,
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": "claude-s3" },
|
||||
method: "POST",
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
requestId: "request-claude-compact-ccr"
|
||||
});
|
||||
|
||||
assert.ok(result);
|
||||
assert.match(result.diagnostic, /^client-compact-ccr:claude-code:claude-s3:/);
|
||||
const prepared = JSON.parse(result.body.toString("utf8"));
|
||||
assert.equal(prepared.messages.length, 2);
|
||||
assert.equal(JSON.stringify(prepared.messages).includes("Deep historical decision"), false);
|
||||
assert.equal("mcp_servers" in prepared, false);
|
||||
assert.equal("parallel_tool_calls" in prepared, false);
|
||||
assert.equal("tool_choice" in prepared, false);
|
||||
assert.equal("tools" in prepared, false);
|
||||
assert.match(prepared.messages.at(-1).content, /Do not create, edit, or write files/);
|
||||
assert.match(prepared.messages.at(-1).content, /Do not call tools/);
|
||||
assert.match(prepared.system, /CCR detected this as a Claude Code context compaction request/);
|
||||
assert.match(prepared.system, /ccr_history_search/);
|
||||
assert.match(prepared.system, /claude-s3/);
|
||||
|
||||
const search = await contextArchiveService.search({
|
||||
prompt: "Which durable context archive storage was chosen?",
|
||||
sessionId: "claude-s3"
|
||||
}, config.contextArchive);
|
||||
assert.match(search.answer, /PostgreSQL/);
|
||||
});
|
||||
|
||||
test("context archive replacement trims dangling Claude Code tool tails before compacting", async () => {
|
||||
contextArchiveService.clear();
|
||||
const config = testConfig({ claudeCodeCompact: true, retainRecentItems: 12, triggerTokenLimit: 999999 });
|
||||
const body = {
|
||||
messages: [
|
||||
{
|
||||
content: "Recent implementation context: inspect the gateway compact path.",
|
||||
role: "user"
|
||||
},
|
||||
{
|
||||
content: [
|
||||
{ text: "I will inspect the files.", type: "text" },
|
||||
{ id: "call_1", input: { command: "rg compact" }, name: "Bash", type: "tool_use" }
|
||||
],
|
||||
role: "assistant"
|
||||
},
|
||||
{
|
||||
content: [
|
||||
{
|
||||
cache_control: { type: "ephemeral" },
|
||||
content: "packages/core/src/gateway/context-archive.ts: compactBody",
|
||||
tool_use_id: "call_1",
|
||||
type: "tool_result"
|
||||
}
|
||||
],
|
||||
role: "user"
|
||||
}
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code.",
|
||||
tool_choice: { type: "auto" },
|
||||
tools: [{ input_schema: { type: "object" }, name: "Bash", type: "custom" }]
|
||||
};
|
||||
|
||||
const result = await prepareContextArchiveRequest({
|
||||
body: Buffer.from(JSON.stringify(body)),
|
||||
config,
|
||||
headers: {
|
||||
"user-agent": "claude-code/2.0",
|
||||
"x-claude-code-session-id": "claude-tool-tail",
|
||||
"x-ccr-context-compact": "compact"
|
||||
},
|
||||
method: "POST",
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
requestId: "request-claude-tool-tail"
|
||||
});
|
||||
|
||||
assert.ok(result);
|
||||
assert.match(result.diagnostic, /^client-compact-ccr:claude-code:claude-tool-tail:/);
|
||||
const prepared = JSON.parse(result.body.toString("utf8"));
|
||||
assert.equal(prepared.messages.at(-1).role, "user");
|
||||
assert.match(prepared.messages.at(-1).content, /plain assistant message text/);
|
||||
assert.equal("tool_choice" in prepared, false);
|
||||
assert.equal("tools" in prepared, false);
|
||||
assert.equal(JSON.stringify(prepared.messages).includes("tool_result"), false);
|
||||
assert.match(prepared.system, /packages\/core\/src\/gateway\/context-archive\.ts/);
|
||||
});
|
||||
|
||||
test("context archive detects Claude Code compact context-management edits", async () => {
|
||||
contextArchiveService.clear();
|
||||
const config = testConfig({ claudeCodeCompact: true, triggerTokenLimit: 999999 });
|
||||
const body = {
|
||||
context_management: {
|
||||
edits: [{ type: "compact_20260112" }]
|
||||
},
|
||||
messages: [
|
||||
{ content: "Recent work: keep CCR compact replacement enabled for slash compact.", role: "user" }
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code."
|
||||
};
|
||||
|
||||
const result = await prepareContextArchiveRequest({
|
||||
body: Buffer.from(JSON.stringify(body)),
|
||||
config,
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": "claude-struct-compact" },
|
||||
method: "POST",
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
requestId: "request-claude-struct-compact"
|
||||
});
|
||||
|
||||
assert.ok(result);
|
||||
assert.match(result.diagnostic, /^client-compact-ccr:claude-code:claude-struct-compact:/);
|
||||
});
|
||||
|
||||
test("context archive ignores non-compact context-management edits", async () => {
|
||||
contextArchiveService.clear();
|
||||
const config = testConfig({ claudeCodeCompact: true, triggerTokenLimit: 999999 });
|
||||
const body = {
|
||||
context_management: {
|
||||
edits: [{ keep: "all", type: "clear_thinking_20251015" }]
|
||||
},
|
||||
messages: [
|
||||
{ content: "普通请求,不应该被当成 slash compact。", role: "user" }
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code."
|
||||
};
|
||||
|
||||
const result = await prepareContextArchiveRequest({
|
||||
body: Buffer.from(JSON.stringify(body)),
|
||||
config,
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": "claude-clear-thinking" },
|
||||
method: "POST",
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
requestId: "request-claude-clear-thinking"
|
||||
});
|
||||
|
||||
assert.ok(result);
|
||||
assert.match(result.diagnostic, /^archived:claude-clear-thinking:/);
|
||||
assert.deepEqual(JSON.parse(result.body.toString("utf8")), body);
|
||||
});
|
||||
|
||||
test("context archive does not re-trigger Claude Code compact from existing CCR summary", async () => {
|
||||
contextArchiveService.clear();
|
||||
const config = testConfig({ claudeCodeCompact: true, triggerTokenLimit: 999999 });
|
||||
const body = {
|
||||
context_management: {
|
||||
edits: [{ keep: "all", type: "clear_thinking_20251015" }]
|
||||
},
|
||||
messages: [
|
||||
{
|
||||
content: [
|
||||
{
|
||||
text: [
|
||||
"该项目是 Claude Code 的核心代码库。",
|
||||
"Archived history access:",
|
||||
"- Archive session id: claude-existing-summary",
|
||||
"- Tool call: ccr_history_search({ \"prompt\": \"specific historical detail to recover\", \"deep\": false, \"session_id\": \"claude-existing-summary\" })",
|
||||
"When you produce the compacted summary for the next context window, include this section."
|
||||
].join("\n"),
|
||||
type: "text"
|
||||
},
|
||||
{
|
||||
text: "现在始终会有压缩的信息",
|
||||
type: "text"
|
||||
}
|
||||
],
|
||||
role: "user"
|
||||
}
|
||||
],
|
||||
model: "claude-sonnet-4-5",
|
||||
system: "You are Claude Code."
|
||||
};
|
||||
|
||||
const result = await prepareContextArchiveRequest({
|
||||
body: Buffer.from(JSON.stringify(body)),
|
||||
config,
|
||||
headers: { "user-agent": "claude-code/2.0", "x-claude-code-session-id": "claude-existing-summary" },
|
||||
method: "POST",
|
||||
path: "/v1/messages",
|
||||
protocol: "anthropic_messages",
|
||||
requestId: "request-claude-existing-summary"
|
||||
});
|
||||
|
||||
assert.ok(result);
|
||||
assert.match(result.diagnostic, /^archived:claude-existing-summary:/);
|
||||
assert.deepEqual(JSON.parse(result.body.toString("utf8")), body);
|
||||
});
|
||||
|
||||
test("context archive does not treat generic summary prompts as client compact requests", async () => {
|
||||
contextArchiveService.clear();
|
||||
const config = testConfig({ triggerTokenLimit: 999999 });
|
||||
@@ -197,6 +426,7 @@ test("context archive does not treat unrelated Claude Code compact wording as co
|
||||
const config = testConfig({ triggerTokenLimit: 999999 });
|
||||
const body = {
|
||||
messages: [
|
||||
{ content: "We are benchmarking context archive compression efficiency and retrieval quality.", role: "assistant" },
|
||||
{ content: "Please set the UI density option to compact.", role: "user" }
|
||||
],
|
||||
model: "claude-sonnet-4-5"
|
||||
|
||||
@@ -85,6 +85,10 @@ test("profile service overwrites generated bin files without creating backups",
|
||||
}
|
||||
]
|
||||
};
|
||||
config.contextArchive = {
|
||||
...config.contextArchive,
|
||||
enabled: true
|
||||
};
|
||||
config.APIKEY = "ccr-profile-test";
|
||||
config.APIKEYS = [
|
||||
{
|
||||
@@ -118,9 +122,13 @@ test("profile service overwrites generated bin files without creating backups",
|
||||
const toolHubMcpConfigFile = path.join(CONFIGDIR, "profiles", profileId, "claude", "toolhub-mcp.json");
|
||||
const toolHubMcpConfig = JSON.parse(readFileSync(toolHubMcpConfigFile, "utf8"));
|
||||
const toolHubMcpServerEnv = toolHubMcpConfig.mcpServers["ccr-toolhub"].env;
|
||||
const contextArchiveMcpServer = toolHubMcpConfig.mcpServers["ccr-context-archive"];
|
||||
assert.equal(toolHubMcpServerEnv.TOOLHUB_OPENAI_API_KEY, "ccr-profile-test");
|
||||
assert.equal(toolHubMcpServerEnv.TOOLHUB_OPENAI_BASE_URL, `http://127.0.0.1:${config.gateway.port}/v1`);
|
||||
assert.equal(toolHubMcpServerEnv.TOOLHUB_OPENAI_MODEL, "Provider/model");
|
||||
assert.equal(contextArchiveMcpServer.type, "http");
|
||||
assert.equal(contextArchiveMcpServer.url, `http://127.0.0.1:${config.gateway.port}/__ccr/context-archive/mcp`);
|
||||
assert.equal(contextArchiveMcpServer.headers.Authorization, "Bearer ccr-profile-test");
|
||||
const backupEntries = readdirSync(binDir).filter((entry) =>
|
||||
(
|
||||
entry.startsWith(`ccr-claude-code-api-key-${profileId}`) ||
|
||||
@@ -209,6 +217,58 @@ test("profile service injects ToolHub MCP into Codex config", { skip: !process.e
|
||||
assert.match(content, /TOOLHUB_OPENAI_MODEL = "Provider\/model"/);
|
||||
});
|
||||
|
||||
test("profile service injects Context Archive MCP for Claude Code without ToolHub", { skip: !process.env.CCR_INTERNAL_HOME_DIR }, async () => {
|
||||
const profileId = "context-archive-mcp-only";
|
||||
const config = createDefaultAppConfig({
|
||||
generatedConfigFile: path.join(CONFIGDIR, "gateway.config.json")
|
||||
});
|
||||
config.Providers = [
|
||||
{
|
||||
api_base_url: "https://example.test/v1",
|
||||
api_key: "provider-key",
|
||||
models: ["model"],
|
||||
name: "Provider"
|
||||
}
|
||||
];
|
||||
config.preferredProvider = "Provider";
|
||||
config.contextArchive = {
|
||||
...config.contextArchive,
|
||||
enabled: true
|
||||
};
|
||||
config.APIKEY = "ccr-context-archive-profile-test";
|
||||
config.APIKEYS = [
|
||||
{
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
id: `profile:${profileId}`,
|
||||
key: "ccr-context-archive-profile-test",
|
||||
name: "Profile: Context Archive MCP Test"
|
||||
}
|
||||
];
|
||||
config.profile.profiles = [
|
||||
{
|
||||
agent: "claude-code",
|
||||
enabled: true,
|
||||
env: {},
|
||||
id: profileId,
|
||||
model: "Provider/model",
|
||||
name: "Context Archive MCP Test",
|
||||
scope: "ccr",
|
||||
settingsFile: "~/.claude/settings.json",
|
||||
smallFastModel: "",
|
||||
surface: "auto"
|
||||
}
|
||||
];
|
||||
|
||||
const result = await applyProfileConfig(config);
|
||||
assert.equal(result.clients.length, 1);
|
||||
assert.equal(result.clients[0].ok, true);
|
||||
const mcpConfigFile = path.join(CONFIGDIR, "profiles", profileId, "claude", "toolhub-mcp.json");
|
||||
const mcpConfig = JSON.parse(readFileSync(mcpConfigFile, "utf8"));
|
||||
assert.deepEqual(Object.keys(mcpConfig.mcpServers), ["ccr-context-archive"]);
|
||||
assert.equal(mcpConfig.mcpServers["ccr-context-archive"].type, "http");
|
||||
assert.equal(mcpConfig.mcpServers["ccr-context-archive"].headers.Authorization, "Bearer ccr-context-archive-profile-test");
|
||||
});
|
||||
|
||||
test("profile service clears stale Claude Code ToolHub artifacts when no gateway models are available", { skip: !process.env.CCR_INTERNAL_HOME_DIR }, async () => {
|
||||
const profileId = "stale-toolhub-no-models";
|
||||
const settingsFile = path.join(CONFIGDIR, "profiles", profileId, "claude", "settings.json");
|
||||
|
||||
Reference in New Issue
Block a user