From 944ab18304ff4b26574daa546ef745b2c6337efe Mon Sep 17 00:00:00 2001 From: chrarnoldus <12196001+chrarnoldus@users.noreply.github.com> Date: Sat, 8 Aug 2026 18:17:49 +0000 Subject: [PATCH] fix(llm): place prompt cache breakpoint before trailing environment details Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com> --- packages/llm/src/cache-policy.ts | 21 +++++++-- packages/llm/test/cache-policy.test.ts | 46 +++++++++++++++++++ packages/opencode/src/provider/transform.ts | 28 ++++++++--- .../opencode/test/provider/transform.test.ts | 35 ++++++++++++++ 4 files changed, 120 insertions(+), 10 deletions(-) diff --git a/packages/llm/src/cache-policy.ts b/packages/llm/src/cache-policy.ts index e1caf0d67e5..96cd7b38bdb 100644 --- a/packages/llm/src/cache-policy.ts +++ b/packages/llm/src/cache-policy.ts @@ -66,15 +66,28 @@ const markLastSystem = (system: LLMRequest["system"], hint: CacheHint): LLMReque const lastIndexOfRole = (messages: ReadonlyArray, role: Message["role"]): number => messages.findLastIndex((m) => m.role === role) -// Mark the last text part of `messages[index]`. If no text part exists, mark -// the last content part regardless of type — that's the breakpoint position -// in tool-result-only messages too. +// kilocode_change start - mark the last stable text part of `messages[index]` (skipping trailing ephemeral details) +const isEphemeralPart = (part: ContentPart): boolean => { + if (part.type === "text" && (part.text.startsWith("") || part.text.includes(""))) + return true + if ("metadata" in part && part.metadata && Boolean((part.metadata as Record).synthetic)) + return true + return false +} + const markMessageAt = (messages: ReadonlyArray, index: number, hint: CacheHint): ReadonlyArray => { if (index < 0 || index >= messages.length) return messages const target = messages[index]! if (target.content.length === 0) return messages + const lastStableTextIndex = target.content.findLastIndex((part) => part.type === "text" && !isEphemeralPart(part)) const lastTextIndex = target.content.findLastIndex((part) => part.type === "text") - const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1 + const markAt = + lastStableTextIndex >= 0 + ? lastStableTextIndex + : lastTextIndex >= 0 + ? lastTextIndex + : target.content.length - 1 + // kilocode_change end const existing = target.content[markAt]! if ("cache" in existing && existing.cache) return messages const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part)) diff --git a/packages/llm/test/cache-policy.test.ts b/packages/llm/test/cache-policy.test.ts index 91a6712d108..b0f3256e988 100644 --- a/packages/llm/test/cache-policy.test.ts +++ b/packages/llm/test/cache-policy.test.ts @@ -176,6 +176,52 @@ describe("applyCachePolicy", () => { }) }), ) + + it.effect("places prompt_cache_breakpoint BEFORE trailing part", () => + Effect.gen(function* () { + const envBlock = "\nCurrent time: 2026-08-08T18:00:00+00:00\n" + const prepared = yield* LLMClient.prepare( + LLM.request({ + model: openaiGpt56ResponsesModel, + system: "System instructions", + messages: [ + Message.user([ + { type: "text", text: "Please inspect the codebase" }, + { type: "text", text: envBlock }, + ]), + ], + cache: "auto", + }), + ) + + expect(prepared.body).toMatchObject({ + input: [ + { + role: "system", + content: [{ type: "input_text", text: "System instructions", prompt_cache_breakpoint: { mode: "explicit" } }], + }, + { + role: "user", + content: [ + { + type: "input_text", + text: "Please inspect the codebase", + prompt_cache_breakpoint: { mode: "explicit" }, + }, + { + type: "input_text", + text: envBlock, + }, + ], + }, + ], + }) + + const userContent = (prepared.body as any).input[1].content + expect(userContent[0].prompt_cache_breakpoint).toEqual({ mode: "explicit" }) + expect(userContent[1].prompt_cache_breakpoint).toBeUndefined() + }), + ) // kilocode_change end it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () => diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts index dbe3f58da25..f0173b50303 100644 --- a/packages/opencode/src/provider/transform.ts +++ b/packages/opencode/src/provider/transform.ts @@ -371,20 +371,36 @@ function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage model.providerID === "anthropic" || model.providerID.includes("bedrock") || model.api.npm === "@ai-sdk/amazon-bedrock" + // kilocode_change start - place caching breakpoint on stable content before trailing const shouldUseContentOptions = !useMessageLevelOptions && Array.isArray(msg.content) && msg.content.length > 0 if (shouldUseContentOptions) { - const lastContent = msg.content[msg.content.length - 1] + const parts = msg.content as any[] + const targetIndex = parts.findLastIndex( + (part) => + part && + typeof part === "object" && + part.type !== "tool-approval-request" && + part.type !== "tool-approval-response" && + !( + part.type === "text" && + typeof part.text === "string" && + (part.text.startsWith("") || part.text.includes("")) + ) && + !part.synthetic, + ) + const targetContent = targetIndex >= 0 ? parts[targetIndex] : parts[parts.length - 1] if ( - lastContent && - typeof lastContent === "object" && - lastContent.type !== "tool-approval-request" && - lastContent.type !== "tool-approval-response" + targetContent && + typeof targetContent === "object" && + targetContent.type !== "tool-approval-request" && + targetContent.type !== "tool-approval-response" ) { - lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, providerOptions) + targetContent.providerOptions = mergeDeep(targetContent.providerOptions ?? {}, providerOptions) continue } } + // kilocode_change end msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, providerOptions) } diff --git a/packages/opencode/test/provider/transform.test.ts b/packages/opencode/test/provider/transform.test.ts index 14542cdaa41..2f8c7d23b18 100644 --- a/packages/opencode/test/provider/transform.test.ts +++ b/packages/opencode/test/provider/transform.test.ts @@ -3092,6 +3092,41 @@ describe("ProviderTransform.message - cache control on gateway", () => { expect(result[0].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined() expect(result[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined() }) + + test("openai gpt-5.6 places promptCacheBreakpoint before trailing environment_details", () => { + const model = createModel({ + providerID: "openai", + api: { + id: "gpt-5.6", + url: "https://api.openai.com/v1", + npm: "@ai-sdk/openai", + }, + id: "gpt-5.6", + }) + const envBlock = "\nCurrent time: 2026-08-08T18:00:00+00:00\n" + const msgs = [ + { + role: "system", + content: "You are a helpful assistant", + }, + { + role: "user", + content: [ + { type: "text", text: "Please review the changes" }, + { type: "text", text: envBlock }, + ], + }, + ] as any[] + + const result = ProviderTransform.message(msgs, model, {}) as any[] + + expect(result[1].content[0].providerOptions.openai).toEqual({ + promptCacheBreakpoint: { + mode: "explicit", + }, + }) + expect(result[1].content[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined() + }) // kilocode_change end })