fix(llm): place prompt cache breakpoint before trailing environment details

Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
This commit is contained in:
chrarnoldus
2026-08-08 18:17:49 +00:00
co-authored by kiloconnect[bot]
parent 435c029f8c
commit 944ab18304
4 changed files with 120 additions and 10 deletions
+17 -4
View File
@@ -66,15 +66,28 @@ const markLastSystem = (system: LLMRequest["system"], hint: CacheHint): LLMReque
const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number =>
messages.findLastIndex((m) => m.role === role)
// Mark the last text part of `messages[index]`. If no text part exists, mark
// the last content part regardless of type — that's the breakpoint position
// in tool-result-only messages too.
// kilocode_change start - mark the last stable text part of `messages[index]` (skipping trailing ephemeral details)
const isEphemeralPart = (part: ContentPart): boolean => {
if (part.type === "text" && (part.text.startsWith("<environment_details>") || part.text.includes("<environment_details>")))
return true
if ("metadata" in part && part.metadata && Boolean((part.metadata as Record<string, unknown>).synthetic))
return true
return false
}
const markMessageAt = (messages: ReadonlyArray<Message>, index: number, hint: CacheHint): ReadonlyArray<Message> => {
if (index < 0 || index >= messages.length) return messages
const target = messages[index]!
if (target.content.length === 0) return messages
const lastStableTextIndex = target.content.findLastIndex((part) => part.type === "text" && !isEphemeralPart(part))
const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
const markAt =
lastStableTextIndex >= 0
? lastStableTextIndex
: lastTextIndex >= 0
? lastTextIndex
: target.content.length - 1
// kilocode_change end
const existing = target.content[markAt]!
if ("cache" in existing && existing.cache) return messages
const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
+46
View File
@@ -176,6 +176,52 @@ describe("applyCachePolicy", () => {
})
}),
)
it.effect("places prompt_cache_breakpoint BEFORE trailing <environment_details> part", () =>
Effect.gen(function* () {
const envBlock = "<environment_details>\nCurrent time: 2026-08-08T18:00:00+00:00\n</environment_details>"
const prepared = yield* LLMClient.prepare(
LLM.request({
model: openaiGpt56ResponsesModel,
system: "System instructions",
messages: [
Message.user([
{ type: "text", text: "Please inspect the codebase" },
{ type: "text", text: envBlock },
]),
],
cache: "auto",
}),
)
expect(prepared.body).toMatchObject({
input: [
{
role: "system",
content: [{ type: "input_text", text: "System instructions", prompt_cache_breakpoint: { mode: "explicit" } }],
},
{
role: "user",
content: [
{
type: "input_text",
text: "Please inspect the codebase",
prompt_cache_breakpoint: { mode: "explicit" },
},
{
type: "input_text",
text: envBlock,
},
],
},
],
})
const userContent = (prepared.body as any).input[1].content
expect(userContent[0].prompt_cache_breakpoint).toEqual({ mode: "explicit" })
expect(userContent[1].prompt_cache_breakpoint).toBeUndefined()
}),
)
// kilocode_change end
it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
+22 -6
View File
@@ -371,20 +371,36 @@ function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage
model.providerID === "anthropic" ||
model.providerID.includes("bedrock") ||
model.api.npm === "@ai-sdk/amazon-bedrock"
// kilocode_change start - place caching breakpoint on stable content before trailing <environment_details>
const shouldUseContentOptions = !useMessageLevelOptions && Array.isArray(msg.content) && msg.content.length > 0
if (shouldUseContentOptions) {
const lastContent = msg.content[msg.content.length - 1]
const parts = msg.content as any[]
const targetIndex = parts.findLastIndex(
(part) =>
part &&
typeof part === "object" &&
part.type !== "tool-approval-request" &&
part.type !== "tool-approval-response" &&
!(
part.type === "text" &&
typeof part.text === "string" &&
(part.text.startsWith("<environment_details>") || part.text.includes("<environment_details>"))
) &&
!part.synthetic,
)
const targetContent = targetIndex >= 0 ? parts[targetIndex] : parts[parts.length - 1]
if (
lastContent &&
typeof lastContent === "object" &&
lastContent.type !== "tool-approval-request" &&
lastContent.type !== "tool-approval-response"
targetContent &&
typeof targetContent === "object" &&
targetContent.type !== "tool-approval-request" &&
targetContent.type !== "tool-approval-response"
) {
lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, providerOptions)
targetContent.providerOptions = mergeDeep(targetContent.providerOptions ?? {}, providerOptions)
continue
}
}
// kilocode_change end
msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, providerOptions)
}
@@ -3092,6 +3092,41 @@ describe("ProviderTransform.message - cache control on gateway", () => {
expect(result[0].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
expect(result[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
})
test("openai gpt-5.6 places promptCacheBreakpoint before trailing environment_details", () => {
const model = createModel({
providerID: "openai",
api: {
id: "gpt-5.6",
url: "https://api.openai.com/v1",
npm: "@ai-sdk/openai",
},
id: "gpt-5.6",
})
const envBlock = "<environment_details>\nCurrent time: 2026-08-08T18:00:00+00:00\n</environment_details>"
const msgs = [
{
role: "system",
content: "You are a helpful assistant",
},
{
role: "user",
content: [
{ type: "text", text: "Please review the changes" },
{ type: "text", text: envBlock },
],
},
] as any[]
const result = ProviderTransform.message(msgs, model, {}) as any[]
expect(result[1].content[0].providerOptions.openai).toEqual({
promptCacheBreakpoint: {
mode: "explicit",
},
})
expect(result[1].content[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
})
// kilocode_change end
})