mirror of
https://github.com/Kilo-Org/kilocode.git
synced 2026-10-29 18:52:45 +08:00
fix(llm): place prompt cache breakpoint before trailing environment details
Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
kiloconnect[bot]
parent
435c029f8c
commit
944ab18304
@@ -66,15 +66,28 @@ const markLastSystem = (system: LLMRequest["system"], hint: CacheHint): LLMReque
|
||||
const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number =>
|
||||
messages.findLastIndex((m) => m.role === role)
|
||||
|
||||
// Mark the last text part of `messages[index]`. If no text part exists, mark
|
||||
// the last content part regardless of type — that's the breakpoint position
|
||||
// in tool-result-only messages too.
|
||||
// kilocode_change start - mark the last stable text part of `messages[index]` (skipping trailing ephemeral details)
|
||||
const isEphemeralPart = (part: ContentPart): boolean => {
|
||||
if (part.type === "text" && (part.text.startsWith("<environment_details>") || part.text.includes("<environment_details>")))
|
||||
return true
|
||||
if ("metadata" in part && part.metadata && Boolean((part.metadata as Record<string, unknown>).synthetic))
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
const markMessageAt = (messages: ReadonlyArray<Message>, index: number, hint: CacheHint): ReadonlyArray<Message> => {
|
||||
if (index < 0 || index >= messages.length) return messages
|
||||
const target = messages[index]!
|
||||
if (target.content.length === 0) return messages
|
||||
const lastStableTextIndex = target.content.findLastIndex((part) => part.type === "text" && !isEphemeralPart(part))
|
||||
const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
|
||||
const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
|
||||
const markAt =
|
||||
lastStableTextIndex >= 0
|
||||
? lastStableTextIndex
|
||||
: lastTextIndex >= 0
|
||||
? lastTextIndex
|
||||
: target.content.length - 1
|
||||
// kilocode_change end
|
||||
const existing = target.content[markAt]!
|
||||
if ("cache" in existing && existing.cache) return messages
|
||||
const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
|
||||
|
||||
@@ -176,6 +176,52 @@ describe("applyCachePolicy", () => {
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("places prompt_cache_breakpoint BEFORE trailing <environment_details> part", () =>
|
||||
Effect.gen(function* () {
|
||||
const envBlock = "<environment_details>\nCurrent time: 2026-08-08T18:00:00+00:00\n</environment_details>"
|
||||
const prepared = yield* LLMClient.prepare(
|
||||
LLM.request({
|
||||
model: openaiGpt56ResponsesModel,
|
||||
system: "System instructions",
|
||||
messages: [
|
||||
Message.user([
|
||||
{ type: "text", text: "Please inspect the codebase" },
|
||||
{ type: "text", text: envBlock },
|
||||
]),
|
||||
],
|
||||
cache: "auto",
|
||||
}),
|
||||
)
|
||||
|
||||
expect(prepared.body).toMatchObject({
|
||||
input: [
|
||||
{
|
||||
role: "system",
|
||||
content: [{ type: "input_text", text: "System instructions", prompt_cache_breakpoint: { mode: "explicit" } }],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: "Please inspect the codebase",
|
||||
prompt_cache_breakpoint: { mode: "explicit" },
|
||||
},
|
||||
{
|
||||
type: "input_text",
|
||||
text: envBlock,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const userContent = (prepared.body as any).input[1].content
|
||||
expect(userContent[0].prompt_cache_breakpoint).toEqual({ mode: "explicit" })
|
||||
expect(userContent[1].prompt_cache_breakpoint).toBeUndefined()
|
||||
}),
|
||||
)
|
||||
// kilocode_change end
|
||||
|
||||
it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
|
||||
|
||||
@@ -371,20 +371,36 @@ function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage
|
||||
model.providerID === "anthropic" ||
|
||||
model.providerID.includes("bedrock") ||
|
||||
model.api.npm === "@ai-sdk/amazon-bedrock"
|
||||
// kilocode_change start - place caching breakpoint on stable content before trailing <environment_details>
|
||||
const shouldUseContentOptions = !useMessageLevelOptions && Array.isArray(msg.content) && msg.content.length > 0
|
||||
|
||||
if (shouldUseContentOptions) {
|
||||
const lastContent = msg.content[msg.content.length - 1]
|
||||
const parts = msg.content as any[]
|
||||
const targetIndex = parts.findLastIndex(
|
||||
(part) =>
|
||||
part &&
|
||||
typeof part === "object" &&
|
||||
part.type !== "tool-approval-request" &&
|
||||
part.type !== "tool-approval-response" &&
|
||||
!(
|
||||
part.type === "text" &&
|
||||
typeof part.text === "string" &&
|
||||
(part.text.startsWith("<environment_details>") || part.text.includes("<environment_details>"))
|
||||
) &&
|
||||
!part.synthetic,
|
||||
)
|
||||
const targetContent = targetIndex >= 0 ? parts[targetIndex] : parts[parts.length - 1]
|
||||
if (
|
||||
lastContent &&
|
||||
typeof lastContent === "object" &&
|
||||
lastContent.type !== "tool-approval-request" &&
|
||||
lastContent.type !== "tool-approval-response"
|
||||
targetContent &&
|
||||
typeof targetContent === "object" &&
|
||||
targetContent.type !== "tool-approval-request" &&
|
||||
targetContent.type !== "tool-approval-response"
|
||||
) {
|
||||
lastContent.providerOptions = mergeDeep(lastContent.providerOptions ?? {}, providerOptions)
|
||||
targetContent.providerOptions = mergeDeep(targetContent.providerOptions ?? {}, providerOptions)
|
||||
continue
|
||||
}
|
||||
}
|
||||
// kilocode_change end
|
||||
|
||||
msg.providerOptions = mergeDeep(msg.providerOptions ?? {}, providerOptions)
|
||||
}
|
||||
|
||||
@@ -3092,6 +3092,41 @@ describe("ProviderTransform.message - cache control on gateway", () => {
|
||||
expect(result[0].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
|
||||
expect(result[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
|
||||
})
|
||||
|
||||
test("openai gpt-5.6 places promptCacheBreakpoint before trailing environment_details", () => {
|
||||
const model = createModel({
|
||||
providerID: "openai",
|
||||
api: {
|
||||
id: "gpt-5.6",
|
||||
url: "https://api.openai.com/v1",
|
||||
npm: "@ai-sdk/openai",
|
||||
},
|
||||
id: "gpt-5.6",
|
||||
})
|
||||
const envBlock = "<environment_details>\nCurrent time: 2026-08-08T18:00:00+00:00\n</environment_details>"
|
||||
const msgs = [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are a helpful assistant",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "Please review the changes" },
|
||||
{ type: "text", text: envBlock },
|
||||
],
|
||||
},
|
||||
] as any[]
|
||||
|
||||
const result = ProviderTransform.message(msgs, model, {}) as any[]
|
||||
|
||||
expect(result[1].content[0].providerOptions.openai).toEqual({
|
||||
promptCacheBreakpoint: {
|
||||
mode: "explicit",
|
||||
},
|
||||
})
|
||||
expect(result[1].content[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
|
||||
})
|
||||
// kilocode_change end
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user