opensource/kilocode
3
0
Fork 0
mirror of https://github.com/Kilo-Org/kilocode.git synced 2026-09-01 15:32:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm): set explicit prompt cache breakpoints on stable prefix for GPT-5.6+

Browse Source
This commit is contained in:
chrarnoldus
2026-08-08 17:26:07 +00:00
parent 001fb21f7c
commit 9dbf276ade
7 changed files with 335 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/openai-explicit-prompt-cache-breakpoints.md
+5
View File
@@ -0,0 +1,5 @@
---
"@kilocode/cli": patch
---
Set explicit prompt cache breakpoints on stable prefixes for OpenAI GPT-5.6+ models.
packages/llm/src/cache-policy.ts
+10 -4
View File
@@ -36,10 +36,16 @@ const resolve = (policy: CachePolicy | undefined): CachePolicyObject => {
return policy
}
// Protocols whose wire format ignores inline cache markers (OpenAI's implicit
// prefix caching, Gemini's implicit + out-of-band CachedContent). Skip the
// whole policy pass for these — emitting hints would be harmless but pointless.
const RESPECTS_INLINE_HINTS = new Set(["anthropic-messages", "bedrock-converse"])
// kilocode_change start - Protocols whose wire format supports inline cache markers / explicit breakpoints.
// Gemini uses out-of-band CachedContent.
const RESPECTS_INLINE_HINTS = new Set([
"anthropic-messages",
"bedrock-converse",
"openai-responses",
"openai-chat",
"openai-compatible-chat",
])
// kilocode_change end
const makeHint = (ttlSeconds: number | undefined): CacheHint =>
ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" })
packages/llm/src/protocols/openai-chat.ts
+76 -11
View File
@@ -28,6 +28,24 @@ const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/chat/completions"
// kilocode_change start - explicit prompt cache breakpoints for GPT-5.6+
const supportsBreakpoint = (modelId: string) => {
const match = modelId.match(/gpt-(\d+)\.(\d+)/)
if (match) {
const major = Number(match[1])
const minor = Number(match[2])
if (major > 5 || (major === 5 && minor >= 6)) return true
}
const majorMatch = modelId.match(/gpt-(\d+)/)
if (majorMatch && Number(majorMatch[1]) >= 6) return true
return false
}
const OpenAIChatPromptCacheBreakpoint = Schema.Struct({
mode: Schema.Literal("explicit"),
})
// kilocode_change end
// =============================================================================
// Request Body Schema
// =============================================================================
@@ -57,15 +75,29 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
const OpenAIChatUserContent = Schema.Union([
Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
Schema.Struct({
type: Schema.Literal("text"),
text: Schema.String,
prompt_cache_breakpoint: Schema.optional(OpenAIChatPromptCacheBreakpoint), // kilocode_change
}),
Schema.Struct({
type: Schema.Literal("image_url"),
image_url: Schema.Struct({ url: Schema.String }),
prompt_cache_breakpoint: Schema.optional(OpenAIChatPromptCacheBreakpoint), // kilocode_change
}),
])
const OpenAIChatMessage = Schema.Union([
Schema.Struct({ role: Schema.Literal("system"), content: Schema.String }),
// kilocode_change start - support content block array for system/developer messages
Schema.Struct({
role: Schema.Literal("system"),
content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
}),
Schema.Struct({
role: Schema.Literal("developer"),
content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
}),
// kilocode_change end
Schema.Struct({
role: Schema.Literal("user"),
content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
@@ -210,21 +242,33 @@ const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart
const openAICompatibleReasoningContent = (native: unknown) =>
isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) {
const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (
message: OpenAIChatRequestMessage,
modelId: string, // kilocode_change
) {
const content: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
for (const part of message.content) {
// kilocode_change start
const breakpoint =
"cache" in part && part.cache && supportsBreakpoint(modelId)
? { prompt_cache_breakpoint: { mode: "explicit" as const } }
: {}
if (part.type === "text") {
content.push({ type: "text", text: part.text })
content.push({ type: "text", text: part.text, ...breakpoint })
continue
}
if (part.type === "media") {
content.push(yield* lowerMedia(part))
content.push({ ...(yield* lowerMedia(part)), ...breakpoint })
continue
}
// kilocode_change end
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text", "media"])
}
if (content.every((part) => part.type === "text"))
return { role: "user" as const, content: content.map((part) => part.text).join("\n") } // kilocode_change
// kilocode_change start
const hasBreakpoint = content.some((part) => "prompt_cache_breakpoint" in part && part.prompt_cache_breakpoint)
if (!hasBreakpoint && content.every((part) => part.type === "text"))
return { role: "user" as const, content: content.map((part) => (part as { text: string }).text).join("\n") }
// kilocode_change end
return { role: "user" as const, content }
})
@@ -284,15 +328,36 @@ const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (m
return { messages, images }
})
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (message: OpenAIChatRequestMessage) {
if (message.role === "user") return [yield* lowerUserMessage(message)]
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
message: OpenAIChatRequestMessage,
modelId: string, // kilocode_change
) {
if (message.role === "user") return [yield* lowerUserMessage(message, modelId)] // kilocode_change
if (message.role === "assistant") return [yield* lowerAssistantMessage(message)]
return (yield* lowerToolMessages(message)).messages
})
const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: LLMRequest) {
// kilocode_change start
const hasSystemCache = request.system.some((part) => part.cache) && supportsBreakpoint(request.model.id)
const system: OpenAIChatMessage[] =
request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
request.system.length === 0
? []
: hasSystemCache
? [
{
role: "system",
content: [
{
type: "text",
text: ProviderShared.joinText(request.system),
prompt_cache_breakpoint: { mode: "explicit" },
},
],
},
]
: [{ role: "system", content: ProviderShared.joinText(request.system) }]
// kilocode_change end
const messages = [...system]
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
const flushImages = () => {
@@ -324,7 +389,7 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
continue
}
flushImages()
messages.push(...(yield* lowerMessage(message)))
messages.push(...(yield* lowerMessage(message, request.model.id))) // kilocode_change
}
flushImages()
return messages
packages/llm/src/protocols/openai-responses.ts
+64 -5
View File
@@ -29,16 +29,36 @@ const ADAPTER = "openai-responses"
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/responses"
// kilocode_change start - explicit prompt cache breakpoints for GPT-5.6+
const supportsBreakpoint = (modelId: string) => {
const match = modelId.match(/gpt-(\d+)\.(\d+)/)
if (match) {
const major = Number(match[1])
const minor = Number(match[2])
if (major > 5 || (major === 5 && minor >= 6)) return true
}
const majorMatch = modelId.match(/gpt-(\d+)/)
if (majorMatch && Number(majorMatch[1]) >= 6) return true
return false
}
const OpenAIResponsesPromptCacheBreakpoint = Schema.Struct({
mode: Schema.Literal("explicit"),
})
// kilocode_change end
// =============================================================================
// Request Body Schema
// =============================================================================
const OpenAIResponsesInputText = Schema.Struct({
type: Schema.tag("input_text"),
text: Schema.String,
prompt_cache_breakpoint: Schema.optional(OpenAIResponsesPromptCacheBreakpoint), // kilocode_change
})
const OpenAIResponsesInputImage = Schema.Struct({
type: Schema.tag("input_image"),
image_url: Schema.String,
prompt_cache_breakpoint: Schema.optional(OpenAIResponsesPromptCacheBreakpoint), // kilocode_change
})
const OpenAIResponsesInputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage])
type OpenAIResponsesInputContent = Schema.Schema.Type<typeof OpenAIResponsesInputContent>
@@ -76,7 +96,16 @@ const OpenAIResponsesFunctionCallOutput = Schema.Union([
])
const OpenAIResponsesInputItem = Schema.Union([
Schema.Struct({ role: Schema.tag("system"), content: Schema.String }),
// kilocode_change start - support content block array for system/developer messages
Schema.Struct({
role: Schema.Literal("system"),
content: Schema.Union([Schema.String, Schema.Array(OpenAIResponsesInputContent)]),
}),
Schema.Struct({
role: Schema.Literal("developer"),
content: Schema.Union([Schema.String, Schema.Array(OpenAIResponsesInputContent)]),
}),
// kilocode_change end
Schema.Struct({ role: Schema.tag("user"), content: Schema.Array(OpenAIResponsesInputContent) }),
Schema.Struct({ role: Schema.tag("assistant"), content: Schema.Array(OpenAIResponsesOutputText) }),
OpenAIResponsesReasoningItem,
@@ -307,16 +336,23 @@ const hostedToolItemID = (part: ToolResultPart) => {
const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* (
part: LLMRequest["messages"][number]["content"][number],
modelId: string, // kilocode_change
) {
if (part.type === "text") return { type: "input_text" as const, text: part.text }
// kilocode_change start
const breakpoint =
"cache" in part && part.cache && supportsBreakpoint(modelId)
? { prompt_cache_breakpoint: { mode: "explicit" as const } }
: {}
if (part.type === "text") return { type: "input_text" as const, text: part.text, ...breakpoint }
if (part.type === "media") {
const media = yield* ProviderShared.validateMedia(
"OpenAI Responses",
part,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return { type: "input_image" as const, image_url: media.dataUrl }
return { type: "input_image" as const, image_url: media.dataUrl, ...breakpoint }
}
// kilocode_change end
return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
})
@@ -344,8 +380,26 @@ const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")
})
const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (request: LLMRequest) {
// kilocode_change start
const hasSystemCache = request.system.some((part) => part.cache) && supportsBreakpoint(request.model.id)
const system: OpenAIResponsesInputItem[] =
request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
request.system.length === 0
? []
: hasSystemCache
? [
{
role: "system",
content: [
{
type: "input_text",
text: ProviderShared.joinText(request.system),
prompt_cache_breakpoint: { mode: "explicit" },
},
],
},
]
: [{ role: "system", content: ProviderShared.joinText(request.system) }]
// kilocode_change end
const input: OpenAIResponsesInputItem[] = [...system]
const store = OpenAIOptions.store(request)
@@ -363,7 +417,12 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ
}
if (message.role === "user") {
input.push({ role: "user", content: yield* Effect.forEach(message.content, lowerUserContent) })
// kilocode_change start
input.push({
role: "user",
content: yield* Effect.forEach(message.content, (part) => lowerUserContent(part, request.model.id)),
})
// kilocode_change end
continue
}
packages/llm/test/cache-policy.test.ts
+82 -2
View File
@@ -6,6 +6,9 @@ import { AmazonBedrock } from "../src/providers"
import * as AnthropicMessages from "../src/protocols/anthropic-messages"
import * as Gemini from "../src/protocols/gemini"
import * as OpenAIChat from "../src/protocols/openai-chat"
// kilocode_change start
import * as OpenAIResponses from "../src/protocols/openai-responses"
// kilocode_change end
import { applyCachePolicy } from "../src/cache-policy"
import { it } from "./lib/effect"
@@ -21,6 +24,16 @@ const openaiModel = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-4o-mini" })
// kilocode_change start
const openaiGpt56ResponsesModel = OpenAIResponses.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-5.6" })
const openaiGpt56ChatModel = OpenAIChat.route
.with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
.model({ id: "gpt-5.6" })
// kilocode_change end
const geminiModel = Gemini.route
.with({
endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
@@ -79,7 +92,8 @@ describe("applyCachePolicy", () => {
}),
)
it.effect("'auto' is a no-op on OpenAI (implicit caching protocol)", () =>
// kilocode_change start
it.effect("'auto' does not emit explicit breakpoints on pre-5.6 OpenAI models", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
@@ -91,13 +105,79 @@ describe("applyCachePolicy", () => {
)
const body = prepared.body as { messages: Array<{ content: unknown }> }
// OpenAI doesn't accept cache_control on messages — policy must skip.
// Older OpenAI models reject prompt_cache_breakpoint — policy must skip.
const flat = JSON.stringify(body)
expect(flat).not.toContain("prompt_cache_breakpoint")
expect(flat).not.toContain("cache_control")
expect(flat).not.toContain("cachePoint")
}),
)
it.effect("'auto' emits prompt_cache_breakpoint on stable system prefix and latest user on GPT-5.6 Responses", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: openaiGpt56ResponsesModel,
system: "System instructions",
messages: [
Message.user("first question"),
Message.assistant("assistant reply"),
Message.user("latest question"),
],
cache: "auto",
}),
)
expect(prepared.body).toMatchObject({
input: [
{
role: "system",
content: [{ type: "input_text", text: "System instructions", prompt_cache_breakpoint: { mode: "explicit" } }],
},
{ role: "user", content: [{ type: "input_text", text: "first question" }] },
{ role: "assistant", content: [{ type: "output_text", text: "assistant reply" }] },
{
role: "user",
content: [{ type: "input_text", text: "latest question", prompt_cache_breakpoint: { mode: "explicit" } }],
},
],
})
}),
)
it.effect("'auto' emits prompt_cache_breakpoint on stable system prefix and latest user on GPT-5.6 Chat", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
LLM.request({
model: openaiGpt56ChatModel,
system: "System instructions",
messages: [
Message.user("first question"),
Message.assistant("assistant reply"),
Message.user("latest question"),
],
cache: "auto",
}),
)
expect(prepared.body).toMatchObject({
messages: [
{
role: "system",
content: [{ type: "text", text: "System instructions", prompt_cache_breakpoint: { mode: "explicit" } }],
},
{ role: "user", content: "first question" },
{ role: "assistant", content: "assistant reply" },
{
role: "user",
content: [{ type: "text", text: "latest question", prompt_cache_breakpoint: { mode: "explicit" } }],
},
],
})
}),
)
// kilocode_change end
it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
Effect.gen(function* () {
const prepared = yield* LLMClient.prepare(
packages/opencode/src/provider/transform.ts
+34 -1
View File
@@ -328,6 +328,20 @@ function normalizeMessages(
return msgs
}
// kilocode_change start - explicit prompt cache breakpoints for GPT-5.6+
function supportsOpenAICacheBreakpoint(modelId: string): boolean {
const match = modelId.match(/gpt-(\d+)\.(\d+)/)
if (match) {
const major = Number(match[1])
const minor = Number(match[2])
if (major > 5 || (major === 5 && minor >= 6)) return true
}
const majorMatch = modelId.match(/gpt-(\d+)/)
if (majorMatch && Number(majorMatch[1]) >= 6) return true
return false
}
// kilocode_change end
function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage[] {
const system = msgs.filter((msg) => msg.role === "system").slice(0, 2)
const final = msgs.filter((msg) => msg.role !== "system").slice(-2)
@@ -351,6 +365,18 @@ function applyCaching(msgs: ModelMessage[], model: Provider.Model): ModelMessage
alibaba: {
cacheControl: { type: "ephemeral" },
},
// kilocode_change start
...(supportsOpenAICacheBreakpoint(model.api.id)
? {
openai: {
promptCacheBreakpoint: { mode: "explicit" },
},
azure: {
promptCacheBreakpoint: { mode: "explicit" },
},
}
: {}),
// kilocode_change end
}
for (const msg of unique([...system, ...final])) {
@@ -438,6 +464,7 @@ function mapProviderOptions(
export function message(msgs: ModelMessage[], model: Provider.Model, options: Record<string, unknown>) {
msgs = unsupportedParts(msgs, model)
msgs = normalizeMessages(msgs, model, options)
// kilocode_change start - apply caching for anthropic, alibaba, and GPT-5.6+ openai/azure
if (
(model.providerID === "anthropic" ||
model.providerID === "google-vertex-anthropic" ||
@@ -446,11 +473,17 @@ export function message(msgs: ModelMessage[], model: Provider.Model, options: Re
model.id.includes("anthropic") ||
model.id.includes("claude") ||
model.api.npm === "@ai-sdk/anthropic" ||
model.api.npm === "@ai-sdk/alibaba") &&
model.api.npm === "@ai-sdk/alibaba" ||
((model.api.npm === "@ai-sdk/openai" ||
model.api.npm === "@ai-sdk/azure" ||
model.providerID === "openai" ||
model.providerID === "azure") &&
supportsOpenAICacheBreakpoint(model.api.id))) &&
model.api.npm !== "@ai-sdk/gateway"
) {
msgs = applyCaching(msgs, model)
}
// kilocode_change end
// Remap providerOptions keys from stored providerID to expected SDK key
const key = sdkKey(model.api.npm)
packages/opencode/test/provider/transform.test.ts
+64
View File
@@ -3029,6 +3029,70 @@ describe("ProviderTransform.message - cache control on gateway", () => {
},
})
})
// kilocode_change start
test("openai gpt-5.6 applies promptCacheBreakpoint", () => {
const model = createModel({
providerID: "openai",
api: {
id: "gpt-5.6",
url: "https://api.openai.com/v1",
npm: "@ai-sdk/openai",
},
id: "gpt-5.6",
})
const msgs = [
{
role: "system",
content: "You are a helpful assistant",
},
{
role: "user",
content: "Hello",
},
] as any[]
const result = ProviderTransform.message(msgs, model, {}) as any[]
expect(result[0].providerOptions.openai).toEqual({
promptCacheBreakpoint: {
mode: "explicit",
},
})
expect(result[1].providerOptions.openai).toEqual({
promptCacheBreakpoint: {
mode: "explicit",
},
})
})
test("openai pre-5.6 does not apply promptCacheBreakpoint", () => {
const model = createModel({
providerID: "openai",
api: {
id: "gpt-4o",
url: "https://api.openai.com/v1",
npm: "@ai-sdk/openai",
},
id: "gpt-4o",
})
const msgs = [
{
role: "system",
content: "You are a helpful assistant",
},
{
role: "user",
content: "Hello",
},
] as any[]
const result = ProviderTransform.message(msgs, model, {}) as any[]
expect(result[0].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
expect(result[1].providerOptions?.openai?.promptCacheBreakpoint).toBeUndefined()
})
// kilocode_change end
})
describe("ProviderTransform.temperature - Cohere North", () => {