From d8a07e91a5945882a18de104d389ab23460c0b11 Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Tue, 14 Jul 2026 12:23:54 +0800 Subject: [PATCH] fix(grok): stabilize prompt cache routing identity --- .../service/openai_content_session_seed.go | 157 ++++++++++++++++++ .../openai_content_session_seed_test.go | 151 +++++++++++++++++ .../service/openai_gateway_grok_cache.go | 8 +- .../service/openai_gateway_grok_cache_test.go | 57 +++++++ 4 files changed, 372 insertions(+), 1 deletion(-) diff --git a/backend/internal/service/openai_content_session_seed.go b/backend/internal/service/openai_content_session_seed.go index 7c2ba25140..fce85f11bd 100644 --- a/backend/internal/service/openai_content_session_seed.go +++ b/backend/internal/service/openai_content_session_seed.go @@ -11,6 +11,10 @@ import ( // and explicit session IDs (e.g. "sess-xxx" or "compat_cc_xxx"). const contentSessionSeedPrefix = "compat_cs_" +// contentStablePrefixSessionSeedPrefix distinguishes cache identities derived +// only from request fields that remain stable across independent prompts. +const contentStablePrefixSessionSeedPrefix = "compat_csp_" + // deriveOpenAIContentSessionSeed builds a stable session seed from an // OpenAI-format request body. Only fields constant across conversation turns // are included: model, tools/functions definitions, system/developer prompts, @@ -105,3 +109,156 @@ func deriveOpenAIContentSessionSeed(body []byte) string { } return contentSessionSeedPrefix + b.String() } + +// deriveOpenAIAnchoredContentSessionSeed returns the legacy content-derived +// seed only when it contains a meaningful user/input anchor. This preserves +// the existing session derivation while preventing model-only requests from +// becoming a tenant-wide cache routing identity. +func deriveOpenAIAnchoredContentSessionSeed(body []byte) string { + if !hasOpenAIContentSessionUserAnchor(body) { + return "" + } + return deriveOpenAIContentSessionSeed(body) +} + +func hasOpenAIContentSessionUserAnchor(body []byte) bool { + if len(body) == 0 { + return false + } + + if messages := gjson.GetBytes(body, "messages"); messages.Exists() && messages.IsArray() { + anchored := false + messages.ForEach(func(_, message gjson.Result) bool { + if strings.TrimSpace(message.Get("role").String()) != "user" { + return true + } + anchored = hasMeaningfulOpenAIContent(message.Get("content")) + return false + }) + return anchored + } + + input := gjson.GetBytes(body, "input") + if !input.Exists() { + return false + } + if input.Type == gjson.String { + return strings.TrimSpace(input.String()) != "" + } + if !input.IsArray() { + return false + } + + anchored := false + input.ForEach(func(_, item gjson.Result) bool { + if strings.TrimSpace(item.Get("role").String()) == "user" { + anchored = hasMeaningfulOpenAIContent(item.Get("content")) + return false + } + if strings.TrimSpace(item.Get("type").String()) == "input_text" { + anchored = strings.TrimSpace(item.Get("text").String()) != "" + return false + } + return true + }) + return anchored +} + +func hasMeaningfulOpenAIContent(content gjson.Result) bool { + if !content.Exists() || content.Type == gjson.Null { + return false + } + if content.Type == gjson.String { + return strings.TrimSpace(content.String()) != "" + } + if !content.IsArray() { + normalized, ok := normalizeNonEmptyCompatSeedJSON(content) + return ok && strings.TrimSpace(normalized) != "" + } + + meaningful := false + content.ForEach(func(_, item gjson.Result) bool { + if item.Type == gjson.String { + meaningful = strings.TrimSpace(item.String()) != "" + } else if text := item.Get("text"); text.Exists() { + meaningful = strings.TrimSpace(text.String()) != "" + } else { + _, meaningful = normalizeNonEmptyCompatSeedJSON(item) + } + return !meaningful + }) + return meaningful +} + +// deriveOpenAIStablePrefixSessionSeed builds a seed from the reusable prefix +// of an OpenAI-format request. User and assistant content are deliberately +// excluded so independent prompts with the same system/tool prefix can share +// an upstream prompt-cache routing identity. +// +// An empty result means the request has no meaningful stable prefix. Callers +// must then use a narrower fallback instead of grouping all requests by tenant +// and model alone. +func deriveOpenAIStablePrefixSessionSeed(body []byte) string { + if len(body) == 0 { + return "" + } + + var b strings.Builder + hasStablePrefix := false + appendJSON := func(label string, value gjson.Result) { + normalized, ok := normalizeNonEmptyCompatSeedJSON(value) + if !ok { + return + } + _, _ = b.WriteString("|") + _, _ = b.WriteString(label) + _, _ = b.WriteString("=") + _, _ = b.WriteString(normalized) + hasStablePrefix = true + } + + if tools := gjson.GetBytes(body, "tools"); tools.Exists() && tools.IsArray() { + appendJSON("tools", tools) + } + if funcs := gjson.GetBytes(body, "functions"); funcs.Exists() && funcs.IsArray() { + appendJSON("functions", funcs) + } + if instructions := gjson.GetBytes(body, "instructions"); strings.TrimSpace(instructions.String()) != "" { + appendJSON("instructions", instructions) + } + + appendSystemMessages := func(items gjson.Result) { + items.ForEach(func(_, item gjson.Result) bool { + role := strings.TrimSpace(item.Get("role").String()) + switch role { + case "system", "developer": + appendJSON(role, item.Get("content")) + } + return true + }) + } + + if messages := gjson.GetBytes(body, "messages"); messages.Exists() && messages.IsArray() { + appendSystemMessages(messages) + } else if input := gjson.GetBytes(body, "input"); input.Exists() && input.IsArray() { + appendSystemMessages(input) + } + + if !hasStablePrefix { + return "" + } + return contentStablePrefixSessionSeedPrefix + b.String() +} + +func normalizeNonEmptyCompatSeedJSON(value gjson.Result) (string, bool) { + if !value.Exists() || value.Type == gjson.Null { + return "", false + } + normalized := normalizeCompatSeedJSON(json.RawMessage(value.Raw)) + switch normalized { + case "", `""`, "[]", "{}", "null": + return "", false + default: + return normalized, true + } +} diff --git a/backend/internal/service/openai_content_session_seed_test.go b/backend/internal/service/openai_content_session_seed_test.go index 65a0bf1808..6dadc5cf53 100644 --- a/backend/internal/service/openai_content_session_seed_test.go +++ b/backend/internal/service/openai_content_session_seed_test.go @@ -216,3 +216,154 @@ func TestDeriveOpenAIContentSessionSeed_ResponsesAPI_TypedMessageItem(t *testing require.Contains(t, seed, "|first_user=") require.Contains(t, seed, "Hello from typed message") } + +func TestDeriveOpenAIStablePrefixSessionSeed_IgnoresUserContent(t *testing.T) { + first := []byte(`{ + "model": "grok", + "instructions": "Be concise.", + "tools": [{"type":"function","name":"lookup","parameters":{"type":"object"}}], + "input": [{"role":"user","content":"Question A"}] + }`) + second := []byte(`{ + "model": "grok", + "instructions": "Be concise.", + "tools": [{"parameters":{"type":"object"},"name":"lookup","type":"function"}], + "input": [{"role":"user","content":"Question B"}] + }`) + + firstSeed := deriveOpenAIStablePrefixSessionSeed(first) + secondSeed := deriveOpenAIStablePrefixSessionSeed(second) + + require.NotEmpty(t, firstSeed) + require.Equal(t, firstSeed, secondSeed) + require.NotContains(t, firstSeed, "Question A") + require.NotContains(t, firstSeed, "first_user") +} + +func TestDeriveOpenAIStablePrefixSessionSeed_IsolatesStablePrefixFields(t *testing.T) { + base := []byte(`{ + "instructions":"Be concise.", + "tools":[{"type":"function","name":"lookup"}], + "input":[{"role":"system","content":"System A"},{"role":"user","content":"Question"}] + }`) + differentInstructions := []byte(`{ + "instructions":"Be detailed.", + "tools":[{"type":"function","name":"lookup"}], + "input":[{"role":"system","content":"System A"},{"role":"user","content":"Question"}] + }`) + differentTools := []byte(`{ + "instructions":"Be concise.", + "tools":[{"type":"function","name":"search"}], + "input":[{"role":"system","content":"System A"},{"role":"user","content":"Question"}] + }`) + differentSystem := []byte(`{ + "instructions":"Be concise.", + "tools":[{"type":"function","name":"lookup"}], + "input":[{"role":"system","content":"System B"},{"role":"user","content":"Question"}] + }`) + + baseSeed := deriveOpenAIStablePrefixSessionSeed(base) + require.NotEqual(t, baseSeed, deriveOpenAIStablePrefixSessionSeed(differentInstructions)) + require.NotEqual(t, baseSeed, deriveOpenAIStablePrefixSessionSeed(differentTools)) + require.NotEqual(t, baseSeed, deriveOpenAIStablePrefixSessionSeed(differentSystem)) +} + +func TestDeriveOpenAIStablePrefixSessionSeed_ChatSystemAndDeveloper(t *testing.T) { + first := []byte(`{ + "messages":[ + {"role":"system","content":"System prompt"}, + {"role":"developer","content":[{"type":"text","text":"Developer prompt"}]}, + {"role":"user","content":"Question A"} + ] + }`) + second := []byte(`{ + "messages":[ + {"role":"system","content":"System prompt"}, + {"role":"developer","content":[{"text":"Developer prompt","type":"text"}]}, + {"role":"user","content":"Question B"} + ] + }`) + + firstSeed := deriveOpenAIStablePrefixSessionSeed(first) + require.Equal(t, firstSeed, deriveOpenAIStablePrefixSessionSeed(second)) + require.Contains(t, firstSeed, "System prompt") + require.Contains(t, firstSeed, "Developer prompt") +} + +func TestDeriveOpenAIStablePrefixSessionSeed_EncodesSystemAndDeveloperRoles(t *testing.T) { + systemThenDeveloper := []byte(`{ + "messages":[ + {"role":"system","content":"Prompt A"}, + {"role":"developer","content":"Prompt B"} + ] + }`) + developerThenSystem := []byte(`{ + "messages":[ + {"role":"developer","content":"Prompt A"}, + {"role":"system","content":"Prompt B"} + ] + }`) + + firstSeed := deriveOpenAIStablePrefixSessionSeed(systemThenDeveloper) + secondSeed := deriveOpenAIStablePrefixSessionSeed(developerThenSystem) + + require.NotEqual(t, firstSeed, secondSeed) + require.Contains(t, firstSeed, "|system=") + require.Contains(t, firstSeed, "|developer=") +} + +func TestDeriveOpenAIStablePrefixSessionSeed_EncodesInstructionDelimiters(t *testing.T) { + instructionOnly := []byte(`{ + "instructions":"foo|system=\"bar\"" + }`) + instructionAndSystem := []byte(`{ + "instructions":"foo", + "input":[{"role":"system","content":"bar"}] + }`) + + firstSeed := deriveOpenAIStablePrefixSessionSeed(instructionOnly) + secondSeed := deriveOpenAIStablePrefixSessionSeed(instructionAndSystem) + + require.NotEmpty(t, firstSeed) + require.NotEmpty(t, secondSeed) + require.NotEqual(t, firstSeed, secondSeed) +} + +func TestDeriveOpenAIAnchoredContentSessionSeed_RequiresMeaningfulAnchor(t *testing.T) { + emptyAnchors := [][]byte{ + nil, + []byte(`{"model":"grok"}`), + []byte(`{"model":"grok","messages":[{"role":"assistant","content":"answer"}]}`), + []byte(`{"model":"grok","messages":[{"role":"user","content":" "}]}`), + []byte(`{"model":"grok","messages":[{"role":"user","content":[{"type":"text","text":""}]}]}`), + []byte(`{"model":"grok","input":" "}`), + []byte(`{"model":"grok","input":[{"type":"input_text","text":""}]}`), + } + for _, body := range emptyAnchors { + require.Empty(t, deriveOpenAIAnchoredContentSessionSeed(body)) + } + + meaningfulAnchors := [][]byte{ + []byte(`{"model":"grok","messages":[{"role":"user","content":"question"}]}`), + []byte(`{"model":"grok","messages":[{"role":"user","content":[{"type":"text","text":"question"}]}]}`), + []byte(`{"model":"grok","input":"question"}`), + []byte(`{"model":"grok","input":[{"type":"input_text","text":"question"}]}`), + } + for _, body := range meaningfulAnchors { + require.NotEmpty(t, deriveOpenAIAnchoredContentSessionSeed(body)) + } +} + +func TestDeriveOpenAIStablePrefixSessionSeed_RequiresMeaningfulPrefix(t *testing.T) { + tests := [][]byte{ + nil, + []byte(`{}`), + []byte(`{"model":"grok","input":"Question A"}`), + []byte(`{"model":"grok","tools":[],"input":"Question A"}`), + []byte(`{"model":"grok","functions":[],"instructions":" ","messages":[{"role":"system","content":""},{"role":"user","content":"Question A"}]}`), + } + + for _, body := range tests { + require.Empty(t, deriveOpenAIStablePrefixSessionSeed(body)) + } +} diff --git a/backend/internal/service/openai_gateway_grok_cache.go b/backend/internal/service/openai_gateway_grok_cache.go index 20934b94c3..1d689bce8a 100644 --- a/backend/internal/service/openai_gateway_grok_cache.go +++ b/backend/internal/service/openai_gateway_grok_cache.go @@ -42,7 +42,13 @@ func resolveGrokCacheIdentity(c *gin.Context, body []byte, explicitKey, upstream seed := explicitGrokCacheSeed(c, body, explicitKey) if seed == "" { - seed = deriveOpenAIContentSessionSeed(body) + seed = deriveOpenAIStablePrefixSessionSeed(body) + if seed == "" { + // A model alone is too broad for cache routing. Preserve the + // existing first-user-derived identity when no reusable prefix is + // available so unrelated prompts do not share one tenant-wide key. + seed = deriveOpenAIAnchoredContentSessionSeed(body) + } } if seed == "" { return "" diff --git a/backend/internal/service/openai_gateway_grok_cache_test.go b/backend/internal/service/openai_gateway_grok_cache_test.go index 556f19304f..42abfc5800 100644 --- a/backend/internal/service/openai_gateway_grok_cache_test.go +++ b/backend/internal/service/openai_gateway_grok_cache_test.go @@ -37,6 +37,63 @@ func TestResolveGrokCacheIdentityStableAcrossAppendOnlyTurns(t *testing.T) { require.Equal(t, first, second) } +func TestResolveGrokCacheIdentityStableAcrossIndependentPromptsWithSamePrefix(t *testing.T) { + gin.SetMode(gin.TestMode) + c := newGrokCacheTestContext(102) + firstBody := []byte(`{"model":"grok","instructions":"be concise","tools":[{"type":"function","name":"lookup"}],"input":[{"role":"user","content":"Question A"}]}`) + secondBody := []byte(`{"model":"grok","instructions":"be concise","tools":[{"type":"function","name":"lookup"}],"input":[{"role":"user","content":"Question B"}]}`) + + first := resolveGrokCacheIdentity(c, firstBody, "", "grok-4.5") + second := resolveGrokCacheIdentity(c, secondBody, "", "grok-4.5") + + require.NotEmpty(t, first) + require.Equal(t, first, second) +} + +func TestResolveGrokCacheIdentityStablePrefixIsolation(t *testing.T) { + gin.SetMode(gin.TestMode) + baseBody := []byte(`{"model":"grok","instructions":"be concise","tools":[{"type":"function","name":"lookup"}],"input":[{"role":"system","content":"System A"},{"role":"user","content":"Question A"}]}`) + differentInstructions := []byte(`{"model":"grok","instructions":"be detailed","tools":[{"type":"function","name":"lookup"}],"input":[{"role":"system","content":"System A"},{"role":"user","content":"Question B"}]}`) + differentSystem := []byte(`{"model":"grok","instructions":"be concise","tools":[{"type":"function","name":"lookup"}],"input":[{"role":"system","content":"System B"},{"role":"user","content":"Question B"}]}`) + differentTools := []byte(`{"model":"grok","instructions":"be concise","tools":[{"type":"function","name":"search"}],"input":[{"role":"system","content":"System A"},{"role":"user","content":"Question B"}]}`) + + base := resolveGrokCacheIdentity(newGrokCacheTestContext(103), baseBody, "", "grok-4.5") + require.NotEqual(t, base, resolveGrokCacheIdentity(newGrokCacheTestContext(104), baseBody, "", "grok-4.5")) + require.NotEqual(t, base, resolveGrokCacheIdentity(newGrokCacheTestContext(103), baseBody, "", "grok-4.3")) + require.NotEqual(t, base, resolveGrokCacheIdentity(newGrokCacheTestContext(103), differentInstructions, "", "grok-4.5")) + require.NotEqual(t, base, resolveGrokCacheIdentity(newGrokCacheTestContext(103), differentSystem, "", "grok-4.5")) + require.NotEqual(t, base, resolveGrokCacheIdentity(newGrokCacheTestContext(103), differentTools, "", "grok-4.5")) +} + +func TestResolveGrokCacheIdentityFallsBackWhenStablePrefixIsEmpty(t *testing.T) { + gin.SetMode(gin.TestMode) + c := newGrokCacheTestContext(105) + firstBody := []byte(`{"model":"grok","tools":[],"input":"Question A"}`) + secondBody := []byte(`{"model":"grok","tools":[],"input":"Question B"}`) + + first := resolveGrokCacheIdentity(c, firstBody, "", "grok-4.5") + second := resolveGrokCacheIdentity(c, secondBody, "", "grok-4.5") + + require.NotEmpty(t, first) + require.NotEmpty(t, second) + require.NotEqual(t, first, second) +} + +func TestResolveGrokCacheIdentitySkipsUnanchoredFallback(t *testing.T) { + gin.SetMode(gin.TestMode) + c := newGrokCacheTestContext(106) + tests := [][]byte{ + []byte(`{"model":"grok"}`), + []byte(`{"model":"grok","messages":[{"role":"assistant","content":"answer"}]}`), + []byte(`{"model":"grok","messages":[{"role":"user","content":""}]}`), + []byte(`{"model":"grok","input":" "}`), + } + + for _, body := range tests { + require.Empty(t, resolveGrokCacheIdentity(c, body, "", "grok-4.5")) + } +} + func TestResolveGrokCacheIdentityIsolatesAPIKeyAndMappedModel(t *testing.T) { gin.SetMode(gin.TestMode) body := []byte(`{"model":"grok","input":"same prompt"}`)