From 410ea8490cae48615894772915212593f30b0b0b Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Thu, 16 Jul 2026 00:16:21 +0800 Subject: [PATCH 1/2] fix(grok): allow passive Codex image tool declarations --- .../handler/gateway_handler_responses.go | 2 +- .../handler/openai_gateway_handler.go | 4 +- .../openai_grok_image_intent_gate_test.go | 101 ++++++++++++++++ .../service/image_generation_intent.go | 106 ++++++++++++++++- .../image_generation_intent_grok_test.go | 108 ++++++++++++++++++ .../service/openai_ws_forwarder_ingress.go | 2 +- 6 files changed, 316 insertions(+), 7 deletions(-) create mode 100644 backend/internal/handler/openai_grok_image_intent_gate_test.go create mode 100644 backend/internal/service/image_generation_intent_grok_test.go diff --git a/backend/internal/handler/gateway_handler_responses.go b/backend/internal/handler/gateway_handler_responses.go index a5060b7263..ce1ae40811 100644 --- a/backend/internal/handler/gateway_handler_responses.go +++ b/backend/internal/handler/gateway_handler_responses.go @@ -85,7 +85,7 @@ func (h *GatewayHandler) Responses(c *gin.Context) { setOpsRequestContext(c, reqModel, reqStream) setOpsEndpointContext(c, "", int16(service.RequestTypeFromLegacy(reqStream, false))) requestCtx := c.Request.Context() - if service.IsImageGenerationIntent("/v1/responses", reqModel, body) { + if service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, body, openAICompatibleRequestPlatform(apiKey)) { requestCtx = service.WithOpenAIImageGenerationIntent(requestCtx) } diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 83c4b4ac23..6a634bb595 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -272,7 +272,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { return } - imageIntent := service.IsImageGenerationIntent("/v1/responses", reqModel, body) + imageIntent := service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, body, openAICompatibleRequestPlatform(apiKey)) if imageIntent && !service.GroupAllowsImageGeneration(apiKey.Group) { h.errorResponse(c, http.StatusForbidden, "permission_error", service.ImageGenerationPermissionMessage()) return @@ -1461,7 +1461,7 @@ func (h *OpenAIGatewayHandler) ResponsesWebSocket(c *gin.Context) { return } - if service.IsImageGenerationIntent("/v1/responses", reqModel, firstMessage) && !service.GroupAllowsImageGeneration(apiKey.Group) { + if service.IsImageGenerationIntentForPlatform("/v1/responses", reqModel, firstMessage, openAICompatibleRequestPlatform(apiKey)) && !service.GroupAllowsImageGeneration(apiKey.Group) { closeOpenAIClientWS(wsConn, coderws.StatusPolicyViolation, service.ImageGenerationPermissionMessage()) return } diff --git a/backend/internal/handler/openai_grok_image_intent_gate_test.go b/backend/internal/handler/openai_grok_image_intent_gate_test.go new file mode 100644 index 0000000000..3c20160c05 --- /dev/null +++ b/backend/internal/handler/openai_grok_image_intent_gate_test.go @@ -0,0 +1,101 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + middleware2 "github.com/Wei-Shaw/sub2api/internal/server/middleware" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func TestOpenAIGatewayHandlerResponses_GrokPassiveImageToolDeclarationBypassesPermissionGate(t *testing.T) { + body := `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}` + rec := runOpenAIResponsesImagePermissionGateTest(t, service.PlatformGrok, body) + + require.NotEqual(t, http.StatusForbidden, rec.Code) + require.NotContains(t, rec.Body.String(), service.ImageGenerationPermissionMessage()) +} + +func TestOpenAIGatewayHandlerResponses_GrokResponsesLiteImageToolDeclarationBypassesPermissionGate(t *testing.T) { + body := `{"model":"grok-4.5","tool_choice":"auto","input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]},{"type":"message","role":"user","content":"write code"}]}` + rec := runOpenAIResponsesImagePermissionGateTest(t, service.PlatformGrok, body) + + require.NotEqual(t, http.StatusForbidden, rec.Code) + require.NotContains(t, rec.Body.String(), service.ImageGenerationPermissionMessage()) +} + +func TestOpenAIGatewayHandlerResponses_ImagePermissionHardSignalsStillRejected(t *testing.T) { + passiveNamespace := `{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}` + tests := []struct { + name string + platform string + body string + }{ + { + name: "OpenAI keeps declaration semantics", + platform: service.PlatformOpenAI, + body: passiveNamespace, + }, + { + name: "Grok native image_generation declaration", + platform: service.PlatformGrok, + body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"input":"draw"}`, + }, + { + name: "Grok explicit image_gen tool choice", + platform: service.PlatformGrok, + body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen"}],"tool_choice":{"type":"namespace","name":"image_gen"},"input":"draw"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := runOpenAIResponsesImagePermissionGateTest(t, tt.platform, tt.body) + + require.Equal(t, http.StatusForbidden, rec.Code) + require.Contains(t, rec.Body.String(), service.ImageGenerationPermissionMessage()) + }) + } +} + +func runOpenAIResponsesImagePermissionGateTest(t *testing.T, platform string, body string) *httptest.ResponseRecorder { + t.Helper() + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", strings.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + groupID := int64(6301) + userID := int64(6302) + c.Set(string(middleware2.ContextKeyAPIKey), &service.APIKey{ + ID: 6303, + GroupID: &groupID, + Group: &service.Group{ + ID: groupID, + Platform: platform, + AllowImageGeneration: false, + }, + User: &service.User{ID: userID, Status: service.StatusActive}, + }) + c.Set(string(middleware2.ContextKeyUser), middleware2.AuthSubject{UserID: userID, Concurrency: 1}) + + h := &OpenAIGatewayHandler{ + gatewayService: &service.OpenAIGatewayService{}, + billingCacheService: service.NewBillingCacheService(nil, nil, nil, nil, nil, nil, &config.Config{RunMode: config.RunModeSimple}, nil), + apiKeyService: &service.APIKeyService{}, + concurrencyHelper: &ConcurrencyHelper{concurrencyService: service.NewConcurrencyService( + &helperConcurrencyCacheStub{userSeq: []bool{true}}, + )}, + cfg: &config.Config{}, + imageLimiter: &imageConcurrencyLimiter{}, + } + + h.Responses(c) + return rec +} diff --git a/backend/internal/service/image_generation_intent.go b/backend/internal/service/image_generation_intent.go index 72f3be7db6..9351fcd682 100644 --- a/backend/internal/service/image_generation_intent.go +++ b/backend/internal/service/image_generation_intent.go @@ -79,6 +79,56 @@ func IsImageGenerationIntent(endpoint string, requestedModel string, body []byte return imageIntent } +// IsImageGenerationIntentForPlatform applies platform-specific intent rules. +// +// Codex advertises the image_gen namespace on ordinary Responses requests so +// that it is available if the model needs it. Grok strips namespace and +// Responses Lite additional_tools declarations before forwarding, so those +// declarations alone must not turn every Codex request into an image request. +// Native image_generation tools, explicit image selection and image models +// remain image intent. Other platforms retain the original declaration rule. +func IsImageGenerationIntentForPlatform(endpoint string, requestedModel string, body []byte, platform string) bool { + if !strings.EqualFold(strings.TrimSpace(platform), PlatformGrok) { + return IsImageGenerationIntent(endpoint, requestedModel, body) + } + return isExplicitGrokImageGenerationIntent(endpoint, requestedModel, body) +} + +func isExplicitGrokImageGenerationIntent(endpoint string, requestedModel string, body []byte) bool { + if IsImageGenerationEndpoint(endpoint) || isOpenAIImageGenerationModel(requestedModel) { + return true + } + if len(body) == 0 || !gjson.ValidBytes(body) { + return false + } + + var modelSeen, toolsSeen, toolChoiceSeen bool + imageIntent := false + parseRawJSONView(body).ForEach(func(key, value gjson.Result) bool { + switch key.Str { + case "model": + if !modelSeen { + modelSeen = true + imageIntent = isOpenAIImageGenerationModel(strings.TrimSpace(value.String())) + } + case "tools": + if !toolsSeen { + toolsSeen = true + // Grok removes namespace catalogs before forwarding. Native + // image_generation remains an explicit capability request. + imageIntent = openAIJSONToolsContainNativeImageGeneration(value) + } + case "tool_choice": + if !toolChoiceSeen { + toolChoiceSeen = true + imageIntent = openAIJSONToolChoiceSelectsExplicitImageGeneration(value) + } + } + return !imageIntent && (!modelSeen || !toolsSeen || !toolChoiceSeen) + }) + return imageIntent +} + // IsImageGenerationIntentMap is the map-backed variant used after service-side request mutation. func IsImageGenerationIntentMap(endpoint string, requestedModel string, reqBody map[string]any) bool { if IsImageGenerationEndpoint(endpoint) { @@ -140,6 +190,18 @@ func openAIJSONToolsContainImageGeneration(tools gjson.Result) bool { return found } +func openAIJSONToolsContainNativeImageGeneration(tools gjson.Result) bool { + if !tools.IsArray() { + return false + } + found := false + tools.ForEach(func(_, item gjson.Result) bool { + found = isOpenAIImageGenerationType(openAIJSONString(item.Get("type"))) + return !found + }) + return found +} + func isOpenAIImageGenerationType(value string) bool { return strings.TrimSpace(value) == "image_generation" } @@ -148,9 +210,8 @@ func isOpenAIImageGenNamespaceName(value string) bool { return strings.TrimSpace(value) == "image_gen" } -// isImageGenNamespaceTool detects the Codex namespace-style image generation -// tool declaration: { "type": "namespace", "name": "image_gen", ... }. -// Codex /image uses this instead of the flat { "type": "image_generation" }. +// isImageGenNamespaceTool detects the namespace advertised by Codex's built-in +// image-generation extension instead of a hosted image_generation tool. func isImageGenNamespaceTool(tool gjson.Result) bool { return openAIJSONString(tool.Get("type")) == "namespace" && isOpenAIImageGenNamespaceName(openAIJSONString(tool.Get("name"))) @@ -235,6 +296,45 @@ func openAIJSONToolChoiceSelectsImageGeneration(choice gjson.Result) bool { return false } +func openAIJSONToolChoiceSelectsExplicitImageGeneration(choice gjson.Result) bool { + if openAIJSONToolChoiceSelectsImageGeneration(choice) { + return true + } + if !choice.IsObject() { + return false + } + if tool := choice.Get("tool"); tool.IsObject() && openAIJSONToolChoiceSelectsExplicitImageGeneration(tool) { + return true + } + if isOpenAIImageGenFunctionReference( + openAIJSONString(choice.Get("namespace")), + openAIJSONString(choice.Get("name")), + ) { + return true + } + if fn := choice.Get("function"); fn.IsObject() { + return isOpenAIImageGenFunctionReference( + openAIJSONString(fn.Get("namespace")), + openAIJSONString(fn.Get("name")), + ) + } + return false +} + +func isOpenAIImageGenFunctionReference(namespace string, name string) bool { + namespace = strings.TrimSpace(namespace) + name = strings.TrimSpace(name) + if namespace == "image_gen" && name == "imagegen" { + return true + } + switch name { + case "image_gen.imagegen", "image_gen__imagegen": + return true + default: + return false + } +} + func openAIAnyToolChoiceSelectsImageGeneration(choice any) bool { switch v := choice.(type) { case string: diff --git a/backend/internal/service/image_generation_intent_grok_test.go b/backend/internal/service/image_generation_intent_grok_test.go new file mode 100644 index 0000000000..cfef208d45 --- /dev/null +++ b/backend/internal/service/image_generation_intent_grok_test.go @@ -0,0 +1,108 @@ +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsImageGenerationIntentForPlatform_GrokCodexDeclarations(t *testing.T) { + tests := []struct { + name string + body string + want bool + }{ + { + name: "top-level image_gen namespace is passive", + body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"tool_choice":"auto","input":"write code"}`, + }, + { + name: "responses lite additional_tools image_gen is passive", + body: `{"model":"grok-4.5","tool_choice":"auto","input":[{"type":"additional_tools","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}]},{"type":"message","role":"user","content":"write code"}]}`, + }, + { + name: "legacy adapter flattened image_gen function declaration is passive", + body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"input":"write code"}`, + }, + { + name: "native image generation declaration remains explicit", + body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"input":"draw a cat"}`, + want: true, + }, + { + name: "native image generation tool choice is explicit", + body: `{"model":"grok-4.5","tools":[{"type":"image_generation"}],"tool_choice":{"type":"image_generation"},"input":"draw a cat"}`, + want: true, + }, + { + name: "image_gen namespace tool choice is explicit", + body: `{"model":"grok-4.5","tools":[{"type":"namespace","name":"image_gen"}],"tool_choice":{"type":"namespace","name":"image_gen"},"input":"draw a cat"}`, + want: true, + }, + { + name: "flattened image_gen function tool choice is explicit", + body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"tool_choice":{"type":"function","name":"image_gen.imagegen"},"input":"draw a cat"}`, + want: true, + }, + { + name: "wrapped image_gen function tool choice is explicit", + body: `{"model":"grok-4.5","tools":[{"type":"function","name":"image_gen.imagegen"}],"tool_choice":{"tool":{"type":"function","name":"image_gen.imagegen"}},"input":"draw a cat"}`, + want: true, + }, + { + name: "vision input is not image generation", + body: `{"model":"grok-4.5","input":[{"type":"message","role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,AA=="}]}]}`, + }, + { + name: "ordinary function declaration is not image generation", + body: `{"model":"grok-4.5","tools":[{"type":"function","name":"lookup"}],"tool_choice":"auto","input":"write code"}`, + }, + { + name: "image_gen function call history is not current intent", + body: `{"model":"grok-4.5","input":[{"type":"function_call","namespace":"image_gen","name":"imagegen","arguments":"{}"}]}`, + }, + { + name: "image generation call history is not current intent", + body: `{"model":"grok-4.5","input":[{"type":"image_generation_call","id":"ig_1"}]}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, IsImageGenerationIntentForPlatform( + openAIResponsesEndpoint, + "grok-4.5", + []byte(tt.body), + PlatformGrok, + )) + }) + } +} + +func TestIsImageGenerationIntentForPlatform_GrokPreservesHardSignals(t *testing.T) { + require.True(t, IsImageGenerationIntentForPlatform( + "/v1/images/generations", + "grok-4.5", + []byte(`{"input":"draw"}`), + PlatformGrok, + )) + require.True(t, IsImageGenerationIntentForPlatform( + openAIResponsesEndpoint, + "gpt-image-2", + []byte(`{"input":"draw"}`), + PlatformGrok, + )) + require.True(t, IsImageGenerationIntentForPlatform( + openAIResponsesEndpoint, + "grok-4.5", + []byte(`{"model":"gpt-image-2","input":"draw"}`), + PlatformGrok, + )) +} + +func TestIsImageGenerationIntentForPlatform_OtherPlatformsKeepDeclarationSemantics(t *testing.T) { + body := []byte(`{"model":"gpt-5.5","tools":[{"type":"namespace","name":"image_gen","tools":[{"type":"function","name":"imagegen"}]}],"input":"write code"}`) + + require.True(t, IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, "gpt-5.5", body, PlatformOpenAI)) + require.True(t, IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, "gpt-5.5", body, PlatformAnthropic)) +} diff --git a/backend/internal/service/openai_ws_forwarder_ingress.go b/backend/internal/service/openai_ws_forwarder_ingress.go index 51aace6419..89846e1541 100644 --- a/backend/internal/service/openai_ws_forwarder_ingress.go +++ b/backend/internal/service/openai_ws_forwarder_ingress.go @@ -305,7 +305,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient( normalized = stripped logOpenAIWSModeInfo("ingress_ws_codex_spark_image_tool_stripped account_id=%d", account.ID) } - imageIntent := IsImageGenerationIntent(openAIResponsesEndpoint, originalModel, normalized) + imageIntent := IsImageGenerationIntentForPlatform(openAIResponsesEndpoint, originalModel, normalized, account.Platform) if imageIntent && !imageGenerationAllowed { return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, ImageGenerationPermissionMessage(), nil) } From e8cc1fe26b07152ee9a5a3cb584d2499e2d21f3d Mon Sep 17 00:00:00 2001 From: superman2003 <2112076433zcr@gmail.com> Date: Thu, 16 Jul 2026 00:15:38 +0800 Subject: [PATCH 2/2] feat(frontend): add Grok key setup templates --- frontend/src/components/keys/UseKeyModal.vue | 136 +++++++++++++++- .../keys/__tests__/UseKeyModal.spec.ts | 153 +++++++++++++++++- frontend/src/i18n/locales/en/dashboard.ts | 9 +- frontend/src/i18n/locales/zh/dashboard.ts | 11 +- 4 files changed, 297 insertions(+), 12 deletions(-) diff --git a/frontend/src/components/keys/UseKeyModal.vue b/frontend/src/components/keys/UseKeyModal.vue index 1dda0c51eb..4663b2b7d8 100644 --- a/frontend/src/components/keys/UseKeyModal.vue +++ b/frontend/src/components/keys/UseKeyModal.vue @@ -29,11 +29,12 @@

-
-