fix(grok): bridge image_url content parts through Responses API for vision models

The eligibility gate in grokChatResponsesBridgeEligibility rejected any
structured content (arrays), including standard OpenAI image_url parts.
This forced image-bearing requests to fall back to raw Chat Completions
forwarding, where non-Composer models have no image bridge — silently
dropping the images or causing upstream errors.

Changes:
- Allow content arrays containing only text and image_url parts to pass
  eligibility, so ChatCompletionsToResponses can convert them to
  input_image parts that Grok vision models natively support.
- Force image-bearing requests to use Responses even when no prompt
  cache identity is available, since the raw path cannot forward images
  for non-Composer models.
- Add grokChatStructuredContentBridgeable helper with conservative
  validation: unknown part types (e.g. input_audio) still fall back.
- Update tests: image_url content is now bridgeable; add cases for
  text+image, text-only arrays, unknown parts, and empty arrays.
This commit is contained in:
lenzhang
2026-07-14 22:24:03 -07:00
parent 3f605c3543
commit d5bc576b4f
2 changed files with 79 additions and 10 deletions
@@ -11,6 +11,7 @@ import (
"github.com/Wei-Shaw/sub2api/internal/pkg/apicompat"
"github.com/gin-gonic/gin"
"github.com/tidwall/gjson"
)
const (
@@ -141,20 +142,63 @@ func grokChatResponsesBridgeEligibility(body []byte) (bool, string) {
default:
return false, "unsupported_message_role_" + role
}
var content string
if raw, exists := message["content"]; !exists || json.Unmarshal(raw, &content) != nil {
// Structured content includes image_url and other parts whose exact
// behavior is not guaranteed by this bridge.
raw, exists := message["content"]
if !exists {
return false, "non_text_message_content"
}
if strings.TrimSpace(content) == "" {
return false, "empty_message_content"
var content string
if json.Unmarshal(raw, &content) == nil {
if strings.TrimSpace(content) == "" {
return false, "empty_message_content"
}
continue
}
// Structured content: only allow arrays whose parts are text or
// image_url. These are losslessly convertible to Responses input_text/
// input_image parts, so the bridge preserves Chat Completions semantics.
if ok, reason := grokChatStructuredContentBridgeable(raw); !ok {
return false, reason
}
}
return true, ""
}
func grokChatStructuredContentBridgeable(raw json.RawMessage) (bool, string) {
var parts []map[string]json.RawMessage
if err := json.Unmarshal(raw, &parts); err != nil {
return false, "non_text_message_content"
}
if len(parts) == 0 {
return false, "empty_message_content"
}
hasContent := false
for _, part := range parts {
var partType string
rawType, ok := part["type"]
if !ok || json.Unmarshal(rawType, &partType) != nil {
return false, "non_text_message_content"
}
switch strings.TrimSpace(partType) {
case "text":
var text string
if raw, ok := part["text"]; ok && json.Unmarshal(raw, &text) == nil {
if strings.TrimSpace(text) != "" {
hasContent = true
}
}
case "image_url", "input_image":
hasContent = true
default:
return false, "unsupported_content_part_" + strings.TrimSpace(partType)
}
}
if !hasContent {
return false, "empty_message_content"
}
return true, ""
}
func grokChatNullOrEmptyArray(raw json.RawMessage) bool {
if strings.TrimSpace(string(raw)) == "null" {
return true
@@ -209,7 +253,12 @@ func (s *OpenAIGatewayService) forwardGrokChatCompletionsViaResponses(
billingModel := resolveOpenAIForwardModel(account, originalModel, defaultMappedModel)
upstreamModel := normalizeOpenAIModelForUpstream(account, billingModel)
cacheIdentity := resolveGrokCacheIdentity(c, body, promptCacheKey, upstreamModel)
if !grokChatResponsesRuntimeEligible(upstreamModel, cacheIdentity) {
// Image inputs must go through the Responses bridge: the raw Chat
// Completions path cannot forward image_url parts to Grok's native vision
// for non-composer models, so they would be silently dropped. Route them to
// Responses even when no prompt-cache identity is available.
hasImageInput := openAIJSONValueMayContainImageInput(gjson.GetBytes(body, "messages"))
if !grokChatResponsesRuntimeEligible(upstreamModel, cacheIdentity) && !(hasImageInput && strings.TrimSpace(upstreamModel) == "grok-4.5") {
return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel)
}
@@ -50,9 +50,29 @@ func TestGrokChatResponsesBridgeEligibility(t *testing.T) {
reason: "unsupported_message_role_developer",
},
{
name: "image content falls back",
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,QQ=="}}]}]}`,
reason: "non_text_message_content",
name: "image content is bridgeable",
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,QQ=="}}]}]}`,
want: true,
},
{
name: "text and image parts are bridgeable",
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"text","text":"what is this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,QQ=="}}]}]}`,
want: true,
},
{
name: "text only parts are bridgeable",
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}`,
want: true,
},
{
name: "unknown content part falls back",
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"input_audio","input_audio":{"data":"AA=="}}]}]}`,
reason: "unsupported_content_part_input_audio",
},
{
name: "empty content array falls back",
body: `{"model":"grok","messages":[{"role":"user","content":[]}]}`,
reason: "empty_message_content",
},
{
name: "function tools fall back",