mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-09-21 14:19:18 +08:00
fix(grok): bridge image_url content parts through Responses API for vision models
The eligibility gate in grokChatResponsesBridgeEligibility rejected any structured content (arrays), including standard OpenAI image_url parts. This forced image-bearing requests to fall back to raw Chat Completions forwarding, where non-Composer models have no image bridge — silently dropping the images or causing upstream errors. Changes: - Allow content arrays containing only text and image_url parts to pass eligibility, so ChatCompletionsToResponses can convert them to input_image parts that Grok vision models natively support. - Force image-bearing requests to use Responses even when no prompt cache identity is available, since the raw path cannot forward images for non-Composer models. - Add grokChatStructuredContentBridgeable helper with conservative validation: unknown part types (e.g. input_audio) still fall back. - Update tests: image_url content is now bridgeable; add cases for text+image, text-only arrays, unknown parts, and empty arrays.
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/apicompat"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -141,20 +142,63 @@ func grokChatResponsesBridgeEligibility(body []byte) (bool, string) {
|
||||
default:
|
||||
return false, "unsupported_message_role_" + role
|
||||
}
|
||||
var content string
|
||||
if raw, exists := message["content"]; !exists || json.Unmarshal(raw, &content) != nil {
|
||||
// Structured content includes image_url and other parts whose exact
|
||||
// behavior is not guaranteed by this bridge.
|
||||
raw, exists := message["content"]
|
||||
if !exists {
|
||||
return false, "non_text_message_content"
|
||||
}
|
||||
if strings.TrimSpace(content) == "" {
|
||||
return false, "empty_message_content"
|
||||
var content string
|
||||
if json.Unmarshal(raw, &content) == nil {
|
||||
if strings.TrimSpace(content) == "" {
|
||||
return false, "empty_message_content"
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Structured content: only allow arrays whose parts are text or
|
||||
// image_url. These are losslessly convertible to Responses input_text/
|
||||
// input_image parts, so the bridge preserves Chat Completions semantics.
|
||||
if ok, reason := grokChatStructuredContentBridgeable(raw); !ok {
|
||||
return false, reason
|
||||
}
|
||||
}
|
||||
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func grokChatStructuredContentBridgeable(raw json.RawMessage) (bool, string) {
|
||||
var parts []map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &parts); err != nil {
|
||||
return false, "non_text_message_content"
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return false, "empty_message_content"
|
||||
}
|
||||
hasContent := false
|
||||
for _, part := range parts {
|
||||
var partType string
|
||||
rawType, ok := part["type"]
|
||||
if !ok || json.Unmarshal(rawType, &partType) != nil {
|
||||
return false, "non_text_message_content"
|
||||
}
|
||||
switch strings.TrimSpace(partType) {
|
||||
case "text":
|
||||
var text string
|
||||
if raw, ok := part["text"]; ok && json.Unmarshal(raw, &text) == nil {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
hasContent = true
|
||||
}
|
||||
}
|
||||
case "image_url", "input_image":
|
||||
hasContent = true
|
||||
default:
|
||||
return false, "unsupported_content_part_" + strings.TrimSpace(partType)
|
||||
}
|
||||
}
|
||||
if !hasContent {
|
||||
return false, "empty_message_content"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func grokChatNullOrEmptyArray(raw json.RawMessage) bool {
|
||||
if strings.TrimSpace(string(raw)) == "null" {
|
||||
return true
|
||||
@@ -209,7 +253,12 @@ func (s *OpenAIGatewayService) forwardGrokChatCompletionsViaResponses(
|
||||
billingModel := resolveOpenAIForwardModel(account, originalModel, defaultMappedModel)
|
||||
upstreamModel := normalizeOpenAIModelForUpstream(account, billingModel)
|
||||
cacheIdentity := resolveGrokCacheIdentity(c, body, promptCacheKey, upstreamModel)
|
||||
if !grokChatResponsesRuntimeEligible(upstreamModel, cacheIdentity) {
|
||||
// Image inputs must go through the Responses bridge: the raw Chat
|
||||
// Completions path cannot forward image_url parts to Grok's native vision
|
||||
// for non-composer models, so they would be silently dropped. Route them to
|
||||
// Responses even when no prompt-cache identity is available.
|
||||
hasImageInput := openAIJSONValueMayContainImageInput(gjson.GetBytes(body, "messages"))
|
||||
if !grokChatResponsesRuntimeEligible(upstreamModel, cacheIdentity) && !(hasImageInput && strings.TrimSpace(upstreamModel) == "grok-4.5") {
|
||||
return s.forwardAsRawChatCompletions(ctx, c, account, body, defaultMappedModel)
|
||||
}
|
||||
|
||||
|
||||
@@ -50,9 +50,29 @@ func TestGrokChatResponsesBridgeEligibility(t *testing.T) {
|
||||
reason: "unsupported_message_role_developer",
|
||||
},
|
||||
{
|
||||
name: "image content falls back",
|
||||
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,QQ=="}}]}]}`,
|
||||
reason: "non_text_message_content",
|
||||
name: "image content is bridgeable",
|
||||
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,QQ=="}}]}]}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "text and image parts are bridgeable",
|
||||
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"text","text":"what is this"},{"type":"image_url","image_url":{"url":"data:image/png;base64,QQ=="}}]}]}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "text only parts are bridgeable",
|
||||
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"text","text":"hello"}]}]}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "unknown content part falls back",
|
||||
body: `{"model":"grok","messages":[{"role":"user","content":[{"type":"input_audio","input_audio":{"data":"AA=="}}]}]}`,
|
||||
reason: "unsupported_content_part_input_audio",
|
||||
},
|
||||
{
|
||||
name: "empty content array falls back",
|
||||
body: `{"model":"grok","messages":[{"role":"user","content":[]}]}`,
|
||||
reason: "empty_message_content",
|
||||
},
|
||||
{
|
||||
name: "function tools fall back",
|
||||
|
||||
Reference in New Issue
Block a user