opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-30 17:13:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Automated cherry pick of #25132: refactor(aiproxy): rename chat log config to API log with S3 fields (#25141)

Browse Source 
* refactor(aiproxy): rename chat log config to API log with S3 fields

* feat(aiproxy): add visual extension for Responses image tools

Wire visual model catalog, orchestration, and OpenAI Responses compat so chat/Codex can invoke image generation and editing tools.
This commit is contained in:
Zexi Li
2026-07-14 11:52:58 +08:00
committed by GitHub
parent 5f1b76755b
commit 73d7cbbb1f
33 changed files with 2073 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/aiproxy/chatlog/chatlog.go
+17 -17
View File
@@ -44,12 +44,12 @@ type Options struct {
UploadEnabled bool
UploadIntervalSeconds int
SegmentMinutes int
MinioEndpoint string
MinioAccessKey string
MinioSecretKey string
MinioBucket string
MinioSecure bool
MinioPrefix string
S3Endpoint string
S3AccessKey string
S3SecretKey string
S3Bucket string
S3Secure bool
S3Prefix string
Instance string
}
@@ -255,12 +255,12 @@ func UploadKey(prefix string, ts time.Time, filename string, instance string) st
}
func (w *Writer) s3Client() (*s3.Client, error) {
if w.opts.MinioEndpoint == "" || w.opts.MinioBucket == "" || w.opts.MinioAccessKey == "" || w.opts.MinioSecretKey == "" {
return nil, errors.New("missing MinIO/S3 config")
if w.opts.S3Endpoint == "" || w.opts.S3Bucket == "" || w.opts.S3AccessKey == "" || w.opts.S3SecretKey == "" {
return nil, errors.New("missing S3 config")
}
endpoint := strings.TrimSpace(w.opts.MinioEndpoint)
endpoint := strings.TrimSpace(w.opts.S3Endpoint)
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
if w.opts.MinioSecure {
if w.opts.S3Secure {
endpoint = "https://" + endpoint
} else {
endpoint = "http://" + endpoint
@@ -268,7 +268,7 @@ func (w *Writer) s3Client() (*s3.Client, error) {
}
return s3.NewFromConfig(aws.Config{
Region: "us-east-1",
Credentials: credentials.NewStaticCredentialsProvider(w.opts.MinioAccessKey, w.opts.MinioSecretKey, ""),
Credentials: credentials.NewStaticCredentialsProvider(w.opts.S3AccessKey, w.opts.S3SecretKey, ""),
BaseEndpoint: aws.String(endpoint),
}, func(o *s3.Options) {
o.UsePathStyle = true
@@ -433,7 +433,7 @@ func (w *Writer) Read(ctx context.Context, opts ReadOptions) (*ReadResult, error
return nil, err
}
ret := &ReadResult{Logs: make([]Record, 0)}
for _, prefix := range hourPrefixes(w.opts.MinioPrefix, opts.Start, opts.End) {
for _, prefix := range hourPrefixes(w.opts.S3Prefix, opts.Start, opts.End) {
if err := w.readPrefix(ctx, client, prefix, opts, ret); err != nil {
return nil, err
}
@@ -446,7 +446,7 @@ func (w *Writer) Read(ctx context.Context, opts ReadOptions) (*ReadResult, error
func (w *Writer) readPrefix(ctx context.Context, client *s3.Client, prefix string, opts ReadOptions, ret *ReadResult) error {
in := &s3.ListObjectsV2Input{
Bucket: aws.String(w.opts.MinioBucket),
Bucket: aws.String(w.opts.S3Bucket),
Prefix: aws.String(prefix),
}
for {
@@ -475,7 +475,7 @@ func (w *Writer) readPrefix(ctx context.Context, client *s3.Client, prefix strin
func (w *Writer) readObject(ctx context.Context, client *s3.Client, key string, opts ReadOptions, ret *ReadResult) error {
out, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(w.opts.MinioBucket),
Bucket: aws.String(w.opts.S3Bucket),
Key: aws.String(key),
})
if err != nil {
@@ -525,12 +525,12 @@ func (w *Writer) uploadFile(ctx context.Context, path string) error {
if err != nil {
return err
}
if err := ensureBucket(ctx, client, w.opts.MinioBucket); err != nil {
if err := ensureBucket(ctx, client, w.opts.S3Bucket); err != nil {
return err
}
key := UploadKey(w.opts.MinioPrefix, ts, path, w.opts.Instance)
key := UploadKey(w.opts.S3Prefix, ts, path, w.opts.Instance)
_, err = client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(w.opts.MinioBucket),
Bucket: aws.String(w.opts.S3Bucket),
Key: aws.String(key),
Body: f,
})
pkg/aiproxy/codexconfig/catalog.go
+8 -3
View File
@@ -32,8 +32,9 @@ const (
// ModelListEntry is one OpenAI-compatible model from GET /ai/openai/v1/models.
type ModelListEntry struct {
ID string
OwnedBy string
ID string
OwnedBy string
InputModalities []string
}
// ModelInfo represents a model entry in Codex models_catalog.json.
@@ -137,7 +138,11 @@ func buildModelInfoFromEntry(entry ModelListEntry) ModelInfo {
slug := strings.TrimSpace(entry.ID)
displayName := displayNameForSlug(slug)
description := seedDescriptionForSlug(slug)
return newModelInfo(slug, displayName, description, 0, nil)
modalities := entry.InputModalities
if len(modalities) == 0 {
modalities = nil
}
return newModelInfo(slug, displayName, description, 0, modalities)
}
func displayNameForSlug(slug string) string {
pkg/aiproxy/codexconfig/catalog_visual_test.go
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package codexconfig
import "testing"
func TestBuildCatalogVisualModalities(t *testing.T) {
catalog := BuildCatalogFromIDs([]ModelListEntry{{
ID: "deepseek-v4-pro",
OwnedBy: "deepseek",
InputModalities: []string{"text", "image"},
}})
if len(catalog) != 1 {
t.Fatalf("catalog len = %d", len(catalog))
}
if len(catalog[0].InputModalities) != 2 || catalog[0].InputModalities[1] != "image" {
t.Fatalf("modalities = %#v", catalog[0].InputModalities)
}
}
pkg/aiproxy/codexconfig/codexconfig.go
+6 -4
View File
@@ -250,8 +250,9 @@ func resolveRoutingModelKey(session *mcclient.ClientSession, routing string) (st
type modelsListResponse struct {
Data []struct {
ID string `json:"id"`
OwnedBy string `json:"owned_by"`
ID string `json:"id"`
OwnedBy string `json:"owned_by"`
InputModalities []string `json:"input_modalities"`
} `json:"data"`
}
@@ -286,8 +287,9 @@ func listModelEntries(session *mcclient.ClientSession, aiproxyURL, virtualKey st
continue
}
entries = append(entries, ModelListEntry{
ID: id,
OwnedBy: strings.TrimSpace(item.OwnedBy),
ID: id,
OwnedBy: strings.TrimSpace(item.OwnedBy),
InputModalities: append([]string(nil), item.InputModalities...),
})
}
return entries, nil
pkg/aiproxy/extensions/visual/client.go
+157
View File
@@ -0,0 +1,157 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/aiproxy/models"
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
"yunion.io/x/onecloud/pkg/aiproxy/upstream"
)
const visualSystemPrompt = "You are a vision analysis model behind Cloudpods AI Proxy Visual. Analyze images carefully, state uncertainty, and do not invent visual facts."
// HTTPVisionClient calls a multimodal upstream using OpenAI chat/completions.
type HTTPVisionClient struct {
up *models.ChatUpstream
maxTokens int
}
func NewHTTPVisionClient(up *models.ChatUpstream, maxTokens int) *HTTPVisionClient {
if maxTokens <= 0 {
maxTokens = 2048
}
return &HTTPVisionClient{up: up, maxTokens: maxTokens}
}
func (c *HTTPVisionClient) Analyze(ctx context.Context, request AnalysisRequest) (string, error) {
if c == nil || c.up == nil {
return "", fmt.Errorf("visual client is nil")
}
parts := jsonutils.NewArray()
textPart := jsonutils.NewDict()
textPart.Set("type", jsonutils.NewString("text"))
textPart.Set("text", jsonutils.NewString(request.Prompt))
parts.Add(textPart)
for _, image := range request.Images {
if part := chatImagePart(image); part != nil {
parts.Add(part)
}
}
userMsg := jsonutils.NewDict()
userMsg.Set("role", jsonutils.NewString("user"))
userMsg.Set("content", parts)
sysMsg := jsonutils.NewDict()
sysMsg.Set("role", jsonutils.NewString("system"))
sysMsg.Set("content", jsonutils.NewString(visualSystemPrompt))
messages := jsonutils.NewArray(sysMsg, userMsg)
body := jsonutils.NewDict()
body.Set("model", jsonutils.NewString(c.up.UpstreamModel))
body.Set("max_tokens", jsonutils.NewInt(int64(c.maxTokens)))
body.Set("messages", messages)
respBody, err := callChatCompletions(ctx, c.up, body)
if err != nil {
return "", err
}
text := textFromChatResponse(respBody)
if text == "" {
return "", fmt.Errorf("visual provider returned empty content")
}
return text, nil
}
func callChatCompletions(ctx context.Context, up *models.ChatUpstream, body *jsonutils.JSONDict) ([]byte, error) {
if up == nil || body == nil {
return nil, fmt.Errorf("nil upstream or body")
}
req := &upstream.Request{
URL: openai.ChatCompletionsURL(ChatBaseURL(up.BaseURL)),
APIKey: up.APIKey,
Body: []byte(body.String()),
}
resp, uerr := upstream.HTTPDo(ctx, http.MethodPost, req)
if uerr != nil {
return nil, uerr
}
return resp.Body, nil
}
// ChatBaseURL strips Anthropic-compatible path suffixes so chat/completions
// can be built against the OpenAI-compatible root (e.g. deepseek.com vs deepseek.com/anthropic).
func ChatBaseURL(baseURL string) string {
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
if base == "" {
return base
}
lower := strings.ToLower(base)
for _, suffix := range []string{"/api/anthropic", "/anthropic"} {
if strings.HasSuffix(lower, suffix) {
return base[:len(base)-len(suffix)]
}
}
return base
}
func finishReasonFromChatResponse(body []byte) string {
var resp struct {
Choices []struct {
FinishReason string `json:"finish_reason"`
} `json:"choices"`
}
if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 {
return ""
}
return strings.TrimSpace(resp.Choices[0].FinishReason)
}
func toolCallsFromChatResponse(body []byte) []openai.ToolCall {
var resp struct {
Choices []struct {
Message struct {
ToolCalls []openai.ToolCall `json:"tool_calls"`
} `json:"message"`
} `json:"choices"`
}
if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 {
return nil
}
return resp.Choices[0].Message.ToolCalls
}
func assistantMessageFromChatResponse(body []byte) *jsonutils.JSONDict {
var resp struct {
Choices []struct {
Message json.RawMessage `json:"message"`
} `json:"choices"`
}
if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 {
return nil
}
parsed, err := jsonutils.Parse(resp.Choices[0].Message)
if err != nil {
return nil
}
if d, ok := parsed.(*jsonutils.JSONDict); ok {
return d
}
return nil
}
pkg/aiproxy/extensions/visual/config.go
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import api "yunion.io/x/onecloud/pkg/apis/aiproxy"
// RuntimeConfig is the resolved visual extension settings for one request.
type RuntimeConfig struct {
Enabled bool
MaxRounds int
MaxTokens int
}
// RuntimeConfigFromModel extracts visual runtime settings from ai_model config.
func RuntimeConfigFromModel(cfg *api.SAiModelConfig) (RuntimeConfig, *api.SAiModelVisualConfig) {
out := RuntimeConfig{MaxRounds: 4, MaxTokens: 2048}
if cfg == nil || cfg.Extensions == nil || cfg.Extensions.Visual == nil {
return out, nil
}
vis := cfg.Extensions.Visual
out.Enabled = vis.Enabled
if vis.MaxRounds > 0 {
out.MaxRounds = vis.MaxRounds
}
if vis.MaxTokens > 0 {
out.MaxTokens = vis.MaxTokens
}
return out, vis
}
// Enabled reports whether visual extension is enabled on the model config.
func Enabled(cfg *api.SAiModelConfig) bool {
return cfg != nil && cfg.VisualEnabled()
}
pkg/aiproxy/extensions/visual/doc.go
+1
View File
@@ -0,0 +1 @@
package visual // import "yunion.io/x/onecloud/pkg/aiproxy/extensions/visual"
pkg/aiproxy/extensions/visual/images.go
+194
View File
@@ -0,0 +1,194 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"encoding/json"
"fmt"
"strings"
"yunion.io/x/jsonutils"
)
func visualAttachmentText(index int) string {
return fmt.Sprintf("[Image #%d is available to Visual Brief and Visual QA. Use image_refs [\"Image #%d\"] or omit image fields to analyze attached images.]", index, index)
}
// StripImagesFromChat removes image_url parts from chat messages and replaces them with placeholders.
func StripImagesFromChat(body *jsonutils.JSONDict) ([]ImageInput, error) {
if body == nil {
return nil, nil
}
msgsRaw, err := body.Get("messages")
if err != nil {
return nil, nil
}
arr, ok := msgsRaw.(*jsonutils.JSONArray)
if !ok {
return nil, nil
}
available := make([]ImageInput, 0, 2)
for i := 0; i < arr.Size(); i++ {
msgRaw, _ := arr.GetAt(i)
msg, ok := msgRaw.(*jsonutils.JSONDict)
if !ok {
continue
}
contentRaw, err := msg.Get("content")
if err != nil {
continue
}
if contentArr, ok := contentRaw.(*jsonutils.JSONArray); ok {
rewritten := jsonutils.NewArray()
for j := 0; j < contentArr.Size(); j++ {
partRaw, _ := contentArr.GetAt(j)
part, ok := partRaw.(*jsonutils.JSONDict)
if !ok {
rewritten.Add(partRaw)
continue
}
partType, _ := part.GetString("type")
if partType == "image_url" {
if image, ok := imageInputFromChatPart(part); ok {
available = append(available, image)
placeholder := jsonutils.NewDict()
placeholder.Set("type", jsonutils.NewString("text"))
placeholder.Set("text", jsonutils.NewString(visualAttachmentText(len(available))))
rewritten.Add(placeholder)
continue
}
}
rewritten.Add(part)
}
msg.Set("content", rewritten)
continue
}
}
return available, nil
}
func imageInputFromChatPart(part *jsonutils.JSONDict) (ImageInput, bool) {
if part == nil {
return ImageInput{}, false
}
var wire struct {
Type string `json:"type"`
ImageURL json.RawMessage `json:"image_url"`
}
if err := json.Unmarshal([]byte(part.String()), &wire); err != nil {
return ImageInput{}, false
}
if wire.Type != "image_url" {
return ImageInput{}, false
}
var url string
if err := json.Unmarshal(wire.ImageURL, &url); err == nil {
url = strings.TrimSpace(url)
if isSupportedImageURL(url) {
return ImageInput{URL: url}, true
}
}
var obj struct {
URL string `json:"url"`
}
if err := json.Unmarshal(wire.ImageURL, &obj); err == nil {
url = strings.TrimSpace(obj.URL)
if isSupportedImageURL(url) {
return ImageInput{URL: url}, true
}
}
return ImageInput{}, false
}
func isSupportedImageURL(value string) bool {
lower := strings.ToLower(strings.TrimSpace(value))
return strings.HasPrefix(lower, "http://") ||
strings.HasPrefix(lower, "https://") ||
strings.HasPrefix(lower, "data:")
}
func splitDataURL(value string) (mediaType, data string) {
header, payload, ok := strings.Cut(value, ",")
if !ok {
return "", value
}
mediaType = strings.TrimPrefix(header, "data:")
if semicolon := strings.IndexByte(mediaType, ';'); semicolon >= 0 {
mediaType = mediaType[:semicolon]
}
if mediaType == "" {
mediaType = "image/png"
}
return mediaType, payload
}
func chatImagePart(image ImageInput) *jsonutils.JSONDict {
part := jsonutils.NewDict()
part.Set("type", jsonutils.NewString("image_url"))
imgURL := jsonutils.NewDict()
if strings.TrimSpace(image.URL) != "" {
imgURL.Set("url", jsonutils.NewString(strings.TrimSpace(image.URL)))
} else if strings.TrimSpace(image.Data) != "" {
data := strings.TrimSpace(image.Data)
mime := strings.TrimSpace(image.MimeType)
if mime == "" {
mime = "image/png"
}
if strings.HasPrefix(data, "data:") {
imgURL.Set("url", jsonutils.NewString(data))
} else {
imgURL.Set("url", jsonutils.NewString("data:"+mime+";base64,"+data))
}
} else {
return nil
}
part.Set("image_url", imgURL)
return part
}
func textFromChatResponse(body []byte) string {
var resp struct {
Choices []struct {
Message struct {
Content json.RawMessage `json:"content"`
} `json:"message"`
} `json:"choices"`
}
if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 {
return ""
}
raw := resp.Choices[0].Message.Content
var s string
if json.Unmarshal(raw, &s) == nil {
return strings.TrimSpace(s)
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
}
if json.Unmarshal(raw, &parts) == nil {
var b strings.Builder
for _, p := range parts {
if p.Type == "text" && p.Text != "" {
if b.Len() > 0 {
b.WriteByte('\n')
}
b.WriteString(p.Text)
}
}
return strings.TrimSpace(b.String())
}
return ""
}
pkg/aiproxy/extensions/visual/messages.go
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"context"
"encoding/json"
"fmt"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/aiproxy/models"
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
)
var ErrVisualStreamingUnsupported = fmt.Errorf("visual extension does not support streaming")
// AnthropicMessagesHasImage reports whether an Anthropic Messages body carries image blocks.
func AnthropicMessagesHasImage(body *jsonutils.JSONDict) bool {
if body == nil {
return false
}
raw, err := body.Get("messages")
if err != nil {
return false
}
rawBytes := []byte(raw.String())
var messages []struct {
Content json.RawMessage `json:"content"`
}
if json.Unmarshal(rawBytes, &messages) != nil {
return false
}
for _, msg := range messages {
if anthropicContentHasImage(msg.Content) {
return true
}
}
return false
}
func anthropicContentHasImage(raw json.RawMessage) bool {
if len(raw) == 0 || string(raw) == "null" {
return false
}
var blocks []struct {
Type string `json:"type"`
}
if json.Unmarshal(raw, &blocks) != nil {
return false
}
for _, blk := range blocks {
if strings.EqualFold(blk.Type, "image") {
return true
}
}
return false
}
// ShouldHandleMessages reports whether the Messages visual path should run (non-stream only).
func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
return false
}
return AnthropicMessagesHasImage(dict)
}
// ShouldRejectMessagesStreaming reports stream+visual+image (unsupported).
func ShouldRejectMessagesStreaming(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
if !isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
return false
}
return AnthropicMessagesHasImage(dict)
}
// HandleMessagesCreate runs visual orchestration for a non-streaming Anthropic Messages request.
func HandleMessagesCreate(
ctx context.Context,
dict *jsonutils.JSONDict,
textUp *models.ChatUpstream,
) ([]byte, error) {
runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig)
if visCfg == nil {
return nil, fmt.Errorf("visual config is missing")
}
userCred := auth.AdminCredential()
vk, err := models.LoadEnabledVirtualKeyById(textUp.VirtualKeyId)
if err != nil {
return nil, err
}
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg)
if err != nil {
return nil, err
}
chatBody, err := openai.AnthropicToChatCompletions(dict, textUp.UpstreamModel)
if err != nil {
return nil, err
}
chatClone := cloneDict(chatBody)
textUpChat := *textUp
textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL)
visUpChat := *visUp
visUpChat.BaseURL = ChatBaseURL(visUp.BaseURL)
respBody, err := RunChatOrchestrator(ctx, &textUpChat, &visUpChat, chatClone, runtime)
if err != nil {
return nil, err
}
return openai.ChatCompletionToAnthropic(respBody)
}
pkg/aiproxy/extensions/visual/orchestrator.go
+279
View File
@@ -0,0 +1,279 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strconv"
"strings"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/aiproxy/models"
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
)
type briefInput struct {
ImageURL string `json:"image_url,omitempty"`
ImageURLs []string `json:"image_urls,omitempty"`
ImageRefs []string `json:"image_refs,omitempty"`
Images []ImageInput `json:"images,omitempty"`
Context string `json:"context,omitempty"`
Focus string `json:"focus,omitempty"`
}
type qaInput struct {
Question string `json:"question,omitempty"`
ImageURL string `json:"image_url,omitempty"`
ImageURLs []string `json:"image_urls,omitempty"`
ImageRefs []string `json:"image_refs,omitempty"`
Images []ImageInput `json:"images,omitempty"`
PriorVisualContext string `json:"prior_visual_context,omitempty"`
Context string `json:"context,omitempty"`
Conversation []ConversationTurn `json:"conversation,omitempty"`
}
// RunChatOrchestrator executes the visual tool loop on a chat/completions request body.
func RunChatOrchestrator(
ctx context.Context,
textUp, visUp *models.ChatUpstream,
body *jsonutils.JSONDict,
runtime RuntimeConfig,
) ([]byte, error) {
if textUp == nil || visUp == nil || body == nil {
return nil, fmt.Errorf("visual orchestrator: missing upstream or body")
}
availableImages, err := StripImagesFromChat(body)
if err != nil {
return nil, err
}
InjectChatTools(body)
vision := NewHTTPVisionClient(visUp, runtime.MaxTokens)
maxRounds := runtime.MaxRounds
if maxRounds <= 0 {
maxRounds = 4
}
for round := 0; round < maxRounds; round++ {
respBody, err := callChatCompletions(ctx, textUp, body)
if err != nil {
return nil, err
}
finish := finishReasonFromChatResponse(respBody)
if finish != "tool_calls" {
return respBody, nil
}
toolCalls := toolCallsFromChatResponse(respBody)
visualCalls, nonVisual := splitVisualToolCalls(toolCalls)
if len(visualCalls) == 0 {
return respBody, nil
}
assistantMsg := assistantMessageFromChatResponse(respBody)
if assistantMsg == nil {
return respBody, nil
}
msgsRaw, err := body.Get("messages")
if err != nil {
return nil, err
}
msgs, ok := msgsRaw.(*jsonutils.JSONArray)
if !ok {
return nil, fmt.Errorf("messages is not an array")
}
msgs.Add(assistantMsg)
for _, tc := range visualCalls {
result, err := executeVisualToolCall(ctx, vision, tc, availableImages)
if err != nil {
result = "Visual error: " + err.Error()
}
toolMsg := jsonutils.NewDict()
toolMsg.Set("role", jsonutils.NewString("tool"))
toolMsg.Set("tool_call_id", jsonutils.NewString(tc.ID))
toolMsg.Set("content", jsonutils.NewString(result))
msgs.Add(toolMsg)
}
body.Set("messages", msgs)
if len(nonVisual) > 0 {
continue
}
}
return nil, fmt.Errorf("visual loop exceeded max rounds (%d)", maxRounds)
}
func splitVisualToolCalls(calls []openai.ToolCall) (visualCalls, nonVisual []openai.ToolCall) {
for _, tc := range calls {
if IsVisualTool(tc.Function.Name) {
visualCalls = append(visualCalls, tc)
} else {
nonVisual = append(nonVisual, tc)
}
}
return visualCalls, nonVisual
}
func executeVisualToolCall(ctx context.Context, client *HTTPVisionClient, tc openai.ToolCall, available []ImageInput) (string, error) {
request, err := analysisRequestFromToolCall(tc, available)
if err != nil {
return "", err
}
result, err := client.Analyze(ctx, request)
if err != nil {
return "", err
}
switch tc.Function.Name {
case ToolVisualBrief:
return "Visual Brief result:\n" + strings.TrimSpace(result), nil
case ToolVisualQA:
return "Visual QA result:\n" + strings.TrimSpace(result), nil
default:
return strings.TrimSpace(result), nil
}
}
func analysisRequestFromToolCall(tc openai.ToolCall, available []ImageInput) (AnalysisRequest, error) {
switch tc.Function.Name {
case ToolVisualBrief:
var input briefInput
if err := json.Unmarshal([]byte(tc.Function.Arguments), &input); err != nil {
return AnalysisRequest{}, fmt.Errorf("parse visual_brief input: %w", err)
}
images := normalizeImages(input.ImageURL, input.ImageURLs, input.Images, input.ImageRefs, available)
if len(images) == 0 {
return AnalysisRequest{}, fmt.Errorf("visual_brief requires valid image URLs/data/images or attached images")
}
return AnalysisRequest{Tool: ToolVisualBrief, Prompt: buildBriefPrompt(input), Images: images}, nil
case ToolVisualQA:
var input qaInput
if err := json.Unmarshal([]byte(tc.Function.Arguments), &input); err != nil {
return AnalysisRequest{}, fmt.Errorf("parse visual_qa input: %w", err)
}
if strings.TrimSpace(input.Question) == "" {
return AnalysisRequest{}, fmt.Errorf("visual_qa requires question")
}
return AnalysisRequest{
Tool: ToolVisualQA,
Prompt: buildQAPrompt(input),
Images: normalizeImages(input.ImageURL, input.ImageURLs, input.Images, input.ImageRefs, available),
}, nil
default:
return AnalysisRequest{}, fmt.Errorf("unknown visual tool %q", tc.Function.Name)
}
}
func buildBriefPrompt(input briefInput) string {
var b strings.Builder
b.WriteString("Provide a first-round visual brief for the main agent.\n")
if strings.TrimSpace(input.Context) != "" {
b.WriteString("\nTask context:\n")
b.WriteString(strings.TrimSpace(input.Context))
b.WriteByte('\n')
}
if strings.TrimSpace(input.Focus) != "" {
b.WriteString("\nFocus:\n")
b.WriteString(strings.TrimSpace(input.Focus))
b.WriteByte('\n')
}
b.WriteString("\nReturn concise sections: overview, important visual details, any readable text/OCR, uncertainties, and useful Visual QA follow-ups.")
return b.String()
}
func buildQAPrompt(input qaInput) string {
var b strings.Builder
b.WriteString("Answer this targeted visual clarification question for the main agent.\n\nQuestion:\n")
b.WriteString(strings.TrimSpace(input.Question))
b.WriteByte('\n')
if strings.TrimSpace(input.Context) != "" {
b.WriteString("\nTask context:\n")
b.WriteString(strings.TrimSpace(input.Context))
b.WriteByte('\n')
}
if strings.TrimSpace(input.PriorVisualContext) != "" {
b.WriteString("\nPrior visual context:\n")
b.WriteString(strings.TrimSpace(input.PriorVisualContext))
b.WriteByte('\n')
}
b.WriteString("\nAnswer directly, call out uncertainty, and say what extra image/detail would resolve ambiguity.")
return b.String()
}
func normalizeImages(single string, urls []string, images []ImageInput, refs []string, availableImages []ImageInput) []ImageInput {
normalized := make([]ImageInput, 0, len(urls)+len(images)+len(refs)+1)
if image, ok := resolveImageValue(single, availableImages); ok {
normalized = append(normalized, image)
}
for _, url := range urls {
if image, ok := resolveImageValue(url, availableImages); ok {
normalized = append(normalized, image)
}
}
for _, ref := range refs {
if image, ok := resolveAttachedImage(ref, availableImages); ok {
normalized = append(normalized, image)
}
}
for _, image := range images {
if strings.TrimSpace(image.URL) != "" {
if resolved, ok := resolveImageValue(image.URL, availableImages); ok {
if resolved.Detail == "" {
resolved.Detail = image.Detail
}
normalized = append(normalized, resolved)
}
continue
}
if strings.TrimSpace(image.Data) != "" || isSupportedImageURL(image.URL) {
normalized = append(normalized, image)
}
}
if len(normalized) == 0 && len(availableImages) > 0 {
normalized = append(normalized, availableImages...)
}
return normalized
}
func resolveImageValue(value string, availableImages []ImageInput) (ImageInput, bool) {
value = strings.TrimSpace(value)
if value == "" {
return ImageInput{}, false
}
if isSupportedImageURL(value) {
return ImageInput{URL: value}, true
}
return resolveAttachedImage(value, availableImages)
}
func resolveAttachedImage(value string, availableImages []ImageInput) (ImageInput, bool) {
index, ok := imageReferenceIndex(value)
if !ok || index <= 0 || index > len(availableImages) {
return ImageInput{}, false
}
return availableImages[index-1], true
}
var imageReferencePattern = regexp.MustCompile(`(?i)\bimage\s*#\s*(\d+)\b`)
func imageReferenceIndex(value string) (int, bool) {
match := imageReferencePattern.FindStringSubmatch(strings.TrimSpace(value))
if len(match) != 2 {
return 0, false
}
index, err := strconv.Atoi(match[1])
if err != nil {
return 0, false
}
return index, true
}
pkg/aiproxy/extensions/visual/responses.go
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"context"
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/aiproxy/models"
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
)
// ShouldHandle reports whether the Responses visual orchestration path should run.
func ShouldHandle(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
return false
}
return openai.ResponsesInputHasImage(dict)
}
// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request.
func HandleResponsesCreate(
ctx context.Context,
dict *jsonutils.JSONDict,
textUp *models.ChatUpstream,
) ([]byte, *openai.ResponsesConvertState, error) {
runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig)
if visCfg == nil {
return nil, nil, fmt.Errorf("visual config is missing")
}
userCred := auth.AdminCredential()
vk, err := models.LoadEnabledVirtualKeyById(textUp.VirtualKeyId)
if err != nil {
return nil, nil, err
}
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg)
if err != nil {
return nil, nil, err
}
chatBody, state, err := openai.ResponsesToChatCompletions(dict, textUp.UpstreamModel)
if err != nil {
return nil, nil, err
}
chatClone := cloneDict(chatBody)
textUpChat := *textUp
textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL)
visUpChat := *visUp
visUpChat.BaseURL = ChatBaseURL(visUp.BaseURL)
respBody, err := RunChatOrchestrator(ctx, &textUpChat, &visUpChat, chatClone, runtime)
if err != nil {
return nil, nil, err
}
out, err := openai.ChatCompletionToResponses(respBody, state)
if err != nil {
return nil, nil, err
}
return out, state, nil
}
func cloneDict(src *jsonutils.JSONDict) *jsonutils.JSONDict {
if src == nil {
return nil
}
parsed, err := jsonutils.Parse([]byte(src.String()))
if err != nil {
return src
}
if d, ok := parsed.(*jsonutils.JSONDict); ok {
return d
}
return src
}
pkg/aiproxy/extensions/visual/tools.go
+178
View File
@@ -0,0 +1,178 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"encoding/json"
"yunion.io/x/jsonutils"
)
const (
ToolVisualBrief = "visual_brief"
ToolVisualQA = "visual_qa"
)
func visualBriefSchema() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"image_urls": map[string]interface{}{
"type": "array",
"items": map[string]interface{}{"type": "string"},
"description": "HTTP(S) image URLs or data URLs only. Do not put attachment labels like Image #1 here.",
},
"image_refs": map[string]interface{}{
"type": "array",
"items": map[string]interface{}{"type": "string"},
"description": "Attached image labels such as Image #1.",
},
"images": map[string]interface{}{
"type": "array",
"description": "Structured images with url or base64 data plus mime_type.",
"items": imageInputSchema(),
},
"context": map[string]interface{}{
"type": "string",
"description": "User task context that helps decide which visual facts matter.",
},
"focus": map[string]interface{}{
"type": "string",
"description": "Optional focus area such as UI layout, OCR, or chart details.",
},
},
}
}
func visualQASchema() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"question": map[string]interface{}{
"type": "string",
"description": "A specific visual question to answer or clarify.",
},
"image_urls": map[string]interface{}{
"type": "array",
"items": map[string]interface{}{"type": "string"},
},
"image_refs": map[string]interface{}{
"type": "array",
"items": map[string]interface{}{"type": "string"},
},
"images": map[string]interface{}{
"type": "array",
"items": imageInputSchema(),
},
"prior_visual_context": map[string]interface{}{
"type": "string",
},
"context": map[string]interface{}{
"type": "string",
},
"conversation": map[string]interface{}{
"type": "array",
"items": map[string]interface{}{"type": "object"},
},
},
"required": []string{"question"},
}
}
func imageInputSchema() map[string]interface{} {
return map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"url": map[string]interface{}{"type": "string"},
"data": map[string]interface{}{"type": "string"},
"mime_type": map[string]interface{}{"type": "string"},
"detail": map[string]interface{}{"type": "string"},
},
}
}
// ChatTools returns OpenAI chat tool definitions for visual analysis.
func ChatTools() []*jsonutils.JSONDict {
return []*jsonutils.JSONDict{
chatTool(ToolVisualBrief, "Visual Brief. Use this as the first visual pass when image understanding is needed. For attached images, pass image_refs like Image #1 or omit image fields; pass image_urls only for real HTTP(S)/data URLs.", visualBriefSchema()),
chatTool(ToolVisualQA, "Visual QA. Use this after a Visual Brief, or for targeted follow-up questions about an image.", visualQASchema()),
}
}
func chatTool(name, description string, schema map[string]interface{}) *jsonutils.JSONDict {
tool := jsonutils.NewDict()
tool.Set("type", jsonutils.NewString("function"))
fn := jsonutils.NewDict()
fn.Set("name", jsonutils.NewString(name))
fn.Set("description", jsonutils.NewString(description))
schemaBytes, _ := json.Marshal(schema)
params, _ := jsonutils.Parse(schemaBytes)
fn.Set("parameters", params)
tool.Set("function", fn)
return tool
}
// IsVisualTool reports whether name is a visual extension tool.
func IsVisualTool(name string) bool {
return name == ToolVisualBrief || name == ToolVisualQA
}
// InjectChatTools merges visual tools into a chat/completions request body.
func InjectChatTools(body *jsonutils.JSONDict) {
if body == nil {
return
}
existing := map[string]struct{}{}
if toolsRaw, err := body.Get("tools"); err == nil {
if arr, ok := toolsRaw.(*jsonutils.JSONArray); ok {
for i := 0; i < arr.Size(); i++ {
tool, _ := arr.GetAt(i)
if d, ok := tool.(*jsonutils.JSONDict); ok {
if fn, err := d.Get("function"); err == nil {
if fnDict, ok := fn.(*jsonutils.JSONDict); ok {
if name, _ := fnDict.GetString("name"); name != "" {
existing[name] = struct{}{}
}
}
}
}
}
}
}
tools := jsonutils.NewArray()
if toolsRaw, err := body.Get("tools"); err == nil {
if arr, ok := toolsRaw.(*jsonutils.JSONArray); ok {
for i := 0; i < arr.Size(); i++ {
item, _ := arr.GetAt(i)
tools.Add(item)
}
}
}
for _, tool := range ChatTools() {
if fn, err := tool.Get("function"); err == nil {
if fnDict, ok := fn.(*jsonutils.JSONDict); ok {
if name, _ := fnDict.GetString("name"); name != "" {
if _, ok := existing[name]; ok {
continue
}
}
}
}
tools.Add(tool)
}
if tools.Length() > 0 {
body.Set("tools", tools)
}
}
pkg/aiproxy/extensions/visual/types.go
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import "context"
// ImageInput carries image data as URL, base64 data, or data URL.
type ImageInput struct {
URL string `json:"url,omitempty"`
Data string `json:"data,omitempty"`
MimeType string `json:"mime_type,omitempty"`
Detail string `json:"detail,omitempty"`
}
// ConversationTurn is one visual clarification history entry.
type ConversationTurn struct {
Role string `json:"role,omitempty"`
Content string `json:"content,omitempty"`
}
// AnalysisRequest bundles the analysis prompt with images for the vision model.
type AnalysisRequest struct {
Tool string
Prompt string
Images []ImageInput
}
// VisionClient analyzes images with a short prompt.
type VisionClient interface {
Analyze(context.Context, AnalysisRequest) (string, error)
}
pkg/aiproxy/extensions/visual/visual_test.go
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package visual
import (
"strings"
"testing"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/aiproxy/models"
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
)
func TestStripImagesFromChat(t *testing.T) {
body := jsonutils.NewDict()
msgs := jsonutils.NewArray()
user := jsonutils.NewDict()
user.Set("role", jsonutils.NewString("user"))
parts := jsonutils.NewArray()
text := jsonutils.NewDict()
text.Set("type", jsonutils.NewString("text"))
text.Set("text", jsonutils.NewString("describe"))
parts.Add(text)
img := jsonutils.NewDict()
img.Set("type", jsonutils.NewString("image_url"))
imgURL := jsonutils.NewDict()
imgURL.Set("url", jsonutils.NewString("data:image/png;base64,abc"))
img.Set("image_url", imgURL)
parts.Add(img)
user.Set("content", parts)
msgs.Add(user)
body.Set("messages", msgs)
images, err := StripImagesFromChat(body)
if err != nil {
t.Fatal(err)
}
if len(images) != 1 {
t.Fatalf("images = %d", len(images))
}
s := body.String()
if strings.Contains(s, "image_url") {
t.Fatalf("expected stripped image_url, got %s", s)
}
if !strings.Contains(s, "Image #1") {
t.Fatalf("expected placeholder, got %s", s)
}
}
func TestEnabled(t *testing.T) {
cfg := &api.SAiModelConfig{
Extensions: &api.SAiModelExtensions{
Visual: &api.SAiModelVisualConfig{Enabled: true},
},
}
if !Enabled(cfg) {
t.Fatal("expected enabled")
}
}
func TestInjectChatTools(t *testing.T) {
body := jsonutils.NewDict()
InjectChatTools(body)
tools, err := body.Get("tools")
if err != nil {
t.Fatal(err)
}
arr, ok := tools.(*jsonutils.JSONArray)
if !ok || arr.Size() < 2 {
t.Fatalf("tools = %v", tools)
}
}
func TestNormalizeImagesUsesAttached(t *testing.T) {
available := []ImageInput{{URL: "data:image/png;base64,abc"}}
images := normalizeImages("", nil, nil, []string{"Image #1"}, available)
if len(images) != 1 || images[0].URL == "" {
t.Fatalf("images = %#v", images)
}
}
func TestChatBaseURLStripsAnthropicSuffix(t *testing.T) {
cases := []struct {
in, want string
}{
{"https://api.deepseek.com/anthropic", "https://api.deepseek.com"},
{"https://api.deepseek.com/anthropic/", "https://api.deepseek.com"},
{"https://open.bigmodel.cn/api/anthropic", "https://open.bigmodel.cn"},
{"https://api.moonshot.cn", "https://api.moonshot.cn"},
}
for _, c := range cases {
if got := ChatBaseURL(c.in); got != c.want {
t.Fatalf("ChatBaseURL(%q)=%q want %q", c.in, got, c.want)
}
}
}
func TestAnthropicMessagesHasImage(t *testing.T) {
body, _ := jsonutils.Parse([]byte(`{"messages":[{"role":"user","content":[{"type":"image","source":{"type":"url","url":"https://x/a.png"}}]}]}`))
if !AnthropicMessagesHasImage(body.(*jsonutils.JSONDict)) {
t.Fatal("expected image")
}
body2, _ := jsonutils.Parse([]byte(`{"messages":[{"role":"user","content":"hello"}]}`))
if AnthropicMessagesHasImage(body2.(*jsonutils.JSONDict)) {
t.Fatal("expected no image")
}
}
func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) {
cfg := &api.SAiModelConfig{
Extensions: &api.SAiModelExtensions{
Visual: &api.SAiModelVisualConfig{Enabled: true},
},
}
up := &models.ChatUpstream{
ModelConfig: cfg,
APIMode: "anthropic",
}
body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`))
if !ShouldHandle(body.(*jsonutils.JSONDict), up, false) {
t.Fatal("expected visual handle even with anthropic api_mode")
}
if ShouldHandle(body.(*jsonutils.JSONDict), up, true) {
t.Fatal("stream should not use visual orchestrator")
}
}
pkg/aiproxy/handlers/messages.go
+23
View File
@@ -24,6 +24,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/aiproxy/extensions/visual"
"yunion.io/x/onecloud/pkg/aiproxy/models"
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
"yunion.io/x/onecloud/pkg/aiproxy/providers"
@@ -91,6 +92,28 @@ func messagesHandler(ctx context.Context, w http.ResponseWriter, r *http.Request
return
}
if visual.ShouldRejectMessagesStreaming(dict, up, isStream) {
dbg.Error("%v", visual.ErrVisualStreamingUnsupported)
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", visual.ErrVisualStreamingUnsupported)
return
}
if visual.ShouldHandleMessages(dict, up, isStream) {
bodyOut, err := visual.HandleMessagesCreate(ctx, dict, up)
if err != nil {
dbg.Error("visual orchestration: %v", err)
writeAnthropicError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err)
return
}
dbg.ClientResponse(bodyOut)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(bodyOut)
if up.AiKeyId != "" {
models.RecordAiKeySuccess(up.AiKeyId)
}
return
}
adapter, err := messages.GetAdapter(up.ProviderKey, up.APIMode)
if err != nil {
dbg.Error("adapter: %v", err)
pkg/aiproxy/handlers/responses.go
+18
View File
@@ -26,6 +26,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/aiproxy/extensions/visual"
"yunion.io/x/onecloud/pkg/aiproxy/models"
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
"yunion.io/x/onecloud/pkg/aiproxy/providers"
@@ -104,6 +105,23 @@ func handleResponsesCreate(ctx context.Context, w http.ResponseWriter, r *http.R
return
}
if visual.ShouldHandle(dict, up, isStream) {
bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up)
if err != nil {
dbg.Error("visual orchestration: %v", err)
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err)
return
}
dbg.ClientResponse(bodyOut)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(bodyOut)
if up.AiKeyId != "" {
models.RecordAiKeySuccess(up.AiKeyId)
}
return
}
adapter, err := responses.GetAdapter(up.ProviderKey, up.APIMode)
if err != nil {
dbg.Error("adapter: %v", err)
pkg/aiproxy/models/ai_models.go
+4
View File
@@ -36,6 +36,8 @@ type SAiModel struct {
AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
// ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo).
ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"`
// Config stores per-model extension settings (e.g. visual delegation).
Config *api.SAiModelConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"`
}
type SAiModelManager struct {
@@ -138,6 +140,8 @@ func (manager *SAiModelManager) FetchCustomizeColumns(
}
for i := range rows {
rows[i].AiProviderName, _ = providerNames[providerIds[i]]
m := objs[i].(*SAiModel)
rows[i].Config = m.Config
}
return rows
}
pkg/aiproxy/models/chat_upstream.go
+69
View File
@@ -24,6 +24,7 @@ import (
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/rbacscope"
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
@@ -67,8 +68,10 @@ type ChatUpstream struct {
UpstreamModel string
ProviderKey string
AiProviderId string
AiModelId string
AiKeyId string
APIMode string
ModelConfig *api.SAiModelConfig
// VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row.
VirtualKeyId string
@@ -150,6 +153,24 @@ func loadEnabledVirtualKey(virtualKey string) (*SAiVirtualKey, error) {
return &vk, nil
}
// LoadEnabledVirtualKeyById loads an enabled virtual key by database id.
func LoadEnabledVirtualKeyById(id string) (*SAiVirtualKey, error) {
id = strings.TrimSpace(id)
if id == "" {
return nil, errors.Wrap(httperrors.ErrInputParameter, "empty virtual key id")
}
vk := SAiVirtualKey{}
qvk := AiVirtualKeyManager.Query().Equals("id", id).Equals("enabled", true)
err := qvk.First(&vk)
if err != nil {
if stderrors.Is(err, sql.ErrNoRows) {
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "virtual key not found or disabled")
}
return nil, errors.Wrap(err, "query ai_virtual_key by id")
}
return &vk, nil
}
// listProjectRoutingsForVirtualKey returns enabled ai_routing rows owned by or shared with the virtual key's project.
func listProjectRoutingsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey) ([]SAiRouting, error) {
if vk == nil {
@@ -330,9 +351,11 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
UpstreamModel: upstreamModel,
ProviderKey: prov.ProviderKey,
AiProviderId: prov.Id,
AiModelId: mdl.Id,
AiKeyId: keyRes.AiKeyId,
VirtualKeyId: vk.Id,
APIMode: apiMode,
ModelConfig: mdl.Config,
ProjectId: vk.ProjectId,
DomainId: vk.DomainId,
RoutingLog: resolved.routingLog,
@@ -343,3 +366,49 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
}
return up, nil
}
// ResolveVisualUpstream resolves the visual provider upstream for tool-delegated image analysis.
func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, cfg *api.SAiModelVisualConfig) (*ChatUpstream, error) {
if cfg == nil || !cfg.Enabled {
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual extension is not enabled")
}
providerID := strings.TrimSpace(cfg.VisualAiProviderId)
modelKey := strings.TrimSpace(cfg.VisualModelKey)
if providerID == "" || modelKey == "" {
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_ai_provider_id and visual_model_key are required")
}
pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerID)
if err != nil {
return nil, errors.Wrap(err, "fetch visual ai_provider")
}
prov := pObj.(*SAiProvider)
if !prov.GetEnabled() {
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider disabled")
}
if !virtualKeyAllowsProvider(vk, prov) {
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider not allowed for this virtual key")
}
if prov.Config == nil {
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider.config is empty")
}
baseURL := prov.Config.EffectiveBaseURL(prov.ProviderKey)
if baseURL == "" {
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider.config must include base_url")
}
keyRes, err := resolveUpstreamAPIKey(prov, modelKey)
if err != nil {
return nil, err
}
if keyRes == nil || keyRes.Secret == "" {
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "add an enabled ai_key with secret for visual provider")
}
return &ChatUpstream{
BaseURL: baseURL,
APIKey: keyRes.Secret,
UpstreamModel: modelKey,
ProviderKey: prov.ProviderKey,
AiProviderId: prov.Id,
AiKeyId: keyRes.AiKeyId,
APIMode: prov.Config.ResolvedAPIMode(),
}, nil
}
pkg/aiproxy/models/list_models.go
+23 -22
View File
@@ -27,10 +27,11 @@ import (
// ModelsListEntry is one OpenAI-compatible model object in GET /openai/v1/models.
type ModelsListEntry struct {
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
OwnedBy string `json:"owned_by"`
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
OwnedBy string `json:"owned_by"`
InputModalities []string `json:"input_modalities,omitempty"`
}
// ListModelsForVirtualKey returns OpenAI-compatible model ids reachable by the virtual key
@@ -113,12 +114,7 @@ func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredent
if _, ok := seen[id]; ok {
continue
}
seen[id] = ModelsListEntry{
ID: id,
Object: "model",
Created: created,
OwnedBy: firstProviderByRouting[routing.Id],
}
seen[id] = modelsListEntryFromModel(id, firstProviderByRouting[routing.Id], created, nil)
}
// Pass 2: ai_routing_model entries as flat or hierarchical client-facing ids.
@@ -145,12 +141,7 @@ func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredent
if _, ok := seen[id]; ok {
continue
}
seen[id] = ModelsListEntry{
ID: id,
Object: "model",
Created: created,
OwnedBy: strings.TrimSpace(prov.ProviderKey),
}
seen[id] = modelsListEntryFromModel(id, strings.TrimSpace(prov.ProviderKey), created, mdl)
continue
}
id := clientFacingModelID(routing, e, mdl)
@@ -160,12 +151,7 @@ func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredent
if _, ok := seen[id]; ok {
continue
}
seen[id] = ModelsListEntry{
ID: id,
Object: "model",
Created: created,
OwnedBy: strings.TrimSpace(prov.ProviderKey),
}
seen[id] = modelsListEntryFromModel(id, strings.TrimSpace(prov.ProviderKey), created, mdl)
}
if len(seen) == 0 {
return nil, nil
@@ -306,3 +292,18 @@ func uniqueNonEmptyStrings(in []string) []string {
}
return out
}
func modelsListEntryFromModel(id, ownedBy string, created int64, mdl *SAiModel) ModelsListEntry {
entry := ModelsListEntry{
ID: id,
Object: "model",
Created: created,
OwnedBy: ownedBy,
}
if mdl != nil && mdl.Config != nil {
if mods := mdl.Config.InputModalitiesForCatalog(); len(mods) > 0 {
entry.InputModalities = mods
}
}
return entry
}
pkg/aiproxy/options/options.go
+11 -11
View File
@@ -25,17 +25,17 @@ type SAiProxyOptions struct {
AdvertiseAddress string `help:"Standby node address advertised to clients, e.g. http://10.0.0.2:30889; default derives from bind address and port" default:""`
NodeHeartbeatIntervalSeconds int `help:"Interval in seconds for standby node registration heartbeat" default:"60"`
ChatLogEnabled bool `help:"Enable OpenAI chat request JSONL logs" default:"true"`
ChatLogLocalDir string `help:"Local directory for chat JSONL logs" default:"/tmp/aiproxy-chatlog"`
ChatLogUploadEnabled bool `help:"Upload closed chat log hour files to MinIO/S3" default:"true"`
ChatLogUploadIntervalSeconds int `help:"Chat log upload interval in seconds" default:"10"`
ChatLogSegmentMinutes int `help:"Chat log file segment duration in minutes" default:"60"`
ChatLogMinioEndpoint string `help:"MinIO/S3 endpoint for chat log upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"`
ChatLogMinioAccessKey string `help:"MinIO/S3 access key for chat log upload" default:"monitor-admin"`
ChatLogMinioSecretKey string `help:"MinIO/S3 secret key for chat log upload" default:"monitor-admin"`
ChatLogMinioBucket string `help:"MinIO/S3 bucket for chat log upload" default:"aiproxy-chat"`
ChatLogMinioSecure bool `help:"Use HTTPS for MinIO/S3 endpoint without scheme" default:"false"`
ChatLogMinioPrefix string `help:"MinIO/S3 object key prefix for chat logs" default:""`
APILogEnabled bool `help:"Enable OpenAI API request JSONL logs" default:"true"`
APILogLocalDir string `help:"Local directory for API JSONL logs" default:"/tmp/aiproxy-apilog"`
APILogUploadEnabled bool `help:"Upload closed API log hour files to S3" default:"true"`
APILogUploadIntervalSeconds int `help:"API log upload interval in seconds" default:"10"`
APILogSegmentMinutes int `help:"API log file segment duration in minutes" default:"60"`
APILogS3Endpoint string `help:"S3 endpoint for API log upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"`
APILogS3AccessKey string `help:"S3 access key for API log upload" default:"monitor-admin"`
APILogS3SecretKey string `help:"S3 secret key for API log upload" default:"monitor-admin"`
APILogS3Bucket string `help:"S3 bucket for API log upload" default:"aiproxy-log"`
APILogS3Secure bool `help:"Use HTTPS for S3 endpoint without scheme" default:"false"`
APILogS3Prefix string `help:"S3 object key prefix for API logs" default:""`
}
var (
pkg/aiproxy/providers/openai/anthropic_compat.go
+67 -12
View File
@@ -221,14 +221,21 @@ func parseAnthropicUserContent(raw json.RawMessage) (jsonutils.JSONObject, []ant
if err := json.Unmarshal(raw, &blocks); err != nil {
return nil, nil, fmt.Errorf("invalid user content: %w", err)
}
var textParts []string
parts := jsonutils.NewArray()
var tools []anthropicToolResult
for _, blk := range blocks {
typ, _ := blk["type"].(string)
switch typ {
case "text":
if t, _ := blk["text"].(string); t != "" {
textParts = append(textParts, t)
part := jsonutils.NewDict()
part.Set("type", jsonutils.NewString("text"))
part.Set("text", jsonutils.NewString(t))
parts.Add(part)
}
case "image":
if part := anthropicImageBlockToChatPart(blk); part != nil {
parts.Add(part)
}
case "tool_result":
id, _ := blk["tool_use_id"].(string)
@@ -236,22 +243,70 @@ func parseAnthropicUserContent(raw json.RawMessage) (jsonutils.JSONObject, []ant
tools = append(tools, anthropicToolResult{ID: id, Content: content})
}
}
if len(textParts) == 0 {
if parts.Size() == 0 {
return nil, tools, nil
}
if len(textParts) == 1 {
return jsonutils.NewString(textParts[0]), tools, nil
}
parts := jsonutils.NewArray()
for _, p := range textParts {
blk := jsonutils.NewDict()
blk.Set("type", jsonutils.NewString("text"))
blk.Set("text", jsonutils.NewString(p))
parts.Add(blk)
// Single text-only part can stay a plain string for compatibility.
if parts.Size() == 1 {
first, _ := parts.GetAt(0)
if d, ok := first.(*jsonutils.JSONDict); ok {
if typ, _ := d.GetString("type"); typ == "text" {
if text, _ := d.GetString("text"); text != "" {
return jsonutils.NewString(text), tools, nil
}
}
}
}
return parts, tools, nil
}
func anthropicImageBlockToChatPart(blk map[string]interface{}) *jsonutils.JSONDict {
srcRaw, ok := blk["source"].(map[string]interface{})
if !ok || srcRaw == nil {
return nil
}
srcType, _ := srcRaw["type"].(string)
var url string
switch strings.ToLower(strings.TrimSpace(srcType)) {
case "url":
url, _ = srcRaw["url"].(string)
case "base64":
data, _ := srcRaw["data"].(string)
mediaType, _ := srcRaw["media_type"].(string)
if strings.TrimSpace(data) == "" {
return nil
}
if mediaType == "" {
mediaType = "image/png"
}
if strings.HasPrefix(data, "data:") {
url = data
} else {
url = "data:" + mediaType + ";base64," + data
}
default:
if u, _ := srcRaw["url"].(string); strings.TrimSpace(u) != "" {
url = u
} else if data, _ := srcRaw["data"].(string); strings.TrimSpace(data) != "" {
mediaType, _ := srcRaw["media_type"].(string)
if mediaType == "" {
mediaType = "image/png"
}
url = "data:" + mediaType + ";base64," + data
}
}
url = strings.TrimSpace(url)
if url == "" {
return nil
}
imgURL := jsonutils.NewDict()
imgURL.Set("url", jsonutils.NewString(url))
part := jsonutils.NewDict()
part.Set("type", jsonutils.NewString("image_url"))
part.Set("image_url", imgURL)
return part
}
func anthropicBlockContentText(v interface{}) string {
switch c := v.(type) {
case string:
pkg/aiproxy/providers/openai/anthropic_compat_test.go
+52
View File
@@ -16,6 +16,7 @@ package openai
import (
"encoding/json"
"strings"
"testing"
"yunion.io/x/jsonutils"
@@ -345,3 +346,54 @@ func TestAnthropicToolRoundTrip(t *testing.T) {
t.Fatalf("stop_reason: %#v", resp["stop_reason"])
}
}
func TestAnthropicToChatCompletionsWithImage(t *testing.T) {
raw := `{
"model":"vision-model",
"max_tokens":256,
"messages":[{"role":"user","content":[
{"type":"text","text":"what is this"},
{"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc123"}}
]}]
}`
body, err := jsonutils.Parse([]byte(raw))
if err != nil {
t.Fatal(err)
}
out, err := AnthropicToChatCompletions(body.(*jsonutils.JSONDict), "vision-model")
if err != nil {
t.Fatal(err)
}
s := out.String()
if !strings.Contains(s, "image_url") {
t.Fatalf("expected image_url, got %s", s)
}
if !strings.Contains(s, "data:image/png;base64,abc123") {
t.Fatalf("expected data url, got %s", s)
}
if !strings.Contains(s, "what is this") {
t.Fatalf("expected text, got %s", s)
}
}
func TestAnthropicToChatCompletionsImageOnly(t *testing.T) {
raw := `{
"model":"vision-model",
"max_tokens":256,
"messages":[{"role":"user","content":[
{"type":"image","source":{"type":"url","url":"https://example.com/a.png"}}
]}]
}`
body, err := jsonutils.Parse([]byte(raw))
if err != nil {
t.Fatal(err)
}
out, err := AnthropicToChatCompletions(body.(*jsonutils.JSONDict), "vision-model")
if err != nil {
t.Fatal(err)
}
s := out.String()
if !strings.Contains(s, "image_url") || !strings.Contains(s, "https://example.com/a.png") {
t.Fatalf("expected image-only message preserved, got %s", s)
}
}
pkg/aiproxy/providers/openai/responses_anthropic_compat.go
+31 -4
View File
@@ -177,14 +177,41 @@ func chatMessagesToAnthropic(chatMsgs []*jsonutils.JSONDict) (*jsonutils.JSONArr
asst.Set("content", blocks)
out.Add(asst)
default:
content, _ := msg.GetString("content")
contentRaw, _ := msg.Get("content")
var contentBytes []byte
if contentRaw != nil {
contentBytes = []byte(contentRaw.String())
}
blocks := ChatContentToAnthropicBlocks(contentBytes)
user := jsonutils.NewDict()
user.Set("role", jsonutils.NewString("user"))
if content != "" {
user.Set("content", jsonutils.NewString(content))
} else {
if len(blocks) == 0 {
user.Set("content", jsonutils.NewString(""))
out.Add(user)
continue
}
if len(blocks) == 1 {
if typ, _ := blocks[0]["type"].(string); typ == "text" {
if text, _ := blocks[0]["text"].(string); text != "" {
user.Set("content", jsonutils.NewString(text))
out.Add(user)
continue
}
}
}
arr := jsonutils.NewArray()
for _, blk := range blocks {
data, err := json.Marshal(blk)
if err != nil {
continue
}
obj, err := jsonutils.Parse(data)
if err != nil {
continue
}
arr.Add(obj)
}
user.Set("content", arr)
out.Add(user)
}
}
pkg/aiproxy/providers/openai/responses_anthropic_compat_test.go
+26
View File
@@ -97,3 +97,29 @@ func TestAnthropicMessagesToResponsesBasic(t *testing.T) {
t.Fatalf("out=%s", out)
}
}
func TestResponsesToAnthropicMessagesWithImage(t *testing.T) {
raw := `{
"model":"claude-test",
"max_output_tokens":128,
"input":[{"role":"user","content":[
{"type":"input_text","text":"look"},
{"type":"input_image","image_url":"https://example.com/x.png"}
]}]
}`
body, err := jsonutils.Parse([]byte(raw))
if err != nil {
t.Fatal(err)
}
out, _, err := ResponsesToAnthropicMessages(body.(*jsonutils.JSONDict), "claude-up")
if err != nil {
t.Fatal(err)
}
s := out.String()
if !strings.Contains(s, `"type":"image"`) && !strings.Contains(s, `"type": "image"`) {
t.Fatalf("expected anthropic image block, got %s", s)
}
if !strings.Contains(s, "https://example.com/x.png") {
t.Fatalf("expected image url, got %s", s)
}
}
pkg/aiproxy/providers/openai/responses_compat.go
+122 -8
View File
@@ -261,18 +261,12 @@ func responsesInputToMessages(body *jsonutils.JSONDict) ([]*jsonutils.JSONDict,
case "assistant":
msg := jsonutils.NewDict()
msg.Set("role", jsonutils.NewString("assistant"))
if text := responsesContentToText(item.Content); text != "" {
msg.Set("content", jsonutils.NewString(text))
}
setResponsesMessageContent(msg, item.Content)
messages = append(messages, msg)
default:
msg := jsonutils.NewDict()
msg.Set("role", jsonutils.NewString("user"))
if text := responsesContentToText(item.Content); text != "" {
msg.Set("content", jsonutils.NewString(text))
} else {
msg.Set("content", jsonutils.NewString(""))
}
setResponsesMessageContent(msg, item.Content)
messages = append(messages, msg)
}
}
@@ -289,6 +283,126 @@ func isResponsesToolOutputType(t string) bool {
}
}
type responsesContentPartRaw struct {
Type string `json:"type"`
Text string `json:"text"`
ImageURL json.RawMessage `json:"image_url"`
}
// ResponsesInputHasImage reports whether a Responses request input carries image parts.
func ResponsesInputHasImage(body *jsonutils.JSONDict) bool {
if body == nil {
return false
}
raw, err := body.Get("input")
if err != nil {
return false
}
rawBytes := []byte(raw.String())
trimmed := strings.TrimSpace(string(rawBytes))
if trimmed == "" || trimmed == "null" || strings.HasPrefix(trimmed, "\"") {
return false
}
if !strings.HasPrefix(trimmed, "[") {
return false
}
var items []responsesInputItem
if json.Unmarshal(rawBytes, &items) != nil {
return false
}
for _, item := range items {
if responsesContentHasImage(item.Content) {
return true
}
}
return false
}
func responsesContentHasImage(raw json.RawMessage) bool {
_, hasImage := responsesContentToChatParts(raw)
return hasImage
}
func setResponsesMessageContent(msg *jsonutils.JSONDict, raw json.RawMessage) {
parts, hasImage := responsesContentToChatParts(raw)
if hasImage {
arr := jsonutils.NewArray()
for _, part := range parts {
arr.Add(part)
}
msg.Set("content", arr)
return
}
text := responsesContentToText(raw)
msg.Set("content", jsonutils.NewString(text))
}
// responsesContentToChatParts converts Responses content to chat/completions content parts.
// Returns parts and whether any image_url part was included.
func responsesContentToChatParts(raw json.RawMessage) ([]*jsonutils.JSONDict, bool) {
if len(raw) == 0 || string(raw) == "null" {
return nil, false
}
var s string
if json.Unmarshal(raw, &s) == nil {
if strings.TrimSpace(s) == "" {
return nil, false
}
part := jsonutils.NewDict()
part.Set("type", jsonutils.NewString("text"))
part.Set("text", jsonutils.NewString(s))
return []*jsonutils.JSONDict{part}, false
}
var parts []responsesContentPartRaw
if json.Unmarshal(raw, &parts) != nil {
return nil, false
}
out := make([]*jsonutils.JSONDict, 0, len(parts))
hasImage := false
for _, p := range parts {
switch p.Type {
case "input_text", "text", "output_text":
if p.Text == "" {
continue
}
part := jsonutils.NewDict()
part.Set("type", jsonutils.NewString("text"))
part.Set("text", jsonutils.NewString(p.Text))
out = append(out, part)
case "input_image", "image", "image_url":
src := imageSourceFromRaw(p.ImageURL)
if src == "" {
continue
}
hasImage = true
imgURL := jsonutils.NewDict()
imgURL.Set("url", jsonutils.NewString(src))
part := jsonutils.NewDict()
part.Set("type", jsonutils.NewString("image_url"))
part.Set("image_url", imgURL)
out = append(out, part)
}
}
return out, hasImage
}
func imageSourceFromRaw(raw json.RawMessage) string {
if len(raw) == 0 || string(raw) == "null" {
return ""
}
var url string
if err := json.Unmarshal(raw, &url); err == nil {
return strings.TrimSpace(url)
}
var obj struct {
URL string `json:"url"`
}
if err := json.Unmarshal(raw, &obj); err == nil {
return strings.TrimSpace(obj.URL)
}
return ""
}
func responsesContentToText(raw json.RawMessage) string {
if len(raw) == 0 || string(raw) == "null" {
return ""
pkg/aiproxy/providers/openai/responses_compat_test.go
+77
View File
@@ -130,6 +130,83 @@ func TestFlattenResponsesNamespaceTool(t *testing.T) {
}
}
func TestResponsesToChatCompletionsWithImageInput(t *testing.T) {
raw := `{
"model":"kimi-k2.6",
"input":[{"role":"user","content":[
{"type":"input_text","text":"describe this image"},
{"type":"input_image","image_url":"data:image/png;base64,abc123"}
]}]
}`
body, _ := jsonutils.Parse([]byte(raw))
out, _, err := ResponsesToChatCompletions(body.(*jsonutils.JSONDict), "kimi-k2.6")
if err != nil {
t.Fatal(err)
}
s := out.String()
if !strings.Contains(s, "image_url") {
t.Fatalf("expected image_url in messages, got %s", s)
}
if !strings.Contains(s, "data:image/png;base64,abc123") {
t.Fatalf("expected image data url in messages, got %s", s)
}
if !strings.Contains(s, "describe this image") {
t.Fatalf("expected text in messages, got %s", s)
}
}
func TestResponsesInputHasImage(t *testing.T) {
body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`))
if !ResponsesInputHasImage(body.(*jsonutils.JSONDict)) {
t.Fatal("expected image input")
}
body2, _ := jsonutils.Parse([]byte(`{"input":"hello"}`))
if ResponsesInputHasImage(body2.(*jsonutils.JSONDict)) {
t.Fatal("expected no image for string input")
}
}
func TestResponsesStreamConverterReasoningAndToolOutputIndex(t *testing.T) {
conv := NewResponsesStreamConverter("kimi-k2.6", nil)
chunk1 := []byte(`{"id":"chatcmpl-1","model":"kimi-k2.6","choices":[{"delta":{"reasoning_content":"think"}}]}`)
events1, err := conv.Feed(chunk1, false)
if err != nil {
t.Fatal(err)
}
chunk2 := []byte(`{"id":"chatcmpl-1","model":"kimi-k2.6","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"view_image","arguments":"{}"}}]}}]}`)
events2, err := conv.Feed(chunk2, false)
if err != nil {
t.Fatal(err)
}
events := append(events1, events2...)
var addedIndex, deltaIndex int
for _, e := range events {
if e.Event != "response.output_item.added" && e.Event != "response.function_call_arguments.delta" {
continue
}
var payload map[string]interface{}
if err := json.Unmarshal(e.Data, &payload); err != nil {
t.Fatal(err)
}
idx, _ := payload["output_index"].(float64)
if e.Event == "response.output_item.added" {
item, _ := payload["item"].(map[string]interface{})
if item["type"] == "function_call" {
addedIndex = int(idx)
}
}
if e.Event == "response.function_call_arguments.delta" {
deltaIndex = int(idx)
}
}
if addedIndex == 0 || deltaIndex == 0 {
t.Fatalf("expected function_call indices, added=%d delta=%d events=%+v", addedIndex, deltaIndex, events)
}
if addedIndex != deltaIndex {
t.Fatalf("output_index mismatch: added=%d delta=%d", addedIndex, deltaIndex)
}
}
func TestResponsesStreamConverterText(t *testing.T) {
conv := NewResponsesStreamConverter("gpt-test", nil)
chunk := []byte(`{"id":"chatcmpl-1","model":"gpt-test","choices":[{"delta":{"content":"Hi"}}]}`)
pkg/aiproxy/providers/openai/responses_stream_compat.go
+6 -2
View File
@@ -289,7 +289,7 @@ func (s *ResponsesStreamConverter) appendToolDelta(tc ToolCall) ([]ResponsesStre
if ns != "" {
item["namespace"] = ns
}
added, err := s.outputItemAdded("function_call", st.itemID, item)
added, err := s.outputItemAddedAt(st.outputIndex, "function_call", st.itemID, item)
if err != nil {
return nil, err
}
@@ -313,10 +313,14 @@ func (s *ResponsesStreamConverter) appendToolDelta(tc ToolCall) ([]ResponsesStre
}
func (s *ResponsesStreamConverter) outputItemAdded(itemType, itemID string, item map[string]interface{}) (ResponsesStreamEvent, error) {
return s.outputItemAddedAt(s.outputIndex, itemType, itemID, item)
}
func (s *ResponsesStreamConverter) outputItemAddedAt(outputIndex int, itemType, itemID string, item map[string]interface{}) (ResponsesStreamEvent, error) {
data := map[string]interface{}{
"type": "response.output_item.added",
"sequence_number": s.nextSeq(),
"output_index": s.outputIndex,
"output_index": outputIndex,
"item": item,
}
b, err := json.Marshal(data)
pkg/aiproxy/providers/openai/tools.go
+86 -12
View File
@@ -168,26 +168,22 @@ func MessagesToAnthropic(msgs []Message) ([]map[string]interface{}, error) {
},
})
case "user":
text := MessageTextContent(m.Content)
if text == "" {
blocks := ChatContentToAnthropicBlocks(m.Content)
if len(blocks) == 0 {
continue
}
out = append(out, map[string]interface{}{
"role": "user",
"content": []map[string]interface{}{
{"type": "text", "text": text},
},
"role": "user",
"content": blocks,
})
default:
text := MessageTextContent(m.Content)
if text == "" {
blocks := ChatContentToAnthropicBlocks(m.Content)
if len(blocks) == 0 {
continue
}
out = append(out, map[string]interface{}{
"role": role,
"content": []map[string]interface{}{
{"type": "text", "text": text},
},
"role": role,
"content": blocks,
})
}
}
@@ -197,6 +193,84 @@ func MessagesToAnthropic(msgs []Message) ([]map[string]interface{}, error) {
return out, nil
}
// ChatContentToAnthropicBlocks converts OpenAI chat message content (string or parts)
// into Anthropic content blocks, preserving image_url as image sources.
func ChatContentToAnthropicBlocks(raw json.RawMessage) []map[string]interface{} {
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
if strings.TrimSpace(s) == "" {
return nil
}
return []map[string]interface{}{{"type": "text", "text": s}}
}
var parts []struct {
Type string `json:"type"`
Text string `json:"text"`
ImageURL json.RawMessage `json:"image_url"`
}
if err := json.Unmarshal(raw, &parts); err != nil {
text := MessageTextContent(raw)
if text == "" {
return nil
}
return []map[string]interface{}{{"type": "text", "text": text}}
}
out := make([]map[string]interface{}, 0, len(parts))
for _, p := range parts {
switch p.Type {
case "text":
if p.Text == "" {
continue
}
out = append(out, map[string]interface{}{"type": "text", "text": p.Text})
case "image_url":
if block := chatImageURLToAnthropicBlock(p.ImageURL); block != nil {
out = append(out, block)
}
}
}
return out
}
func chatImageURLToAnthropicBlock(raw json.RawMessage) map[string]interface{} {
url := imageSourceFromRaw(raw)
if url == "" {
return nil
}
source := map[string]interface{}{}
if strings.HasPrefix(url, "data:") {
mediaType, data := splitDataURLForAnthropic(url)
source["type"] = "base64"
source["media_type"] = mediaType
source["data"] = data
} else {
source["type"] = "url"
source["url"] = url
}
return map[string]interface{}{
"type": "image",
"source": source,
}
}
func splitDataURLForAnthropic(value string) (mediaType, data string) {
header, payload, ok := strings.Cut(value, ",")
if !ok {
return "image/png", value
}
mediaType = strings.TrimPrefix(header, "data:")
if semicolon := strings.IndexByte(mediaType, ';'); semicolon >= 0 {
mediaType = mediaType[:semicolon]
}
if mediaType == "" {
mediaType = "image/png"
}
return mediaType, payload
}
func assistantContentToAnthropic(m Message) []map[string]interface{} {
blocks := make([]map[string]interface{}, 0, 1+len(m.ToolCalls))
if text := MessageTextContent(m.Content); text != "" {
pkg/aiproxy/providers/openai/tools_test.go
+29
View File
@@ -157,3 +157,32 @@ func TestNewChatCompletionWithTools(t *testing.T) {
t.Fatalf("expected finish_reason tool_calls, got %v", choices[0]["finish_reason"])
}
}
func TestMessagesToAnthropicWithImageURL(t *testing.T) {
content, _ := json.Marshal([]map[string]interface{}{
{"type": "text", "text": "describe"},
{"type": "image_url", "image_url": map[string]string{"url": "data:image/png;base64,xyz"}},
})
msgs := []Message{{
Role: "user",
Content: content,
}}
out, err := MessagesToAnthropic(msgs)
if err != nil {
t.Fatal(err)
}
if len(out) != 1 {
t.Fatalf("out=%#v", out)
}
blocks, ok := out[0]["content"].([]map[string]interface{})
if !ok || len(blocks) != 2 {
t.Fatalf("content=%#v", out[0]["content"])
}
if blocks[1]["type"] != "image" {
t.Fatalf("expected image block, got %#v", blocks[1])
}
src, _ := blocks[1]["source"].(map[string]interface{})
if src["type"] != "base64" || src["data"] != "xyz" {
t.Fatalf("source=%#v", src)
}
}
pkg/aiproxy/service/service.go
+11 -11
View File
@@ -48,17 +48,17 @@ func StartService() {
log.Fatalf("init local proxy node id: %v", err)
}
chatlog.Configure(chatlog.Options{
Enabled: opts.ChatLogEnabled,
LocalDir: opts.ChatLogLocalDir,
UploadEnabled: opts.ChatLogUploadEnabled,
UploadIntervalSeconds: opts.ChatLogUploadIntervalSeconds,
SegmentMinutes: opts.ChatLogSegmentMinutes,
MinioEndpoint: opts.ChatLogMinioEndpoint,
MinioAccessKey: opts.ChatLogMinioAccessKey,
MinioSecretKey: opts.ChatLogMinioSecretKey,
MinioBucket: opts.ChatLogMinioBucket,
MinioSecure: opts.ChatLogMinioSecure,
MinioPrefix: opts.ChatLogMinioPrefix,
Enabled: opts.APILogEnabled,
LocalDir: opts.APILogLocalDir,
UploadEnabled: opts.APILogUploadEnabled,
UploadIntervalSeconds: opts.APILogUploadIntervalSeconds,
SegmentMinutes: opts.APILogSegmentMinutes,
S3Endpoint: opts.APILogS3Endpoint,
S3AccessKey: opts.APILogS3AccessKey,
S3SecretKey: opts.APILogS3SecretKey,
S3Bucket: opts.APILogS3Bucket,
S3Secure: opts.APILogS3Secure,
S3Prefix: opts.APILogS3Prefix,
Instance: models.CurrentProxyNodeId(),
})
uploadCtx, stopUpload := context.WithCancel(context.Background())
pkg/apis/aiproxy/ai_model.go
+64 -8
View File
@@ -15,6 +15,8 @@
package aiproxy
import (
"encoding/json"
"yunion.io/x/onecloud/pkg/apis"
)
@@ -29,22 +31,76 @@ type AiModelListInput struct {
type AiModelCreateInput struct {
apis.EnabledStatusStandaloneResourceCreateInput
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
Config *SAiModelConfig `json:"config"`
}
type AiModelUpdateInput struct {
apis.EnabledStatusStandaloneResourceBaseUpdateInput
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
Enabled *bool `json:"enabled"`
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
Enabled *bool `json:"enabled"`
Config *SAiModelConfig `json:"config"`
}
type AiModelDetails struct {
apis.EnabledStatusStandaloneResourceDetails
AiProviderId string `json:"ai_provider_id"`
AiProviderName string `json:"ai_provider_name"`
ModelKey string `json:"model_key"`
AiProviderId string `json:"ai_provider_id"`
AiProviderName string `json:"ai_provider_name"`
ModelKey string `json:"model_key"`
Config *SAiModelConfig `json:"config"`
}
// SAiModelConfig stores per-model extension settings.
type SAiModelConfig struct {
Extensions *SAiModelExtensions `json:"extensions,omitempty"`
}
type SAiModelExtensions struct {
Visual *SAiModelVisualConfig `json:"visual,omitempty"`
}
// SAiModelVisualConfig enables tool-delegated vision for text-only upstream models.
type SAiModelVisualConfig struct {
Enabled bool `json:"enabled"`
VisualAiProviderId string `json:"visual_ai_provider_id"`
VisualModelKey string `json:"visual_model_key"`
MaxRounds int `json:"max_rounds,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
}
// String implements gotypes.ISerializable for sqlchemy JSON/compound columns.
func (c *SAiModelConfig) String() string {
if c == nil {
return "{}"
}
b, err := json.Marshal(c)
if err != nil {
return "{}"
}
return string(b)
}
// IsZero implements gotypes.ISerializable.
func (c *SAiModelConfig) IsZero() bool {
if c == nil {
return true
}
return c.Extensions == nil || c.Extensions.Visual == nil
}
// VisualEnabled reports whether visual extension is configured and enabled.
func (cfg *SAiModelConfig) VisualEnabled() bool {
return cfg != nil && cfg.Extensions != nil && cfg.Extensions.Visual != nil && cfg.Extensions.Visual.Enabled
}
// InputModalitiesForCatalog returns Codex input_modalities for this model config.
func (cfg *SAiModelConfig) InputModalitiesForCatalog() []string {
if cfg.VisualEnabled() {
return []string{"text", "image"}
}
return nil
}
pkg/apis/aiproxy/ai_model_test.go
+42
View File
@@ -0,0 +1,42 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aiproxy
import "testing"
func TestSAiModelConfigVisualEnabled(t *testing.T) {
cfg := &SAiModelConfig{
Extensions: &SAiModelExtensions{
Visual: &SAiModelVisualConfig{Enabled: true},
},
}
if !cfg.VisualEnabled() {
t.Fatal("expected visual enabled")
}
mods := cfg.InputModalitiesForCatalog()
if len(mods) != 2 || mods[1] != "image" {
t.Fatalf("modalities = %#v", mods)
}
if cfg.IsZero() {
t.Fatal("expected non-zero config")
}
if cfg.String() == "{}" {
t.Fatal("expected non-empty String()")
}
var empty *SAiModelConfig
if !empty.IsZero() {
t.Fatal("nil config should be zero")
}
}
pkg/apis/aiproxy/serialize_register.go
+3
View File
@@ -24,6 +24,9 @@ func init() {
gotypes.RegisterSerializable(reflect.TypeOf((*SAiProviderConfig)(nil)), func() gotypes.ISerializable {
return &SAiProviderConfig{}
})
gotypes.RegisterSerializable(reflect.TypeOf((*SAiModelConfig)(nil)), func() gotypes.ISerializable {
return &SAiModelConfig{}
})
gotypes.RegisterSerializable(reflect.TypeOf((*SAiKeyRouting)(nil)), func() gotypes.ISerializable {
return &SAiKeyRouting{}
})