diff --git a/pkg/aiproxy/chatlog/chatlog.go b/pkg/aiproxy/chatlog/chatlog.go index 9159656a71..f439b22dab 100644 --- a/pkg/aiproxy/chatlog/chatlog.go +++ b/pkg/aiproxy/chatlog/chatlog.go @@ -44,12 +44,12 @@ type Options struct { UploadEnabled bool UploadIntervalSeconds int SegmentMinutes int - MinioEndpoint string - MinioAccessKey string - MinioSecretKey string - MinioBucket string - MinioSecure bool - MinioPrefix string + S3Endpoint string + S3AccessKey string + S3SecretKey string + S3Bucket string + S3Secure bool + S3Prefix string Instance string } @@ -255,12 +255,12 @@ func UploadKey(prefix string, ts time.Time, filename string, instance string) st } func (w *Writer) s3Client() (*s3.Client, error) { - if w.opts.MinioEndpoint == "" || w.opts.MinioBucket == "" || w.opts.MinioAccessKey == "" || w.opts.MinioSecretKey == "" { - return nil, errors.New("missing MinIO/S3 config") + if w.opts.S3Endpoint == "" || w.opts.S3Bucket == "" || w.opts.S3AccessKey == "" || w.opts.S3SecretKey == "" { + return nil, errors.New("missing S3 config") } - endpoint := strings.TrimSpace(w.opts.MinioEndpoint) + endpoint := strings.TrimSpace(w.opts.S3Endpoint) if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { - if w.opts.MinioSecure { + if w.opts.S3Secure { endpoint = "https://" + endpoint } else { endpoint = "http://" + endpoint @@ -268,7 +268,7 @@ func (w *Writer) s3Client() (*s3.Client, error) { } return s3.NewFromConfig(aws.Config{ Region: "us-east-1", - Credentials: credentials.NewStaticCredentialsProvider(w.opts.MinioAccessKey, w.opts.MinioSecretKey, ""), + Credentials: credentials.NewStaticCredentialsProvider(w.opts.S3AccessKey, w.opts.S3SecretKey, ""), BaseEndpoint: aws.String(endpoint), }, func(o *s3.Options) { o.UsePathStyle = true @@ -433,7 +433,7 @@ func (w *Writer) Read(ctx context.Context, opts ReadOptions) (*ReadResult, error return nil, err } ret := &ReadResult{Logs: make([]Record, 0)} - for _, prefix := range hourPrefixes(w.opts.MinioPrefix, opts.Start, opts.End) { + for _, prefix := range hourPrefixes(w.opts.S3Prefix, opts.Start, opts.End) { if err := w.readPrefix(ctx, client, prefix, opts, ret); err != nil { return nil, err } @@ -446,7 +446,7 @@ func (w *Writer) Read(ctx context.Context, opts ReadOptions) (*ReadResult, error func (w *Writer) readPrefix(ctx context.Context, client *s3.Client, prefix string, opts ReadOptions, ret *ReadResult) error { in := &s3.ListObjectsV2Input{ - Bucket: aws.String(w.opts.MinioBucket), + Bucket: aws.String(w.opts.S3Bucket), Prefix: aws.String(prefix), } for { @@ -475,7 +475,7 @@ func (w *Writer) readPrefix(ctx context.Context, client *s3.Client, prefix strin func (w *Writer) readObject(ctx context.Context, client *s3.Client, key string, opts ReadOptions, ret *ReadResult) error { out, err := client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(w.opts.MinioBucket), + Bucket: aws.String(w.opts.S3Bucket), Key: aws.String(key), }) if err != nil { @@ -525,12 +525,12 @@ func (w *Writer) uploadFile(ctx context.Context, path string) error { if err != nil { return err } - if err := ensureBucket(ctx, client, w.opts.MinioBucket); err != nil { + if err := ensureBucket(ctx, client, w.opts.S3Bucket); err != nil { return err } - key := UploadKey(w.opts.MinioPrefix, ts, path, w.opts.Instance) + key := UploadKey(w.opts.S3Prefix, ts, path, w.opts.Instance) _, err = client.PutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(w.opts.MinioBucket), + Bucket: aws.String(w.opts.S3Bucket), Key: aws.String(key), Body: f, }) diff --git a/pkg/aiproxy/codexconfig/catalog.go b/pkg/aiproxy/codexconfig/catalog.go index 1c4f0397ac..52e7fbf988 100644 --- a/pkg/aiproxy/codexconfig/catalog.go +++ b/pkg/aiproxy/codexconfig/catalog.go @@ -32,8 +32,9 @@ const ( // ModelListEntry is one OpenAI-compatible model from GET /ai/openai/v1/models. type ModelListEntry struct { - ID string - OwnedBy string + ID string + OwnedBy string + InputModalities []string } // ModelInfo represents a model entry in Codex models_catalog.json. @@ -137,7 +138,11 @@ func buildModelInfoFromEntry(entry ModelListEntry) ModelInfo { slug := strings.TrimSpace(entry.ID) displayName := displayNameForSlug(slug) description := seedDescriptionForSlug(slug) - return newModelInfo(slug, displayName, description, 0, nil) + modalities := entry.InputModalities + if len(modalities) == 0 { + modalities = nil + } + return newModelInfo(slug, displayName, description, 0, modalities) } func displayNameForSlug(slug string) string { diff --git a/pkg/aiproxy/codexconfig/catalog_visual_test.go b/pkg/aiproxy/codexconfig/catalog_visual_test.go new file mode 100644 index 0000000000..2dc7e74292 --- /dev/null +++ b/pkg/aiproxy/codexconfig/catalog_visual_test.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package codexconfig + +import "testing" + +func TestBuildCatalogVisualModalities(t *testing.T) { + catalog := BuildCatalogFromIDs([]ModelListEntry{{ + ID: "deepseek-v4-pro", + OwnedBy: "deepseek", + InputModalities: []string{"text", "image"}, + }}) + if len(catalog) != 1 { + t.Fatalf("catalog len = %d", len(catalog)) + } + if len(catalog[0].InputModalities) != 2 || catalog[0].InputModalities[1] != "image" { + t.Fatalf("modalities = %#v", catalog[0].InputModalities) + } +} diff --git a/pkg/aiproxy/codexconfig/codexconfig.go b/pkg/aiproxy/codexconfig/codexconfig.go index ec9e537a34..4d50b7caf5 100644 --- a/pkg/aiproxy/codexconfig/codexconfig.go +++ b/pkg/aiproxy/codexconfig/codexconfig.go @@ -250,8 +250,9 @@ func resolveRoutingModelKey(session *mcclient.ClientSession, routing string) (st type modelsListResponse struct { Data []struct { - ID string `json:"id"` - OwnedBy string `json:"owned_by"` + ID string `json:"id"` + OwnedBy string `json:"owned_by"` + InputModalities []string `json:"input_modalities"` } `json:"data"` } @@ -286,8 +287,9 @@ func listModelEntries(session *mcclient.ClientSession, aiproxyURL, virtualKey st continue } entries = append(entries, ModelListEntry{ - ID: id, - OwnedBy: strings.TrimSpace(item.OwnedBy), + ID: id, + OwnedBy: strings.TrimSpace(item.OwnedBy), + InputModalities: append([]string(nil), item.InputModalities...), }) } return entries, nil diff --git a/pkg/aiproxy/extensions/visual/client.go b/pkg/aiproxy/extensions/visual/client.go new file mode 100644 index 0000000000..33409b3248 --- /dev/null +++ b/pkg/aiproxy/extensions/visual/client.go @@ -0,0 +1,157 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" +) + +const visualSystemPrompt = "You are a vision analysis model behind Cloudpods AI Proxy Visual. Analyze images carefully, state uncertainty, and do not invent visual facts." + +// HTTPVisionClient calls a multimodal upstream using OpenAI chat/completions. +type HTTPVisionClient struct { + up *models.ChatUpstream + maxTokens int +} + +func NewHTTPVisionClient(up *models.ChatUpstream, maxTokens int) *HTTPVisionClient { + if maxTokens <= 0 { + maxTokens = 2048 + } + return &HTTPVisionClient{up: up, maxTokens: maxTokens} +} + +func (c *HTTPVisionClient) Analyze(ctx context.Context, request AnalysisRequest) (string, error) { + if c == nil || c.up == nil { + return "", fmt.Errorf("visual client is nil") + } + parts := jsonutils.NewArray() + textPart := jsonutils.NewDict() + textPart.Set("type", jsonutils.NewString("text")) + textPart.Set("text", jsonutils.NewString(request.Prompt)) + parts.Add(textPart) + for _, image := range request.Images { + if part := chatImagePart(image); part != nil { + parts.Add(part) + } + } + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", parts) + sysMsg := jsonutils.NewDict() + sysMsg.Set("role", jsonutils.NewString("system")) + sysMsg.Set("content", jsonutils.NewString(visualSystemPrompt)) + messages := jsonutils.NewArray(sysMsg, userMsg) + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString(c.up.UpstreamModel)) + body.Set("max_tokens", jsonutils.NewInt(int64(c.maxTokens))) + body.Set("messages", messages) + respBody, err := callChatCompletions(ctx, c.up, body) + if err != nil { + return "", err + } + text := textFromChatResponse(respBody) + if text == "" { + return "", fmt.Errorf("visual provider returned empty content") + } + return text, nil +} + +func callChatCompletions(ctx context.Context, up *models.ChatUpstream, body *jsonutils.JSONDict) ([]byte, error) { + if up == nil || body == nil { + return nil, fmt.Errorf("nil upstream or body") + } + req := &upstream.Request{ + URL: openai.ChatCompletionsURL(ChatBaseURL(up.BaseURL)), + APIKey: up.APIKey, + Body: []byte(body.String()), + } + resp, uerr := upstream.HTTPDo(ctx, http.MethodPost, req) + if uerr != nil { + return nil, uerr + } + return resp.Body, nil +} + +// ChatBaseURL strips Anthropic-compatible path suffixes so chat/completions +// can be built against the OpenAI-compatible root (e.g. deepseek.com vs deepseek.com/anthropic). +func ChatBaseURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if base == "" { + return base + } + lower := strings.ToLower(base) + for _, suffix := range []string{"/api/anthropic", "/anthropic"} { + if strings.HasSuffix(lower, suffix) { + return base[:len(base)-len(suffix)] + } + } + return base +} + +func finishReasonFromChatResponse(body []byte) string { + var resp struct { + Choices []struct { + FinishReason string `json:"finish_reason"` + } `json:"choices"` + } + if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { + return "" + } + return strings.TrimSpace(resp.Choices[0].FinishReason) +} + +func toolCallsFromChatResponse(body []byte) []openai.ToolCall { + var resp struct { + Choices []struct { + Message struct { + ToolCalls []openai.ToolCall `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { + return nil + } + return resp.Choices[0].Message.ToolCalls +} + +func assistantMessageFromChatResponse(body []byte) *jsonutils.JSONDict { + var resp struct { + Choices []struct { + Message json.RawMessage `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { + return nil + } + parsed, err := jsonutils.Parse(resp.Choices[0].Message) + if err != nil { + return nil + } + if d, ok := parsed.(*jsonutils.JSONDict); ok { + return d + } + return nil +} diff --git a/pkg/aiproxy/extensions/visual/config.go b/pkg/aiproxy/extensions/visual/config.go new file mode 100644 index 0000000000..b350909ebd --- /dev/null +++ b/pkg/aiproxy/extensions/visual/config.go @@ -0,0 +1,46 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import api "yunion.io/x/onecloud/pkg/apis/aiproxy" + +// RuntimeConfig is the resolved visual extension settings for one request. +type RuntimeConfig struct { + Enabled bool + MaxRounds int + MaxTokens int +} + +// RuntimeConfigFromModel extracts visual runtime settings from ai_model config. +func RuntimeConfigFromModel(cfg *api.SAiModelConfig) (RuntimeConfig, *api.SAiModelVisualConfig) { + out := RuntimeConfig{MaxRounds: 4, MaxTokens: 2048} + if cfg == nil || cfg.Extensions == nil || cfg.Extensions.Visual == nil { + return out, nil + } + vis := cfg.Extensions.Visual + out.Enabled = vis.Enabled + if vis.MaxRounds > 0 { + out.MaxRounds = vis.MaxRounds + } + if vis.MaxTokens > 0 { + out.MaxTokens = vis.MaxTokens + } + return out, vis +} + +// Enabled reports whether visual extension is enabled on the model config. +func Enabled(cfg *api.SAiModelConfig) bool { + return cfg != nil && cfg.VisualEnabled() +} diff --git a/pkg/aiproxy/extensions/visual/doc.go b/pkg/aiproxy/extensions/visual/doc.go new file mode 100644 index 0000000000..7a92845e7e --- /dev/null +++ b/pkg/aiproxy/extensions/visual/doc.go @@ -0,0 +1 @@ +package visual // import "yunion.io/x/onecloud/pkg/aiproxy/extensions/visual" diff --git a/pkg/aiproxy/extensions/visual/images.go b/pkg/aiproxy/extensions/visual/images.go new file mode 100644 index 0000000000..30c4321afc --- /dev/null +++ b/pkg/aiproxy/extensions/visual/images.go @@ -0,0 +1,194 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "encoding/json" + "fmt" + "strings" + + "yunion.io/x/jsonutils" +) + +func visualAttachmentText(index int) string { + return fmt.Sprintf("[Image #%d is available to Visual Brief and Visual QA. Use image_refs [\"Image #%d\"] or omit image fields to analyze attached images.]", index, index) +} + +// StripImagesFromChat removes image_url parts from chat messages and replaces them with placeholders. +func StripImagesFromChat(body *jsonutils.JSONDict) ([]ImageInput, error) { + if body == nil { + return nil, nil + } + msgsRaw, err := body.Get("messages") + if err != nil { + return nil, nil + } + arr, ok := msgsRaw.(*jsonutils.JSONArray) + if !ok { + return nil, nil + } + available := make([]ImageInput, 0, 2) + for i := 0; i < arr.Size(); i++ { + msgRaw, _ := arr.GetAt(i) + msg, ok := msgRaw.(*jsonutils.JSONDict) + if !ok { + continue + } + contentRaw, err := msg.Get("content") + if err != nil { + continue + } + if contentArr, ok := contentRaw.(*jsonutils.JSONArray); ok { + rewritten := jsonutils.NewArray() + for j := 0; j < contentArr.Size(); j++ { + partRaw, _ := contentArr.GetAt(j) + part, ok := partRaw.(*jsonutils.JSONDict) + if !ok { + rewritten.Add(partRaw) + continue + } + partType, _ := part.GetString("type") + if partType == "image_url" { + if image, ok := imageInputFromChatPart(part); ok { + available = append(available, image) + placeholder := jsonutils.NewDict() + placeholder.Set("type", jsonutils.NewString("text")) + placeholder.Set("text", jsonutils.NewString(visualAttachmentText(len(available)))) + rewritten.Add(placeholder) + continue + } + } + rewritten.Add(part) + } + msg.Set("content", rewritten) + continue + } + } + return available, nil +} + +func imageInputFromChatPart(part *jsonutils.JSONDict) (ImageInput, bool) { + if part == nil { + return ImageInput{}, false + } + var wire struct { + Type string `json:"type"` + ImageURL json.RawMessage `json:"image_url"` + } + if err := json.Unmarshal([]byte(part.String()), &wire); err != nil { + return ImageInput{}, false + } + if wire.Type != "image_url" { + return ImageInput{}, false + } + var url string + if err := json.Unmarshal(wire.ImageURL, &url); err == nil { + url = strings.TrimSpace(url) + if isSupportedImageURL(url) { + return ImageInput{URL: url}, true + } + } + var obj struct { + URL string `json:"url"` + } + if err := json.Unmarshal(wire.ImageURL, &obj); err == nil { + url = strings.TrimSpace(obj.URL) + if isSupportedImageURL(url) { + return ImageInput{URL: url}, true + } + } + return ImageInput{}, false +} + +func isSupportedImageURL(value string) bool { + lower := strings.ToLower(strings.TrimSpace(value)) + return strings.HasPrefix(lower, "http://") || + strings.HasPrefix(lower, "https://") || + strings.HasPrefix(lower, "data:") +} + +func splitDataURL(value string) (mediaType, data string) { + header, payload, ok := strings.Cut(value, ",") + if !ok { + return "", value + } + mediaType = strings.TrimPrefix(header, "data:") + if semicolon := strings.IndexByte(mediaType, ';'); semicolon >= 0 { + mediaType = mediaType[:semicolon] + } + if mediaType == "" { + mediaType = "image/png" + } + return mediaType, payload +} + +func chatImagePart(image ImageInput) *jsonutils.JSONDict { + part := jsonutils.NewDict() + part.Set("type", jsonutils.NewString("image_url")) + imgURL := jsonutils.NewDict() + if strings.TrimSpace(image.URL) != "" { + imgURL.Set("url", jsonutils.NewString(strings.TrimSpace(image.URL))) + } else if strings.TrimSpace(image.Data) != "" { + data := strings.TrimSpace(image.Data) + mime := strings.TrimSpace(image.MimeType) + if mime == "" { + mime = "image/png" + } + if strings.HasPrefix(data, "data:") { + imgURL.Set("url", jsonutils.NewString(data)) + } else { + imgURL.Set("url", jsonutils.NewString("data:"+mime+";base64,"+data)) + } + } else { + return nil + } + part.Set("image_url", imgURL) + return part +} + +func textFromChatResponse(body []byte) string { + var resp struct { + Choices []struct { + Message struct { + Content json.RawMessage `json:"content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(body, &resp); err != nil || len(resp.Choices) == 0 { + return "" + } + raw := resp.Choices[0].Message.Content + var s string + if json.Unmarshal(raw, &s) == nil { + return strings.TrimSpace(s) + } + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if json.Unmarshal(raw, &parts) == nil { + var b strings.Builder + for _, p := range parts { + if p.Type == "text" && p.Text != "" { + if b.Len() > 0 { + b.WriteByte('\n') + } + b.WriteString(p.Text) + } + } + return strings.TrimSpace(b.String()) + } + return "" +} diff --git a/pkg/aiproxy/extensions/visual/messages.go b/pkg/aiproxy/extensions/visual/messages.go new file mode 100644 index 0000000000..30c29ec516 --- /dev/null +++ b/pkg/aiproxy/extensions/visual/messages.go @@ -0,0 +1,123 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient/auth" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +var ErrVisualStreamingUnsupported = fmt.Errorf("visual extension does not support streaming") + +// AnthropicMessagesHasImage reports whether an Anthropic Messages body carries image blocks. +func AnthropicMessagesHasImage(body *jsonutils.JSONDict) bool { + if body == nil { + return false + } + raw, err := body.Get("messages") + if err != nil { + return false + } + rawBytes := []byte(raw.String()) + var messages []struct { + Content json.RawMessage `json:"content"` + } + if json.Unmarshal(rawBytes, &messages) != nil { + return false + } + for _, msg := range messages { + if anthropicContentHasImage(msg.Content) { + return true + } + } + return false +} + +func anthropicContentHasImage(raw json.RawMessage) bool { + if len(raw) == 0 || string(raw) == "null" { + return false + } + var blocks []struct { + Type string `json:"type"` + } + if json.Unmarshal(raw, &blocks) != nil { + return false + } + for _, blk := range blocks { + if strings.EqualFold(blk.Type, "image") { + return true + } + } + return false +} + +// ShouldHandleMessages reports whether the Messages visual path should run (non-stream only). +func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool { + if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) { + return false + } + return AnthropicMessagesHasImage(dict) +} + +// ShouldRejectMessagesStreaming reports stream+visual+image (unsupported). +func ShouldRejectMessagesStreaming(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool { + if !isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) { + return false + } + return AnthropicMessagesHasImage(dict) +} + +// HandleMessagesCreate runs visual orchestration for a non-streaming Anthropic Messages request. +func HandleMessagesCreate( + ctx context.Context, + dict *jsonutils.JSONDict, + textUp *models.ChatUpstream, +) ([]byte, error) { + runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig) + if visCfg == nil { + return nil, fmt.Errorf("visual config is missing") + } + userCred := auth.AdminCredential() + vk, err := models.LoadEnabledVirtualKeyById(textUp.VirtualKeyId) + if err != nil { + return nil, err + } + visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg) + if err != nil { + return nil, err + } + chatBody, err := openai.AnthropicToChatCompletions(dict, textUp.UpstreamModel) + if err != nil { + return nil, err + } + chatClone := cloneDict(chatBody) + textUpChat := *textUp + textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL) + visUpChat := *visUp + visUpChat.BaseURL = ChatBaseURL(visUp.BaseURL) + respBody, err := RunChatOrchestrator(ctx, &textUpChat, &visUpChat, chatClone, runtime) + if err != nil { + return nil, err + } + return openai.ChatCompletionToAnthropic(respBody) +} diff --git a/pkg/aiproxy/extensions/visual/orchestrator.go b/pkg/aiproxy/extensions/visual/orchestrator.go new file mode 100644 index 0000000000..a9f2f07c84 --- /dev/null +++ b/pkg/aiproxy/extensions/visual/orchestrator.go @@ -0,0 +1,279 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type briefInput struct { + ImageURL string `json:"image_url,omitempty"` + ImageURLs []string `json:"image_urls,omitempty"` + ImageRefs []string `json:"image_refs,omitempty"` + Images []ImageInput `json:"images,omitempty"` + Context string `json:"context,omitempty"` + Focus string `json:"focus,omitempty"` +} + +type qaInput struct { + Question string `json:"question,omitempty"` + ImageURL string `json:"image_url,omitempty"` + ImageURLs []string `json:"image_urls,omitempty"` + ImageRefs []string `json:"image_refs,omitempty"` + Images []ImageInput `json:"images,omitempty"` + PriorVisualContext string `json:"prior_visual_context,omitempty"` + Context string `json:"context,omitempty"` + Conversation []ConversationTurn `json:"conversation,omitempty"` +} + +// RunChatOrchestrator executes the visual tool loop on a chat/completions request body. +func RunChatOrchestrator( + ctx context.Context, + textUp, visUp *models.ChatUpstream, + body *jsonutils.JSONDict, + runtime RuntimeConfig, +) ([]byte, error) { + if textUp == nil || visUp == nil || body == nil { + return nil, fmt.Errorf("visual orchestrator: missing upstream or body") + } + availableImages, err := StripImagesFromChat(body) + if err != nil { + return nil, err + } + InjectChatTools(body) + vision := NewHTTPVisionClient(visUp, runtime.MaxTokens) + maxRounds := runtime.MaxRounds + if maxRounds <= 0 { + maxRounds = 4 + } + for round := 0; round < maxRounds; round++ { + respBody, err := callChatCompletions(ctx, textUp, body) + if err != nil { + return nil, err + } + finish := finishReasonFromChatResponse(respBody) + if finish != "tool_calls" { + return respBody, nil + } + toolCalls := toolCallsFromChatResponse(respBody) + visualCalls, nonVisual := splitVisualToolCalls(toolCalls) + if len(visualCalls) == 0 { + return respBody, nil + } + assistantMsg := assistantMessageFromChatResponse(respBody) + if assistantMsg == nil { + return respBody, nil + } + msgsRaw, err := body.Get("messages") + if err != nil { + return nil, err + } + msgs, ok := msgsRaw.(*jsonutils.JSONArray) + if !ok { + return nil, fmt.Errorf("messages is not an array") + } + msgs.Add(assistantMsg) + for _, tc := range visualCalls { + result, err := executeVisualToolCall(ctx, vision, tc, availableImages) + if err != nil { + result = "Visual error: " + err.Error() + } + toolMsg := jsonutils.NewDict() + toolMsg.Set("role", jsonutils.NewString("tool")) + toolMsg.Set("tool_call_id", jsonutils.NewString(tc.ID)) + toolMsg.Set("content", jsonutils.NewString(result)) + msgs.Add(toolMsg) + } + body.Set("messages", msgs) + if len(nonVisual) > 0 { + continue + } + } + return nil, fmt.Errorf("visual loop exceeded max rounds (%d)", maxRounds) +} + +func splitVisualToolCalls(calls []openai.ToolCall) (visualCalls, nonVisual []openai.ToolCall) { + for _, tc := range calls { + if IsVisualTool(tc.Function.Name) { + visualCalls = append(visualCalls, tc) + } else { + nonVisual = append(nonVisual, tc) + } + } + return visualCalls, nonVisual +} + +func executeVisualToolCall(ctx context.Context, client *HTTPVisionClient, tc openai.ToolCall, available []ImageInput) (string, error) { + request, err := analysisRequestFromToolCall(tc, available) + if err != nil { + return "", err + } + result, err := client.Analyze(ctx, request) + if err != nil { + return "", err + } + switch tc.Function.Name { + case ToolVisualBrief: + return "Visual Brief result:\n" + strings.TrimSpace(result), nil + case ToolVisualQA: + return "Visual QA result:\n" + strings.TrimSpace(result), nil + default: + return strings.TrimSpace(result), nil + } +} + +func analysisRequestFromToolCall(tc openai.ToolCall, available []ImageInput) (AnalysisRequest, error) { + switch tc.Function.Name { + case ToolVisualBrief: + var input briefInput + if err := json.Unmarshal([]byte(tc.Function.Arguments), &input); err != nil { + return AnalysisRequest{}, fmt.Errorf("parse visual_brief input: %w", err) + } + images := normalizeImages(input.ImageURL, input.ImageURLs, input.Images, input.ImageRefs, available) + if len(images) == 0 { + return AnalysisRequest{}, fmt.Errorf("visual_brief requires valid image URLs/data/images or attached images") + } + return AnalysisRequest{Tool: ToolVisualBrief, Prompt: buildBriefPrompt(input), Images: images}, nil + case ToolVisualQA: + var input qaInput + if err := json.Unmarshal([]byte(tc.Function.Arguments), &input); err != nil { + return AnalysisRequest{}, fmt.Errorf("parse visual_qa input: %w", err) + } + if strings.TrimSpace(input.Question) == "" { + return AnalysisRequest{}, fmt.Errorf("visual_qa requires question") + } + return AnalysisRequest{ + Tool: ToolVisualQA, + Prompt: buildQAPrompt(input), + Images: normalizeImages(input.ImageURL, input.ImageURLs, input.Images, input.ImageRefs, available), + }, nil + default: + return AnalysisRequest{}, fmt.Errorf("unknown visual tool %q", tc.Function.Name) + } +} + +func buildBriefPrompt(input briefInput) string { + var b strings.Builder + b.WriteString("Provide a first-round visual brief for the main agent.\n") + if strings.TrimSpace(input.Context) != "" { + b.WriteString("\nTask context:\n") + b.WriteString(strings.TrimSpace(input.Context)) + b.WriteByte('\n') + } + if strings.TrimSpace(input.Focus) != "" { + b.WriteString("\nFocus:\n") + b.WriteString(strings.TrimSpace(input.Focus)) + b.WriteByte('\n') + } + b.WriteString("\nReturn concise sections: overview, important visual details, any readable text/OCR, uncertainties, and useful Visual QA follow-ups.") + return b.String() +} + +func buildQAPrompt(input qaInput) string { + var b strings.Builder + b.WriteString("Answer this targeted visual clarification question for the main agent.\n\nQuestion:\n") + b.WriteString(strings.TrimSpace(input.Question)) + b.WriteByte('\n') + if strings.TrimSpace(input.Context) != "" { + b.WriteString("\nTask context:\n") + b.WriteString(strings.TrimSpace(input.Context)) + b.WriteByte('\n') + } + if strings.TrimSpace(input.PriorVisualContext) != "" { + b.WriteString("\nPrior visual context:\n") + b.WriteString(strings.TrimSpace(input.PriorVisualContext)) + b.WriteByte('\n') + } + b.WriteString("\nAnswer directly, call out uncertainty, and say what extra image/detail would resolve ambiguity.") + return b.String() +} + +func normalizeImages(single string, urls []string, images []ImageInput, refs []string, availableImages []ImageInput) []ImageInput { + normalized := make([]ImageInput, 0, len(urls)+len(images)+len(refs)+1) + if image, ok := resolveImageValue(single, availableImages); ok { + normalized = append(normalized, image) + } + for _, url := range urls { + if image, ok := resolveImageValue(url, availableImages); ok { + normalized = append(normalized, image) + } + } + for _, ref := range refs { + if image, ok := resolveAttachedImage(ref, availableImages); ok { + normalized = append(normalized, image) + } + } + for _, image := range images { + if strings.TrimSpace(image.URL) != "" { + if resolved, ok := resolveImageValue(image.URL, availableImages); ok { + if resolved.Detail == "" { + resolved.Detail = image.Detail + } + normalized = append(normalized, resolved) + } + continue + } + if strings.TrimSpace(image.Data) != "" || isSupportedImageURL(image.URL) { + normalized = append(normalized, image) + } + } + if len(normalized) == 0 && len(availableImages) > 0 { + normalized = append(normalized, availableImages...) + } + return normalized +} + +func resolveImageValue(value string, availableImages []ImageInput) (ImageInput, bool) { + value = strings.TrimSpace(value) + if value == "" { + return ImageInput{}, false + } + if isSupportedImageURL(value) { + return ImageInput{URL: value}, true + } + return resolveAttachedImage(value, availableImages) +} + +func resolveAttachedImage(value string, availableImages []ImageInput) (ImageInput, bool) { + index, ok := imageReferenceIndex(value) + if !ok || index <= 0 || index > len(availableImages) { + return ImageInput{}, false + } + return availableImages[index-1], true +} + +var imageReferencePattern = regexp.MustCompile(`(?i)\bimage\s*#\s*(\d+)\b`) + +func imageReferenceIndex(value string) (int, bool) { + match := imageReferencePattern.FindStringSubmatch(strings.TrimSpace(value)) + if len(match) != 2 { + return 0, false + } + index, err := strconv.Atoi(match[1]) + if err != nil { + return 0, false + } + return index, true +} diff --git a/pkg/aiproxy/extensions/visual/responses.go b/pkg/aiproxy/extensions/visual/responses.go new file mode 100644 index 0000000000..46662f997c --- /dev/null +++ b/pkg/aiproxy/extensions/visual/responses.go @@ -0,0 +1,87 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient/auth" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +// ShouldHandle reports whether the Responses visual orchestration path should run. +func ShouldHandle(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool { + if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) { + return false + } + return openai.ResponsesInputHasImage(dict) +} + +// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request. +func HandleResponsesCreate( + ctx context.Context, + dict *jsonutils.JSONDict, + textUp *models.ChatUpstream, +) ([]byte, *openai.ResponsesConvertState, error) { + runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig) + if visCfg == nil { + return nil, nil, fmt.Errorf("visual config is missing") + } + userCred := auth.AdminCredential() + vk, err := models.LoadEnabledVirtualKeyById(textUp.VirtualKeyId) + if err != nil { + return nil, nil, err + } + visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg) + if err != nil { + return nil, nil, err + } + chatBody, state, err := openai.ResponsesToChatCompletions(dict, textUp.UpstreamModel) + if err != nil { + return nil, nil, err + } + chatClone := cloneDict(chatBody) + textUpChat := *textUp + textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL) + visUpChat := *visUp + visUpChat.BaseURL = ChatBaseURL(visUp.BaseURL) + respBody, err := RunChatOrchestrator(ctx, &textUpChat, &visUpChat, chatClone, runtime) + if err != nil { + return nil, nil, err + } + out, err := openai.ChatCompletionToResponses(respBody, state) + if err != nil { + return nil, nil, err + } + return out, state, nil +} + +func cloneDict(src *jsonutils.JSONDict) *jsonutils.JSONDict { + if src == nil { + return nil + } + parsed, err := jsonutils.Parse([]byte(src.String())) + if err != nil { + return src + } + if d, ok := parsed.(*jsonutils.JSONDict); ok { + return d + } + return src +} diff --git a/pkg/aiproxy/extensions/visual/tools.go b/pkg/aiproxy/extensions/visual/tools.go new file mode 100644 index 0000000000..13ac4420ee --- /dev/null +++ b/pkg/aiproxy/extensions/visual/tools.go @@ -0,0 +1,178 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "encoding/json" + + "yunion.io/x/jsonutils" +) + +const ( + ToolVisualBrief = "visual_brief" + ToolVisualQA = "visual_qa" +) + +func visualBriefSchema() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "image_urls": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{"type": "string"}, + "description": "HTTP(S) image URLs or data URLs only. Do not put attachment labels like Image #1 here.", + }, + "image_refs": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{"type": "string"}, + "description": "Attached image labels such as Image #1.", + }, + "images": map[string]interface{}{ + "type": "array", + "description": "Structured images with url or base64 data plus mime_type.", + "items": imageInputSchema(), + }, + "context": map[string]interface{}{ + "type": "string", + "description": "User task context that helps decide which visual facts matter.", + }, + "focus": map[string]interface{}{ + "type": "string", + "description": "Optional focus area such as UI layout, OCR, or chart details.", + }, + }, + } +} + +func visualQASchema() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "question": map[string]interface{}{ + "type": "string", + "description": "A specific visual question to answer or clarify.", + }, + "image_urls": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{"type": "string"}, + }, + "image_refs": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{"type": "string"}, + }, + "images": map[string]interface{}{ + "type": "array", + "items": imageInputSchema(), + }, + "prior_visual_context": map[string]interface{}{ + "type": "string", + }, + "context": map[string]interface{}{ + "type": "string", + }, + "conversation": map[string]interface{}{ + "type": "array", + "items": map[string]interface{}{"type": "object"}, + }, + }, + "required": []string{"question"}, + } +} + +func imageInputSchema() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "url": map[string]interface{}{"type": "string"}, + "data": map[string]interface{}{"type": "string"}, + "mime_type": map[string]interface{}{"type": "string"}, + "detail": map[string]interface{}{"type": "string"}, + }, + } +} + +// ChatTools returns OpenAI chat tool definitions for visual analysis. +func ChatTools() []*jsonutils.JSONDict { + return []*jsonutils.JSONDict{ + chatTool(ToolVisualBrief, "Visual Brief. Use this as the first visual pass when image understanding is needed. For attached images, pass image_refs like Image #1 or omit image fields; pass image_urls only for real HTTP(S)/data URLs.", visualBriefSchema()), + chatTool(ToolVisualQA, "Visual QA. Use this after a Visual Brief, or for targeted follow-up questions about an image.", visualQASchema()), + } +} + +func chatTool(name, description string, schema map[string]interface{}) *jsonutils.JSONDict { + tool := jsonutils.NewDict() + tool.Set("type", jsonutils.NewString("function")) + fn := jsonutils.NewDict() + fn.Set("name", jsonutils.NewString(name)) + fn.Set("description", jsonutils.NewString(description)) + schemaBytes, _ := json.Marshal(schema) + params, _ := jsonutils.Parse(schemaBytes) + fn.Set("parameters", params) + tool.Set("function", fn) + return tool +} + +// IsVisualTool reports whether name is a visual extension tool. +func IsVisualTool(name string) bool { + return name == ToolVisualBrief || name == ToolVisualQA +} + +// InjectChatTools merges visual tools into a chat/completions request body. +func InjectChatTools(body *jsonutils.JSONDict) { + if body == nil { + return + } + existing := map[string]struct{}{} + if toolsRaw, err := body.Get("tools"); err == nil { + if arr, ok := toolsRaw.(*jsonutils.JSONArray); ok { + for i := 0; i < arr.Size(); i++ { + tool, _ := arr.GetAt(i) + if d, ok := tool.(*jsonutils.JSONDict); ok { + if fn, err := d.Get("function"); err == nil { + if fnDict, ok := fn.(*jsonutils.JSONDict); ok { + if name, _ := fnDict.GetString("name"); name != "" { + existing[name] = struct{}{} + } + } + } + } + } + } + } + tools := jsonutils.NewArray() + if toolsRaw, err := body.Get("tools"); err == nil { + if arr, ok := toolsRaw.(*jsonutils.JSONArray); ok { + for i := 0; i < arr.Size(); i++ { + item, _ := arr.GetAt(i) + tools.Add(item) + } + } + } + for _, tool := range ChatTools() { + if fn, err := tool.Get("function"); err == nil { + if fnDict, ok := fn.(*jsonutils.JSONDict); ok { + if name, _ := fnDict.GetString("name"); name != "" { + if _, ok := existing[name]; ok { + continue + } + } + } + } + tools.Add(tool) + } + if tools.Length() > 0 { + body.Set("tools", tools) + } +} diff --git a/pkg/aiproxy/extensions/visual/types.go b/pkg/aiproxy/extensions/visual/types.go new file mode 100644 index 0000000000..d03704088b --- /dev/null +++ b/pkg/aiproxy/extensions/visual/types.go @@ -0,0 +1,43 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import "context" + +// ImageInput carries image data as URL, base64 data, or data URL. +type ImageInput struct { + URL string `json:"url,omitempty"` + Data string `json:"data,omitempty"` + MimeType string `json:"mime_type,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// ConversationTurn is one visual clarification history entry. +type ConversationTurn struct { + Role string `json:"role,omitempty"` + Content string `json:"content,omitempty"` +} + +// AnalysisRequest bundles the analysis prompt with images for the vision model. +type AnalysisRequest struct { + Tool string + Prompt string + Images []ImageInput +} + +// VisionClient analyzes images with a short prompt. +type VisionClient interface { + Analyze(context.Context, AnalysisRequest) (string, error) +} diff --git a/pkg/aiproxy/extensions/visual/visual_test.go b/pkg/aiproxy/extensions/visual/visual_test.go new file mode 100644 index 0000000000..3825fbd0e0 --- /dev/null +++ b/pkg/aiproxy/extensions/visual/visual_test.go @@ -0,0 +1,139 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package visual + +import ( + "strings" + "testing" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) + +func TestStripImagesFromChat(t *testing.T) { + body := jsonutils.NewDict() + msgs := jsonutils.NewArray() + user := jsonutils.NewDict() + user.Set("role", jsonutils.NewString("user")) + parts := jsonutils.NewArray() + text := jsonutils.NewDict() + text.Set("type", jsonutils.NewString("text")) + text.Set("text", jsonutils.NewString("describe")) + parts.Add(text) + img := jsonutils.NewDict() + img.Set("type", jsonutils.NewString("image_url")) + imgURL := jsonutils.NewDict() + imgURL.Set("url", jsonutils.NewString("data:image/png;base64,abc")) + img.Set("image_url", imgURL) + parts.Add(img) + user.Set("content", parts) + msgs.Add(user) + body.Set("messages", msgs) + + images, err := StripImagesFromChat(body) + if err != nil { + t.Fatal(err) + } + if len(images) != 1 { + t.Fatalf("images = %d", len(images)) + } + s := body.String() + if strings.Contains(s, "image_url") { + t.Fatalf("expected stripped image_url, got %s", s) + } + if !strings.Contains(s, "Image #1") { + t.Fatalf("expected placeholder, got %s", s) + } +} + +func TestEnabled(t *testing.T) { + cfg := &api.SAiModelConfig{ + Extensions: &api.SAiModelExtensions{ + Visual: &api.SAiModelVisualConfig{Enabled: true}, + }, + } + if !Enabled(cfg) { + t.Fatal("expected enabled") + } +} + +func TestInjectChatTools(t *testing.T) { + body := jsonutils.NewDict() + InjectChatTools(body) + tools, err := body.Get("tools") + if err != nil { + t.Fatal(err) + } + arr, ok := tools.(*jsonutils.JSONArray) + if !ok || arr.Size() < 2 { + t.Fatalf("tools = %v", tools) + } +} + +func TestNormalizeImagesUsesAttached(t *testing.T) { + available := []ImageInput{{URL: "data:image/png;base64,abc"}} + images := normalizeImages("", nil, nil, []string{"Image #1"}, available) + if len(images) != 1 || images[0].URL == "" { + t.Fatalf("images = %#v", images) + } +} + +func TestChatBaseURLStripsAnthropicSuffix(t *testing.T) { + cases := []struct { + in, want string + }{ + {"https://api.deepseek.com/anthropic", "https://api.deepseek.com"}, + {"https://api.deepseek.com/anthropic/", "https://api.deepseek.com"}, + {"https://open.bigmodel.cn/api/anthropic", "https://open.bigmodel.cn"}, + {"https://api.moonshot.cn", "https://api.moonshot.cn"}, + } + for _, c := range cases { + if got := ChatBaseURL(c.in); got != c.want { + t.Fatalf("ChatBaseURL(%q)=%q want %q", c.in, got, c.want) + } + } +} + +func TestAnthropicMessagesHasImage(t *testing.T) { + body, _ := jsonutils.Parse([]byte(`{"messages":[{"role":"user","content":[{"type":"image","source":{"type":"url","url":"https://x/a.png"}}]}]}`)) + if !AnthropicMessagesHasImage(body.(*jsonutils.JSONDict)) { + t.Fatal("expected image") + } + body2, _ := jsonutils.Parse([]byte(`{"messages":[{"role":"user","content":"hello"}]}`)) + if AnthropicMessagesHasImage(body2.(*jsonutils.JSONDict)) { + t.Fatal("expected no image") + } +} + +func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) { + cfg := &api.SAiModelConfig{ + Extensions: &api.SAiModelExtensions{ + Visual: &api.SAiModelVisualConfig{Enabled: true}, + }, + } + up := &models.ChatUpstream{ + ModelConfig: cfg, + APIMode: "anthropic", + } + body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`)) + if !ShouldHandle(body.(*jsonutils.JSONDict), up, false) { + t.Fatal("expected visual handle even with anthropic api_mode") + } + if ShouldHandle(body.(*jsonutils.JSONDict), up, true) { + t.Fatal("stream should not use visual orchestrator") + } +} diff --git a/pkg/aiproxy/handlers/messages.go b/pkg/aiproxy/handlers/messages.go index c7c79a8229..f6166369ee 100644 --- a/pkg/aiproxy/handlers/messages.go +++ b/pkg/aiproxy/handlers/messages.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/aiproxy/extensions/visual" "yunion.io/x/onecloud/pkg/aiproxy/models" "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers" @@ -91,6 +92,28 @@ func messagesHandler(ctx context.Context, w http.ResponseWriter, r *http.Request return } + if visual.ShouldRejectMessagesStreaming(dict, up, isStream) { + dbg.Error("%v", visual.ErrVisualStreamingUnsupported) + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", visual.ErrVisualStreamingUnsupported) + return + } + if visual.ShouldHandleMessages(dict, up, isStream) { + bodyOut, err := visual.HandleMessagesCreate(ctx, dict, up) + if err != nil { + dbg.Error("visual orchestration: %v", err) + writeAnthropicError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err) + return + } + dbg.ClientResponse(bodyOut) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(bodyOut) + if up.AiKeyId != "" { + models.RecordAiKeySuccess(up.AiKeyId) + } + return + } + adapter, err := messages.GetAdapter(up.ProviderKey, up.APIMode) if err != nil { dbg.Error("adapter: %v", err) diff --git a/pkg/aiproxy/handlers/responses.go b/pkg/aiproxy/handlers/responses.go index f3b8e1f9fd..39ef5eb975 100644 --- a/pkg/aiproxy/handlers/responses.go +++ b/pkg/aiproxy/handlers/responses.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/aiproxy/extensions/visual" "yunion.io/x/onecloud/pkg/aiproxy/models" "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers" @@ -104,6 +105,23 @@ func handleResponsesCreate(ctx context.Context, w http.ResponseWriter, r *http.R return } + if visual.ShouldHandle(dict, up, isStream) { + bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up) + if err != nil { + dbg.Error("visual orchestration: %v", err) + writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err) + return + } + dbg.ClientResponse(bodyOut) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(bodyOut) + if up.AiKeyId != "" { + models.RecordAiKeySuccess(up.AiKeyId) + } + return + } + adapter, err := responses.GetAdapter(up.ProviderKey, up.APIMode) if err != nil { dbg.Error("adapter: %v", err) diff --git a/pkg/aiproxy/models/ai_models.go b/pkg/aiproxy/models/ai_models.go index 29ba1d6102..7fa3c96e0a 100644 --- a/pkg/aiproxy/models/ai_models.go +++ b/pkg/aiproxy/models/ai_models.go @@ -36,6 +36,8 @@ type SAiModel struct { AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` // ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo). ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"` + // Config stores per-model extension settings (e.g. visual delegation). + Config *api.SAiModelConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"` } type SAiModelManager struct { @@ -138,6 +140,8 @@ func (manager *SAiModelManager) FetchCustomizeColumns( } for i := range rows { rows[i].AiProviderName, _ = providerNames[providerIds[i]] + m := objs[i].(*SAiModel) + rows[i].Config = m.Config } return rows } diff --git a/pkg/aiproxy/models/chat_upstream.go b/pkg/aiproxy/models/chat_upstream.go index a797df404c..f59b7b1905 100644 --- a/pkg/aiproxy/models/chat_upstream.go +++ b/pkg/aiproxy/models/chat_upstream.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/rbacscope" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -67,8 +68,10 @@ type ChatUpstream struct { UpstreamModel string ProviderKey string AiProviderId string + AiModelId string AiKeyId string APIMode string + ModelConfig *api.SAiModelConfig // VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row. VirtualKeyId string @@ -150,6 +153,24 @@ func loadEnabledVirtualKey(virtualKey string) (*SAiVirtualKey, error) { return &vk, nil } +// LoadEnabledVirtualKeyById loads an enabled virtual key by database id. +func LoadEnabledVirtualKeyById(id string) (*SAiVirtualKey, error) { + id = strings.TrimSpace(id) + if id == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "empty virtual key id") + } + vk := SAiVirtualKey{} + qvk := AiVirtualKeyManager.Query().Equals("id", id).Equals("enabled", true) + err := qvk.First(&vk) + if err != nil { + if stderrors.Is(err, sql.ErrNoRows) { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "virtual key not found or disabled") + } + return nil, errors.Wrap(err, "query ai_virtual_key by id") + } + return &vk, nil +} + // listProjectRoutingsForVirtualKey returns enabled ai_routing rows owned by or shared with the virtual key's project. func listProjectRoutingsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey) ([]SAiRouting, error) { if vk == nil { @@ -330,9 +351,11 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, UpstreamModel: upstreamModel, ProviderKey: prov.ProviderKey, AiProviderId: prov.Id, + AiModelId: mdl.Id, AiKeyId: keyRes.AiKeyId, VirtualKeyId: vk.Id, APIMode: apiMode, + ModelConfig: mdl.Config, ProjectId: vk.ProjectId, DomainId: vk.DomainId, RoutingLog: resolved.routingLog, @@ -343,3 +366,49 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, } return up, nil } + +// ResolveVisualUpstream resolves the visual provider upstream for tool-delegated image analysis. +func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, cfg *api.SAiModelVisualConfig) (*ChatUpstream, error) { + if cfg == nil || !cfg.Enabled { + return nil, errors.Wrap(httperrors.ErrInputParameter, "visual extension is not enabled") + } + providerID := strings.TrimSpace(cfg.VisualAiProviderId) + modelKey := strings.TrimSpace(cfg.VisualModelKey) + if providerID == "" || modelKey == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_ai_provider_id and visual_model_key are required") + } + pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerID) + if err != nil { + return nil, errors.Wrap(err, "fetch visual ai_provider") + } + prov := pObj.(*SAiProvider) + if !prov.GetEnabled() { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider disabled") + } + if !virtualKeyAllowsProvider(vk, prov) { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider not allowed for this virtual key") + } + if prov.Config == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider.config is empty") + } + baseURL := prov.Config.EffectiveBaseURL(prov.ProviderKey) + if baseURL == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "visual ai_provider.config must include base_url") + } + keyRes, err := resolveUpstreamAPIKey(prov, modelKey) + if err != nil { + return nil, err + } + if keyRes == nil || keyRes.Secret == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "add an enabled ai_key with secret for visual provider") + } + return &ChatUpstream{ + BaseURL: baseURL, + APIKey: keyRes.Secret, + UpstreamModel: modelKey, + ProviderKey: prov.ProviderKey, + AiProviderId: prov.Id, + AiKeyId: keyRes.AiKeyId, + APIMode: prov.Config.ResolvedAPIMode(), + }, nil +} diff --git a/pkg/aiproxy/models/list_models.go b/pkg/aiproxy/models/list_models.go index f52e983004..9881c19322 100644 --- a/pkg/aiproxy/models/list_models.go +++ b/pkg/aiproxy/models/list_models.go @@ -27,10 +27,11 @@ import ( // ModelsListEntry is one OpenAI-compatible model object in GET /openai/v1/models. type ModelsListEntry struct { - ID string `json:"id"` - Object string `json:"object"` - Created int64 `json:"created"` - OwnedBy string `json:"owned_by"` + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + OwnedBy string `json:"owned_by"` + InputModalities []string `json:"input_modalities,omitempty"` } // ListModelsForVirtualKey returns OpenAI-compatible model ids reachable by the virtual key @@ -113,12 +114,7 @@ func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredent if _, ok := seen[id]; ok { continue } - seen[id] = ModelsListEntry{ - ID: id, - Object: "model", - Created: created, - OwnedBy: firstProviderByRouting[routing.Id], - } + seen[id] = modelsListEntryFromModel(id, firstProviderByRouting[routing.Id], created, nil) } // Pass 2: ai_routing_model entries as flat or hierarchical client-facing ids. @@ -145,12 +141,7 @@ func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredent if _, ok := seen[id]; ok { continue } - seen[id] = ModelsListEntry{ - ID: id, - Object: "model", - Created: created, - OwnedBy: strings.TrimSpace(prov.ProviderKey), - } + seen[id] = modelsListEntryFromModel(id, strings.TrimSpace(prov.ProviderKey), created, mdl) continue } id := clientFacingModelID(routing, e, mdl) @@ -160,12 +151,7 @@ func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredent if _, ok := seen[id]; ok { continue } - seen[id] = ModelsListEntry{ - ID: id, - Object: "model", - Created: created, - OwnedBy: strings.TrimSpace(prov.ProviderKey), - } + seen[id] = modelsListEntryFromModel(id, strings.TrimSpace(prov.ProviderKey), created, mdl) } if len(seen) == 0 { return nil, nil @@ -306,3 +292,18 @@ func uniqueNonEmptyStrings(in []string) []string { } return out } + +func modelsListEntryFromModel(id, ownedBy string, created int64, mdl *SAiModel) ModelsListEntry { + entry := ModelsListEntry{ + ID: id, + Object: "model", + Created: created, + OwnedBy: ownedBy, + } + if mdl != nil && mdl.Config != nil { + if mods := mdl.Config.InputModalitiesForCatalog(); len(mods) > 0 { + entry.InputModalities = mods + } + } + return entry +} diff --git a/pkg/aiproxy/options/options.go b/pkg/aiproxy/options/options.go index 9eda76402c..1d4ae8f808 100644 --- a/pkg/aiproxy/options/options.go +++ b/pkg/aiproxy/options/options.go @@ -25,17 +25,17 @@ type SAiProxyOptions struct { AdvertiseAddress string `help:"Standby node address advertised to clients, e.g. http://10.0.0.2:30889; default derives from bind address and port" default:""` NodeHeartbeatIntervalSeconds int `help:"Interval in seconds for standby node registration heartbeat" default:"60"` - ChatLogEnabled bool `help:"Enable OpenAI chat request JSONL logs" default:"true"` - ChatLogLocalDir string `help:"Local directory for chat JSONL logs" default:"/tmp/aiproxy-chatlog"` - ChatLogUploadEnabled bool `help:"Upload closed chat log hour files to MinIO/S3" default:"true"` - ChatLogUploadIntervalSeconds int `help:"Chat log upload interval in seconds" default:"10"` - ChatLogSegmentMinutes int `help:"Chat log file segment duration in minutes" default:"60"` - ChatLogMinioEndpoint string `help:"MinIO/S3 endpoint for chat log upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"` - ChatLogMinioAccessKey string `help:"MinIO/S3 access key for chat log upload" default:"monitor-admin"` - ChatLogMinioSecretKey string `help:"MinIO/S3 secret key for chat log upload" default:"monitor-admin"` - ChatLogMinioBucket string `help:"MinIO/S3 bucket for chat log upload" default:"aiproxy-chat"` - ChatLogMinioSecure bool `help:"Use HTTPS for MinIO/S3 endpoint without scheme" default:"false"` - ChatLogMinioPrefix string `help:"MinIO/S3 object key prefix for chat logs" default:""` + APILogEnabled bool `help:"Enable OpenAI API request JSONL logs" default:"true"` + APILogLocalDir string `help:"Local directory for API JSONL logs" default:"/tmp/aiproxy-apilog"` + APILogUploadEnabled bool `help:"Upload closed API log hour files to S3" default:"true"` + APILogUploadIntervalSeconds int `help:"API log upload interval in seconds" default:"10"` + APILogSegmentMinutes int `help:"API log file segment duration in minutes" default:"60"` + APILogS3Endpoint string `help:"S3 endpoint for API log upload" default:"http://monitor-minio.onecloud-monitoring.svc:9000"` + APILogS3AccessKey string `help:"S3 access key for API log upload" default:"monitor-admin"` + APILogS3SecretKey string `help:"S3 secret key for API log upload" default:"monitor-admin"` + APILogS3Bucket string `help:"S3 bucket for API log upload" default:"aiproxy-log"` + APILogS3Secure bool `help:"Use HTTPS for S3 endpoint without scheme" default:"false"` + APILogS3Prefix string `help:"S3 object key prefix for API logs" default:""` } var ( diff --git a/pkg/aiproxy/providers/openai/anthropic_compat.go b/pkg/aiproxy/providers/openai/anthropic_compat.go index 4a09d86bf5..9d1259daec 100644 --- a/pkg/aiproxy/providers/openai/anthropic_compat.go +++ b/pkg/aiproxy/providers/openai/anthropic_compat.go @@ -221,14 +221,21 @@ func parseAnthropicUserContent(raw json.RawMessage) (jsonutils.JSONObject, []ant if err := json.Unmarshal(raw, &blocks); err != nil { return nil, nil, fmt.Errorf("invalid user content: %w", err) } - var textParts []string + parts := jsonutils.NewArray() var tools []anthropicToolResult for _, blk := range blocks { typ, _ := blk["type"].(string) switch typ { case "text": if t, _ := blk["text"].(string); t != "" { - textParts = append(textParts, t) + part := jsonutils.NewDict() + part.Set("type", jsonutils.NewString("text")) + part.Set("text", jsonutils.NewString(t)) + parts.Add(part) + } + case "image": + if part := anthropicImageBlockToChatPart(blk); part != nil { + parts.Add(part) } case "tool_result": id, _ := blk["tool_use_id"].(string) @@ -236,22 +243,70 @@ func parseAnthropicUserContent(raw json.RawMessage) (jsonutils.JSONObject, []ant tools = append(tools, anthropicToolResult{ID: id, Content: content}) } } - if len(textParts) == 0 { + if parts.Size() == 0 { return nil, tools, nil } - if len(textParts) == 1 { - return jsonutils.NewString(textParts[0]), tools, nil - } - parts := jsonutils.NewArray() - for _, p := range textParts { - blk := jsonutils.NewDict() - blk.Set("type", jsonutils.NewString("text")) - blk.Set("text", jsonutils.NewString(p)) - parts.Add(blk) + // Single text-only part can stay a plain string for compatibility. + if parts.Size() == 1 { + first, _ := parts.GetAt(0) + if d, ok := first.(*jsonutils.JSONDict); ok { + if typ, _ := d.GetString("type"); typ == "text" { + if text, _ := d.GetString("text"); text != "" { + return jsonutils.NewString(text), tools, nil + } + } + } } return parts, tools, nil } +func anthropicImageBlockToChatPart(blk map[string]interface{}) *jsonutils.JSONDict { + srcRaw, ok := blk["source"].(map[string]interface{}) + if !ok || srcRaw == nil { + return nil + } + srcType, _ := srcRaw["type"].(string) + var url string + switch strings.ToLower(strings.TrimSpace(srcType)) { + case "url": + url, _ = srcRaw["url"].(string) + case "base64": + data, _ := srcRaw["data"].(string) + mediaType, _ := srcRaw["media_type"].(string) + if strings.TrimSpace(data) == "" { + return nil + } + if mediaType == "" { + mediaType = "image/png" + } + if strings.HasPrefix(data, "data:") { + url = data + } else { + url = "data:" + mediaType + ";base64," + data + } + default: + if u, _ := srcRaw["url"].(string); strings.TrimSpace(u) != "" { + url = u + } else if data, _ := srcRaw["data"].(string); strings.TrimSpace(data) != "" { + mediaType, _ := srcRaw["media_type"].(string) + if mediaType == "" { + mediaType = "image/png" + } + url = "data:" + mediaType + ";base64," + data + } + } + url = strings.TrimSpace(url) + if url == "" { + return nil + } + imgURL := jsonutils.NewDict() + imgURL.Set("url", jsonutils.NewString(url)) + part := jsonutils.NewDict() + part.Set("type", jsonutils.NewString("image_url")) + part.Set("image_url", imgURL) + return part +} + func anthropicBlockContentText(v interface{}) string { switch c := v.(type) { case string: diff --git a/pkg/aiproxy/providers/openai/anthropic_compat_test.go b/pkg/aiproxy/providers/openai/anthropic_compat_test.go index 28e3627f3b..b5c6f4ff6b 100644 --- a/pkg/aiproxy/providers/openai/anthropic_compat_test.go +++ b/pkg/aiproxy/providers/openai/anthropic_compat_test.go @@ -16,6 +16,7 @@ package openai import ( "encoding/json" + "strings" "testing" "yunion.io/x/jsonutils" @@ -345,3 +346,54 @@ func TestAnthropicToolRoundTrip(t *testing.T) { t.Fatalf("stop_reason: %#v", resp["stop_reason"]) } } + +func TestAnthropicToChatCompletionsWithImage(t *testing.T) { + raw := `{ + "model":"vision-model", + "max_tokens":256, + "messages":[{"role":"user","content":[ + {"type":"text","text":"what is this"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":"abc123"}} + ]}] + }` + body, err := jsonutils.Parse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + out, err := AnthropicToChatCompletions(body.(*jsonutils.JSONDict), "vision-model") + if err != nil { + t.Fatal(err) + } + s := out.String() + if !strings.Contains(s, "image_url") { + t.Fatalf("expected image_url, got %s", s) + } + if !strings.Contains(s, "data:image/png;base64,abc123") { + t.Fatalf("expected data url, got %s", s) + } + if !strings.Contains(s, "what is this") { + t.Fatalf("expected text, got %s", s) + } +} + +func TestAnthropicToChatCompletionsImageOnly(t *testing.T) { + raw := `{ + "model":"vision-model", + "max_tokens":256, + "messages":[{"role":"user","content":[ + {"type":"image","source":{"type":"url","url":"https://example.com/a.png"}} + ]}] + }` + body, err := jsonutils.Parse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + out, err := AnthropicToChatCompletions(body.(*jsonutils.JSONDict), "vision-model") + if err != nil { + t.Fatal(err) + } + s := out.String() + if !strings.Contains(s, "image_url") || !strings.Contains(s, "https://example.com/a.png") { + t.Fatalf("expected image-only message preserved, got %s", s) + } +} diff --git a/pkg/aiproxy/providers/openai/responses_anthropic_compat.go b/pkg/aiproxy/providers/openai/responses_anthropic_compat.go index 457c744e7d..d310733e23 100644 --- a/pkg/aiproxy/providers/openai/responses_anthropic_compat.go +++ b/pkg/aiproxy/providers/openai/responses_anthropic_compat.go @@ -177,14 +177,41 @@ func chatMessagesToAnthropic(chatMsgs []*jsonutils.JSONDict) (*jsonutils.JSONArr asst.Set("content", blocks) out.Add(asst) default: - content, _ := msg.GetString("content") + contentRaw, _ := msg.Get("content") + var contentBytes []byte + if contentRaw != nil { + contentBytes = []byte(contentRaw.String()) + } + blocks := ChatContentToAnthropicBlocks(contentBytes) user := jsonutils.NewDict() user.Set("role", jsonutils.NewString("user")) - if content != "" { - user.Set("content", jsonutils.NewString(content)) - } else { + if len(blocks) == 0 { user.Set("content", jsonutils.NewString("")) + out.Add(user) + continue } + if len(blocks) == 1 { + if typ, _ := blocks[0]["type"].(string); typ == "text" { + if text, _ := blocks[0]["text"].(string); text != "" { + user.Set("content", jsonutils.NewString(text)) + out.Add(user) + continue + } + } + } + arr := jsonutils.NewArray() + for _, blk := range blocks { + data, err := json.Marshal(blk) + if err != nil { + continue + } + obj, err := jsonutils.Parse(data) + if err != nil { + continue + } + arr.Add(obj) + } + user.Set("content", arr) out.Add(user) } } diff --git a/pkg/aiproxy/providers/openai/responses_anthropic_compat_test.go b/pkg/aiproxy/providers/openai/responses_anthropic_compat_test.go index 3788491483..e212abf799 100644 --- a/pkg/aiproxy/providers/openai/responses_anthropic_compat_test.go +++ b/pkg/aiproxy/providers/openai/responses_anthropic_compat_test.go @@ -97,3 +97,29 @@ func TestAnthropicMessagesToResponsesBasic(t *testing.T) { t.Fatalf("out=%s", out) } } + +func TestResponsesToAnthropicMessagesWithImage(t *testing.T) { + raw := `{ + "model":"claude-test", + "max_output_tokens":128, + "input":[{"role":"user","content":[ + {"type":"input_text","text":"look"}, + {"type":"input_image","image_url":"https://example.com/x.png"} + ]}] + }` + body, err := jsonutils.Parse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + out, _, err := ResponsesToAnthropicMessages(body.(*jsonutils.JSONDict), "claude-up") + if err != nil { + t.Fatal(err) + } + s := out.String() + if !strings.Contains(s, `"type":"image"`) && !strings.Contains(s, `"type": "image"`) { + t.Fatalf("expected anthropic image block, got %s", s) + } + if !strings.Contains(s, "https://example.com/x.png") { + t.Fatalf("expected image url, got %s", s) + } +} diff --git a/pkg/aiproxy/providers/openai/responses_compat.go b/pkg/aiproxy/providers/openai/responses_compat.go index 33cc7293a6..533ebd35db 100644 --- a/pkg/aiproxy/providers/openai/responses_compat.go +++ b/pkg/aiproxy/providers/openai/responses_compat.go @@ -261,18 +261,12 @@ func responsesInputToMessages(body *jsonutils.JSONDict) ([]*jsonutils.JSONDict, case "assistant": msg := jsonutils.NewDict() msg.Set("role", jsonutils.NewString("assistant")) - if text := responsesContentToText(item.Content); text != "" { - msg.Set("content", jsonutils.NewString(text)) - } + setResponsesMessageContent(msg, item.Content) messages = append(messages, msg) default: msg := jsonutils.NewDict() msg.Set("role", jsonutils.NewString("user")) - if text := responsesContentToText(item.Content); text != "" { - msg.Set("content", jsonutils.NewString(text)) - } else { - msg.Set("content", jsonutils.NewString("")) - } + setResponsesMessageContent(msg, item.Content) messages = append(messages, msg) } } @@ -289,6 +283,126 @@ func isResponsesToolOutputType(t string) bool { } } +type responsesContentPartRaw struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL json.RawMessage `json:"image_url"` +} + +// ResponsesInputHasImage reports whether a Responses request input carries image parts. +func ResponsesInputHasImage(body *jsonutils.JSONDict) bool { + if body == nil { + return false + } + raw, err := body.Get("input") + if err != nil { + return false + } + rawBytes := []byte(raw.String()) + trimmed := strings.TrimSpace(string(rawBytes)) + if trimmed == "" || trimmed == "null" || strings.HasPrefix(trimmed, "\"") { + return false + } + if !strings.HasPrefix(trimmed, "[") { + return false + } + var items []responsesInputItem + if json.Unmarshal(rawBytes, &items) != nil { + return false + } + for _, item := range items { + if responsesContentHasImage(item.Content) { + return true + } + } + return false +} + +func responsesContentHasImage(raw json.RawMessage) bool { + _, hasImage := responsesContentToChatParts(raw) + return hasImage +} + +func setResponsesMessageContent(msg *jsonutils.JSONDict, raw json.RawMessage) { + parts, hasImage := responsesContentToChatParts(raw) + if hasImage { + arr := jsonutils.NewArray() + for _, part := range parts { + arr.Add(part) + } + msg.Set("content", arr) + return + } + text := responsesContentToText(raw) + msg.Set("content", jsonutils.NewString(text)) +} + +// responsesContentToChatParts converts Responses content to chat/completions content parts. +// Returns parts and whether any image_url part was included. +func responsesContentToChatParts(raw json.RawMessage) ([]*jsonutils.JSONDict, bool) { + if len(raw) == 0 || string(raw) == "null" { + return nil, false + } + var s string + if json.Unmarshal(raw, &s) == nil { + if strings.TrimSpace(s) == "" { + return nil, false + } + part := jsonutils.NewDict() + part.Set("type", jsonutils.NewString("text")) + part.Set("text", jsonutils.NewString(s)) + return []*jsonutils.JSONDict{part}, false + } + var parts []responsesContentPartRaw + if json.Unmarshal(raw, &parts) != nil { + return nil, false + } + out := make([]*jsonutils.JSONDict, 0, len(parts)) + hasImage := false + for _, p := range parts { + switch p.Type { + case "input_text", "text", "output_text": + if p.Text == "" { + continue + } + part := jsonutils.NewDict() + part.Set("type", jsonutils.NewString("text")) + part.Set("text", jsonutils.NewString(p.Text)) + out = append(out, part) + case "input_image", "image", "image_url": + src := imageSourceFromRaw(p.ImageURL) + if src == "" { + continue + } + hasImage = true + imgURL := jsonutils.NewDict() + imgURL.Set("url", jsonutils.NewString(src)) + part := jsonutils.NewDict() + part.Set("type", jsonutils.NewString("image_url")) + part.Set("image_url", imgURL) + out = append(out, part) + } + } + return out, hasImage +} + +func imageSourceFromRaw(raw json.RawMessage) string { + if len(raw) == 0 || string(raw) == "null" { + return "" + } + var url string + if err := json.Unmarshal(raw, &url); err == nil { + return strings.TrimSpace(url) + } + var obj struct { + URL string `json:"url"` + } + if err := json.Unmarshal(raw, &obj); err == nil { + return strings.TrimSpace(obj.URL) + } + return "" +} + func responsesContentToText(raw json.RawMessage) string { if len(raw) == 0 || string(raw) == "null" { return "" diff --git a/pkg/aiproxy/providers/openai/responses_compat_test.go b/pkg/aiproxy/providers/openai/responses_compat_test.go index 1ec1af139c..244aa2ba04 100644 --- a/pkg/aiproxy/providers/openai/responses_compat_test.go +++ b/pkg/aiproxy/providers/openai/responses_compat_test.go @@ -130,6 +130,83 @@ func TestFlattenResponsesNamespaceTool(t *testing.T) { } } +func TestResponsesToChatCompletionsWithImageInput(t *testing.T) { + raw := `{ + "model":"kimi-k2.6", + "input":[{"role":"user","content":[ + {"type":"input_text","text":"describe this image"}, + {"type":"input_image","image_url":"data:image/png;base64,abc123"} + ]}] + }` + body, _ := jsonutils.Parse([]byte(raw)) + out, _, err := ResponsesToChatCompletions(body.(*jsonutils.JSONDict), "kimi-k2.6") + if err != nil { + t.Fatal(err) + } + s := out.String() + if !strings.Contains(s, "image_url") { + t.Fatalf("expected image_url in messages, got %s", s) + } + if !strings.Contains(s, "data:image/png;base64,abc123") { + t.Fatalf("expected image data url in messages, got %s", s) + } + if !strings.Contains(s, "describe this image") { + t.Fatalf("expected text in messages, got %s", s) + } +} + +func TestResponsesInputHasImage(t *testing.T) { + body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`)) + if !ResponsesInputHasImage(body.(*jsonutils.JSONDict)) { + t.Fatal("expected image input") + } + body2, _ := jsonutils.Parse([]byte(`{"input":"hello"}`)) + if ResponsesInputHasImage(body2.(*jsonutils.JSONDict)) { + t.Fatal("expected no image for string input") + } +} + +func TestResponsesStreamConverterReasoningAndToolOutputIndex(t *testing.T) { + conv := NewResponsesStreamConverter("kimi-k2.6", nil) + chunk1 := []byte(`{"id":"chatcmpl-1","model":"kimi-k2.6","choices":[{"delta":{"reasoning_content":"think"}}]}`) + events1, err := conv.Feed(chunk1, false) + if err != nil { + t.Fatal(err) + } + chunk2 := []byte(`{"id":"chatcmpl-1","model":"kimi-k2.6","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"view_image","arguments":"{}"}}]}}]}`) + events2, err := conv.Feed(chunk2, false) + if err != nil { + t.Fatal(err) + } + events := append(events1, events2...) + var addedIndex, deltaIndex int + for _, e := range events { + if e.Event != "response.output_item.added" && e.Event != "response.function_call_arguments.delta" { + continue + } + var payload map[string]interface{} + if err := json.Unmarshal(e.Data, &payload); err != nil { + t.Fatal(err) + } + idx, _ := payload["output_index"].(float64) + if e.Event == "response.output_item.added" { + item, _ := payload["item"].(map[string]interface{}) + if item["type"] == "function_call" { + addedIndex = int(idx) + } + } + if e.Event == "response.function_call_arguments.delta" { + deltaIndex = int(idx) + } + } + if addedIndex == 0 || deltaIndex == 0 { + t.Fatalf("expected function_call indices, added=%d delta=%d events=%+v", addedIndex, deltaIndex, events) + } + if addedIndex != deltaIndex { + t.Fatalf("output_index mismatch: added=%d delta=%d", addedIndex, deltaIndex) + } +} + func TestResponsesStreamConverterText(t *testing.T) { conv := NewResponsesStreamConverter("gpt-test", nil) chunk := []byte(`{"id":"chatcmpl-1","model":"gpt-test","choices":[{"delta":{"content":"Hi"}}]}`) diff --git a/pkg/aiproxy/providers/openai/responses_stream_compat.go b/pkg/aiproxy/providers/openai/responses_stream_compat.go index 2aec92dfb7..ceed85bf27 100644 --- a/pkg/aiproxy/providers/openai/responses_stream_compat.go +++ b/pkg/aiproxy/providers/openai/responses_stream_compat.go @@ -289,7 +289,7 @@ func (s *ResponsesStreamConverter) appendToolDelta(tc ToolCall) ([]ResponsesStre if ns != "" { item["namespace"] = ns } - added, err := s.outputItemAdded("function_call", st.itemID, item) + added, err := s.outputItemAddedAt(st.outputIndex, "function_call", st.itemID, item) if err != nil { return nil, err } @@ -313,10 +313,14 @@ func (s *ResponsesStreamConverter) appendToolDelta(tc ToolCall) ([]ResponsesStre } func (s *ResponsesStreamConverter) outputItemAdded(itemType, itemID string, item map[string]interface{}) (ResponsesStreamEvent, error) { + return s.outputItemAddedAt(s.outputIndex, itemType, itemID, item) +} + +func (s *ResponsesStreamConverter) outputItemAddedAt(outputIndex int, itemType, itemID string, item map[string]interface{}) (ResponsesStreamEvent, error) { data := map[string]interface{}{ "type": "response.output_item.added", "sequence_number": s.nextSeq(), - "output_index": s.outputIndex, + "output_index": outputIndex, "item": item, } b, err := json.Marshal(data) diff --git a/pkg/aiproxy/providers/openai/tools.go b/pkg/aiproxy/providers/openai/tools.go index d8bde19f01..1f80965c69 100644 --- a/pkg/aiproxy/providers/openai/tools.go +++ b/pkg/aiproxy/providers/openai/tools.go @@ -168,26 +168,22 @@ func MessagesToAnthropic(msgs []Message) ([]map[string]interface{}, error) { }, }) case "user": - text := MessageTextContent(m.Content) - if text == "" { + blocks := ChatContentToAnthropicBlocks(m.Content) + if len(blocks) == 0 { continue } out = append(out, map[string]interface{}{ - "role": "user", - "content": []map[string]interface{}{ - {"type": "text", "text": text}, - }, + "role": "user", + "content": blocks, }) default: - text := MessageTextContent(m.Content) - if text == "" { + blocks := ChatContentToAnthropicBlocks(m.Content) + if len(blocks) == 0 { continue } out = append(out, map[string]interface{}{ - "role": role, - "content": []map[string]interface{}{ - {"type": "text", "text": text}, - }, + "role": role, + "content": blocks, }) } } @@ -197,6 +193,84 @@ func MessagesToAnthropic(msgs []Message) ([]map[string]interface{}, error) { return out, nil } +// ChatContentToAnthropicBlocks converts OpenAI chat message content (string or parts) +// into Anthropic content blocks, preserving image_url as image sources. +func ChatContentToAnthropicBlocks(raw json.RawMessage) []map[string]interface{} { + if len(raw) == 0 || string(raw) == "null" { + return nil + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + if strings.TrimSpace(s) == "" { + return nil + } + return []map[string]interface{}{{"type": "text", "text": s}} + } + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + ImageURL json.RawMessage `json:"image_url"` + } + if err := json.Unmarshal(raw, &parts); err != nil { + text := MessageTextContent(raw) + if text == "" { + return nil + } + return []map[string]interface{}{{"type": "text", "text": text}} + } + out := make([]map[string]interface{}, 0, len(parts)) + for _, p := range parts { + switch p.Type { + case "text": + if p.Text == "" { + continue + } + out = append(out, map[string]interface{}{"type": "text", "text": p.Text}) + case "image_url": + if block := chatImageURLToAnthropicBlock(p.ImageURL); block != nil { + out = append(out, block) + } + } + } + return out +} + +func chatImageURLToAnthropicBlock(raw json.RawMessage) map[string]interface{} { + url := imageSourceFromRaw(raw) + if url == "" { + return nil + } + source := map[string]interface{}{} + if strings.HasPrefix(url, "data:") { + mediaType, data := splitDataURLForAnthropic(url) + source["type"] = "base64" + source["media_type"] = mediaType + source["data"] = data + } else { + source["type"] = "url" + source["url"] = url + } + return map[string]interface{}{ + "type": "image", + "source": source, + } +} + +func splitDataURLForAnthropic(value string) (mediaType, data string) { + header, payload, ok := strings.Cut(value, ",") + if !ok { + return "image/png", value + } + mediaType = strings.TrimPrefix(header, "data:") + if semicolon := strings.IndexByte(mediaType, ';'); semicolon >= 0 { + mediaType = mediaType[:semicolon] + } + if mediaType == "" { + mediaType = "image/png" + } + return mediaType, payload +} + func assistantContentToAnthropic(m Message) []map[string]interface{} { blocks := make([]map[string]interface{}, 0, 1+len(m.ToolCalls)) if text := MessageTextContent(m.Content); text != "" { diff --git a/pkg/aiproxy/providers/openai/tools_test.go b/pkg/aiproxy/providers/openai/tools_test.go index 05d8fc64e5..f1f7b5449c 100644 --- a/pkg/aiproxy/providers/openai/tools_test.go +++ b/pkg/aiproxy/providers/openai/tools_test.go @@ -157,3 +157,32 @@ func TestNewChatCompletionWithTools(t *testing.T) { t.Fatalf("expected finish_reason tool_calls, got %v", choices[0]["finish_reason"]) } } + +func TestMessagesToAnthropicWithImageURL(t *testing.T) { + content, _ := json.Marshal([]map[string]interface{}{ + {"type": "text", "text": "describe"}, + {"type": "image_url", "image_url": map[string]string{"url": "data:image/png;base64,xyz"}}, + }) + msgs := []Message{{ + Role: "user", + Content: content, + }} + out, err := MessagesToAnthropic(msgs) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 { + t.Fatalf("out=%#v", out) + } + blocks, ok := out[0]["content"].([]map[string]interface{}) + if !ok || len(blocks) != 2 { + t.Fatalf("content=%#v", out[0]["content"]) + } + if blocks[1]["type"] != "image" { + t.Fatalf("expected image block, got %#v", blocks[1]) + } + src, _ := blocks[1]["source"].(map[string]interface{}) + if src["type"] != "base64" || src["data"] != "xyz" { + t.Fatalf("source=%#v", src) + } +} diff --git a/pkg/aiproxy/service/service.go b/pkg/aiproxy/service/service.go index bd3fc70f67..ec321aee99 100644 --- a/pkg/aiproxy/service/service.go +++ b/pkg/aiproxy/service/service.go @@ -48,17 +48,17 @@ func StartService() { log.Fatalf("init local proxy node id: %v", err) } chatlog.Configure(chatlog.Options{ - Enabled: opts.ChatLogEnabled, - LocalDir: opts.ChatLogLocalDir, - UploadEnabled: opts.ChatLogUploadEnabled, - UploadIntervalSeconds: opts.ChatLogUploadIntervalSeconds, - SegmentMinutes: opts.ChatLogSegmentMinutes, - MinioEndpoint: opts.ChatLogMinioEndpoint, - MinioAccessKey: opts.ChatLogMinioAccessKey, - MinioSecretKey: opts.ChatLogMinioSecretKey, - MinioBucket: opts.ChatLogMinioBucket, - MinioSecure: opts.ChatLogMinioSecure, - MinioPrefix: opts.ChatLogMinioPrefix, + Enabled: opts.APILogEnabled, + LocalDir: opts.APILogLocalDir, + UploadEnabled: opts.APILogUploadEnabled, + UploadIntervalSeconds: opts.APILogUploadIntervalSeconds, + SegmentMinutes: opts.APILogSegmentMinutes, + S3Endpoint: opts.APILogS3Endpoint, + S3AccessKey: opts.APILogS3AccessKey, + S3SecretKey: opts.APILogS3SecretKey, + S3Bucket: opts.APILogS3Bucket, + S3Secure: opts.APILogS3Secure, + S3Prefix: opts.APILogS3Prefix, Instance: models.CurrentProxyNodeId(), }) uploadCtx, stopUpload := context.WithCancel(context.Background()) diff --git a/pkg/apis/aiproxy/ai_model.go b/pkg/apis/aiproxy/ai_model.go index 3d66664000..288d4bf03e 100644 --- a/pkg/apis/aiproxy/ai_model.go +++ b/pkg/apis/aiproxy/ai_model.go @@ -15,6 +15,8 @@ package aiproxy import ( + "encoding/json" + "yunion.io/x/onecloud/pkg/apis" ) @@ -29,22 +31,76 @@ type AiModelListInput struct { type AiModelCreateInput struct { apis.EnabledStatusStandaloneResourceCreateInput - AiProviderId string `json:"ai_provider_id"` - ModelKey string `json:"model_key"` + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` + Config *SAiModelConfig `json:"config"` } type AiModelUpdateInput struct { apis.EnabledStatusStandaloneResourceBaseUpdateInput - AiProviderId string `json:"ai_provider_id"` - ModelKey string `json:"model_key"` - Enabled *bool `json:"enabled"` + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` + Enabled *bool `json:"enabled"` + Config *SAiModelConfig `json:"config"` } type AiModelDetails struct { apis.EnabledStatusStandaloneResourceDetails - AiProviderId string `json:"ai_provider_id"` - AiProviderName string `json:"ai_provider_name"` - ModelKey string `json:"model_key"` + AiProviderId string `json:"ai_provider_id"` + AiProviderName string `json:"ai_provider_name"` + ModelKey string `json:"model_key"` + Config *SAiModelConfig `json:"config"` +} + +// SAiModelConfig stores per-model extension settings. +type SAiModelConfig struct { + Extensions *SAiModelExtensions `json:"extensions,omitempty"` +} + +type SAiModelExtensions struct { + Visual *SAiModelVisualConfig `json:"visual,omitempty"` +} + +// SAiModelVisualConfig enables tool-delegated vision for text-only upstream models. +type SAiModelVisualConfig struct { + Enabled bool `json:"enabled"` + VisualAiProviderId string `json:"visual_ai_provider_id"` + VisualModelKey string `json:"visual_model_key"` + MaxRounds int `json:"max_rounds,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` +} + +// String implements gotypes.ISerializable for sqlchemy JSON/compound columns. +func (c *SAiModelConfig) String() string { + if c == nil { + return "{}" + } + b, err := json.Marshal(c) + if err != nil { + return "{}" + } + return string(b) +} + +// IsZero implements gotypes.ISerializable. +func (c *SAiModelConfig) IsZero() bool { + if c == nil { + return true + } + return c.Extensions == nil || c.Extensions.Visual == nil +} + +// VisualEnabled reports whether visual extension is configured and enabled. +func (cfg *SAiModelConfig) VisualEnabled() bool { + return cfg != nil && cfg.Extensions != nil && cfg.Extensions.Visual != nil && cfg.Extensions.Visual.Enabled +} + +// InputModalitiesForCatalog returns Codex input_modalities for this model config. +func (cfg *SAiModelConfig) InputModalitiesForCatalog() []string { + if cfg.VisualEnabled() { + return []string{"text", "image"} + } + return nil } diff --git a/pkg/apis/aiproxy/ai_model_test.go b/pkg/apis/aiproxy/ai_model_test.go new file mode 100644 index 0000000000..06aef3a889 --- /dev/null +++ b/pkg/apis/aiproxy/ai_model_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import "testing" + +func TestSAiModelConfigVisualEnabled(t *testing.T) { + cfg := &SAiModelConfig{ + Extensions: &SAiModelExtensions{ + Visual: &SAiModelVisualConfig{Enabled: true}, + }, + } + if !cfg.VisualEnabled() { + t.Fatal("expected visual enabled") + } + mods := cfg.InputModalitiesForCatalog() + if len(mods) != 2 || mods[1] != "image" { + t.Fatalf("modalities = %#v", mods) + } + if cfg.IsZero() { + t.Fatal("expected non-zero config") + } + if cfg.String() == "{}" { + t.Fatal("expected non-empty String()") + } + var empty *SAiModelConfig + if !empty.IsZero() { + t.Fatal("nil config should be zero") + } +} diff --git a/pkg/apis/aiproxy/serialize_register.go b/pkg/apis/aiproxy/serialize_register.go index 730639eea7..370e520566 100644 --- a/pkg/apis/aiproxy/serialize_register.go +++ b/pkg/apis/aiproxy/serialize_register.go @@ -24,6 +24,9 @@ func init() { gotypes.RegisterSerializable(reflect.TypeOf((*SAiProviderConfig)(nil)), func() gotypes.ISerializable { return &SAiProviderConfig{} }) + gotypes.RegisterSerializable(reflect.TypeOf((*SAiModelConfig)(nil)), func() gotypes.ISerializable { + return &SAiModelConfig{} + }) gotypes.RegisterSerializable(reflect.TypeOf((*SAiKeyRouting)(nil)), func() gotypes.ISerializable { return &SAiKeyRouting{} })