feat(aiproxy): move visual provider to model columns and support streaming responses (#25144)

Promote visual_provider_id/visual_model_key to dedicated ai_model columns,
force non-stream upstream orchestration for visual paths, and synthesize SSE
chunks for streaming Responses requests that include images.
This commit is contained in:
Zexi Li
2026-07-14 18:39:30 +08:00
committed by GitHub
parent 73d7cbbb1f
commit cdc8ab78b5
20 changed files with 817 additions and 97 deletions
+1 -1
View File
@@ -301,7 +301,7 @@ startup_timeout_sec = 3600
tool_timeout_sec = 3600
```
**models_catalog.json**(节选):包含该 virtual key 可见模型的 Codex 元数据(`base_instructions``truncation_policy``shell_type` 等),避免 Codex 回退到内置 preset。
**models_catalog.json**(节选):包含该 virtual key 可见模型的 Codex 元数据(`base_instructions``truncation_policy``shell_type``input_modalities` 等),避免 Codex 回退到内置 preset。若绑定 `ai_model` 已启用 Visual`visual_active``config.extensions.visual.enabled` + `visual_provider_id` + `visual_model_key`),对应条目会写入 `input_modalities: ["text","image"]`,使 Codex 按多模态发图。
**aiproxy.env**
+52 -2
View File
@@ -207,8 +207,50 @@ func newModelInfo(slug, displayName, description string, contextWindow int, inpu
}
}
// ensureImageModality appends "image" when missing, defaulting empty lists to ["text"].
func ensureImageModality(modalities []string) []string {
hasImage := false
hasText := false
for _, m := range modalities {
switch strings.TrimSpace(m) {
case "image":
hasImage = true
case "text":
hasText = true
}
}
if hasImage {
return modalities
}
out := append([]string(nil), modalities...)
if !hasText && len(out) == 0 {
out = []string{"text"}
} else if !hasText {
out = append([]string{"text"}, out...)
}
return append(out, "image")
}
// ApplyVisualModalities sets input_modalities to include image for Visual-active slugs.
func ApplyVisualModalities(catalog []ModelInfo, visualActiveSlugs map[string]struct{}) []ModelInfo {
if len(catalog) == 0 || len(visualActiveSlugs) == 0 {
return catalog
}
out := make([]ModelInfo, len(catalog))
copy(out, catalog)
for i := range out {
if _, ok := visualActiveSlugs[out[i].Slug]; !ok {
continue
}
out[i].InputModalities = ensureImageModality(out[i].InputModalities)
out[i].SupportsImageDetailOriginal = true
}
return out
}
// EnsureModelInCatalog appends a fallback catalog entry when model is not listed.
func EnsureModelInCatalog(catalog []ModelInfo, model string) []ModelInfo {
// When visualActiveSlugs contains the model, the fallback includes image modality.
func EnsureModelInCatalog(catalog []ModelInfo, model string, visualActiveSlugs map[string]struct{}) []ModelInfo {
model = strings.TrimSpace(model)
if model == "" {
return catalog
@@ -218,7 +260,15 @@ func EnsureModelInCatalog(catalog []ModelInfo, model string) []ModelInfo {
return catalog
}
}
return append(catalog, buildModelInfoFromEntry(ModelListEntry{ID: model}))
entry := ModelListEntry{ID: model}
if _, ok := visualActiveSlugs[model]; ok {
entry.InputModalities = []string{"text", "image"}
}
info := buildModelInfoFromEntry(entry)
if _, ok := visualActiveSlugs[model]; ok {
info.SupportsImageDetailOriginal = true
}
return append(catalog, info)
}
// CatalogContextWindow returns context_window for model when present in catalog.
@@ -29,3 +29,50 @@ func TestBuildCatalogVisualModalities(t *testing.T) {
t.Fatalf("modalities = %#v", catalog[0].InputModalities)
}
}
func TestApplyVisualModalities(t *testing.T) {
catalog := BuildCatalogFromIDs([]ModelListEntry{
{ID: "text-only", OwnedBy: "deepseek"},
{ID: "route/flash", OwnedBy: "deepseek", InputModalities: []string{"text"}},
{ID: "already-image", OwnedBy: "deepseek", InputModalities: []string{"text", "image"}},
})
visual := map[string]struct{}{
"route/flash": {},
"already-image": {},
"missing-slug": {},
}
out := ApplyVisualModalities(catalog, visual)
if len(out) != 3 {
t.Fatalf("len = %d", len(out))
}
bySlug := map[string]ModelInfo{}
for _, m := range out {
bySlug[m.Slug] = m
}
if len(bySlug["text-only"].InputModalities) != 1 || bySlug["text-only"].InputModalities[0] != "text" {
t.Fatalf("text-only = %#v", bySlug["text-only"].InputModalities)
}
if !bySlug["route/flash"].SupportsImageDetailOriginal {
t.Fatal("route/flash should support image detail original")
}
mods := bySlug["route/flash"].InputModalities
if len(mods) != 2 || mods[0] != "text" || mods[1] != "image" {
t.Fatalf("route/flash modalities = %#v", mods)
}
mods = bySlug["already-image"].InputModalities
if len(mods) != 2 || mods[1] != "image" {
t.Fatalf("already-image modalities = %#v", mods)
}
}
func TestEnsureImageModality(t *testing.T) {
if got := ensureImageModality(nil); len(got) != 2 || got[0] != "text" || got[1] != "image" {
t.Fatalf("nil = %#v", got)
}
if got := ensureImageModality([]string{"text", "image"}); len(got) != 2 {
t.Fatalf("already has image = %#v", got)
}
if got := ensureImageModality([]string{"audio"}); len(got) != 3 || got[0] != "text" || got[2] != "image" {
t.Fatalf("preserve audio = %#v", got)
}
}
+6 -1
View File
@@ -180,7 +180,12 @@ func Run(session *mcclient.ClientSession, opts *Options) error {
if err != nil {
return err
}
catalog := EnsureModelInCatalog(BuildCatalogFromIDs(entries), model)
visualSlugs, err := resolveVisualActiveClientModelIDs(session, opts.Routing)
if err != nil {
return err
}
catalog := ApplyVisualModalities(BuildCatalogFromIDs(entries), visualSlugs)
catalog = EnsureModelInCatalog(catalog, model, visualSlugs)
providerName := strings.TrimSpace(opts.ProviderName)
if providerName == "" {
+16 -2
View File
@@ -157,14 +157,28 @@ func TestBuildCatalogFromIDs(t *testing.T) {
}
func TestEnsureModelInCatalog(t *testing.T) {
catalog := EnsureModelInCatalog(nil, "custom-model")
catalog := EnsureModelInCatalog(nil, "custom-model", nil)
if len(catalog) != 1 || catalog[0].Slug != "custom-model" {
t.Fatalf("catalog = %+v", catalog)
}
catalog = EnsureModelInCatalog(catalog, "custom-model")
if len(catalog[0].InputModalities) != 1 || catalog[0].InputModalities[0] != "text" {
t.Fatalf("default modalities = %#v", catalog[0].InputModalities)
}
catalog = EnsureModelInCatalog(catalog, "custom-model", nil)
if len(catalog) != 1 {
t.Fatalf("duplicate append: %+v", catalog)
}
visual := map[string]struct{}{"visual-model": {}}
catalog = EnsureModelInCatalog(nil, "visual-model", visual)
if len(catalog) != 1 || catalog[0].Slug != "visual-model" {
t.Fatalf("visual catalog = %+v", catalog)
}
if len(catalog[0].InputModalities) != 2 || catalog[0].InputModalities[1] != "image" {
t.Fatalf("visual modalities = %#v", catalog[0].InputModalities)
}
if !catalog[0].SupportsImageDetailOriginal {
t.Fatal("expected SupportsImageDetailOriginal")
}
}
func TestWriteModelsCatalogJSON(t *testing.T) {
+61 -14
View File
@@ -21,6 +21,7 @@ import (
"yunion.io/x/pkg/errors"
apmodels "yunion.io/x/onecloud/pkg/aiproxy/models"
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
"yunion.io/x/onecloud/pkg/mcclient"
apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
)
@@ -38,9 +39,41 @@ func filterModelEntriesForRouting(session *mcclient.ClientSession, routingNameOr
}
func allowedClientModelIDsForRouting(session *mcclient.ClientSession, routingNameOrID string) (map[string]struct{}, error) {
routing, bindings, modelsById, providers, err := loadRoutingCatalogContext(session, routingNameOrID)
if err != nil {
return nil, err
}
ids := apmodels.ClientFacingModelIDsForRouting(routing, bindings, modelsById, providers)
allowed := make(map[string]struct{}, len(ids))
for _, id := range ids {
allowed[id] = struct{}{}
}
return allowed, nil
}
// resolveVisualActiveClientModelIDs returns client model ids with VisualActive() when --routing is set.
func resolveVisualActiveClientModelIDs(session *mcclient.ClientSession, routingNameOrID string) (map[string]struct{}, error) {
routingNameOrID = strings.TrimSpace(routingNameOrID)
if routingNameOrID == "" {
return nil, nil
}
routing, bindings, modelsById, providers, err := loadRoutingCatalogContext(session, routingNameOrID)
if err != nil {
return nil, err
}
return apmodels.VisualActiveClientModelIDsForRouting(routing, bindings, modelsById, providers), nil
}
func loadRoutingCatalogContext(session *mcclient.ClientSession, routingNameOrID string) (
*apmodels.SAiRouting,
[]apmodels.SAiRoutingModel,
map[string]*apmodels.SAiModel,
map[string]*apmodels.SAiProvider,
error,
) {
routingObj, err := apmodules.AiRoutings.Get(session, routingNameOrID, nil)
if err != nil {
return nil, errors.Wrapf(err, "ai-routing-show %s", routingNameOrID)
return nil, nil, nil, nil, errors.Wrapf(err, "ai-routing-show %s", routingNameOrID)
}
routingID, _ := routingObj.GetString("id")
routing := &apmodels.SAiRouting{
@@ -50,23 +83,17 @@ func allowedClientModelIDsForRouting(session *mcclient.ClientSession, routingNam
bindings, providerIDs, err := fetchRoutingBindings(session, routingID)
if err != nil {
return nil, err
return nil, nil, nil, nil, err
}
modelsById, err := fetchCatalogModelsForRouting(session, routingID)
if err != nil {
return nil, err
return nil, nil, nil, nil, err
}
providers, err := fetchProvidersByIDs(session, providerIDs)
if err != nil {
return nil, err
return nil, nil, nil, nil, err
}
ids := apmodels.ClientFacingModelIDsForRouting(routing, bindings, modelsById, providers)
allowed := make(map[string]struct{}, len(ids))
for _, id := range ids {
allowed[id] = struct{}{}
}
return allowed, nil
return routing, bindings, modelsById, providers, nil
}
func fetchRoutingBindings(session *mcclient.ClientSession, routingID string) ([]apmodels.SAiRoutingModel, []string, error) {
@@ -100,6 +127,7 @@ func fetchCatalogModelsForRouting(session *mcclient.ClientSession, routingID str
params := jsonutils.NewDict()
params.Set("ai_routing_id", jsonutils.NewString(routingID))
params.Set("enabled", jsonutils.JSONTrue)
params.Set("details", jsonutils.JSONTrue)
params.Set("limit", jsonutils.NewInt(500))
result, err := apmodules.AiModels.List(session, params)
if err != nil {
@@ -111,14 +139,33 @@ func fetchCatalogModelsForRouting(session *mcclient.ClientSession, routingID str
if id == "" {
continue
}
out[id] = &apmodels.SAiModel{
AiProviderId: strings.TrimSpace(mustString(item, "ai_provider_id")),
ModelKey: strings.TrimSpace(mustString(item, "model_key")),
mdl := &apmodels.SAiModel{
AiProviderId: strings.TrimSpace(mustString(item, "ai_provider_id")),
ModelKey: strings.TrimSpace(mustString(item, "model_key")),
VisualProviderId: strings.TrimSpace(mustString(item, "visual_provider_id")),
VisualModelKey: strings.TrimSpace(mustString(item, "visual_model_key")),
Config: parseAiModelConfig(item),
}
out[id] = mdl
}
return out, nil
}
func parseAiModelConfig(obj jsonutils.JSONObject) *api.SAiModelConfig {
if obj == nil {
return nil
}
cfgObj, err := obj.Get("config")
if err != nil || cfgObj == nil || cfgObj == jsonutils.JSONNull {
return nil
}
cfg := &api.SAiModelConfig{}
if err := cfgObj.Unmarshal(cfg); err != nil {
return nil
}
return cfg
}
func fetchProvidersByIDs(session *mcclient.ClientSession, ids []string) (map[string]*apmodels.SAiProvider, error) {
out := make(map[string]*apmodels.SAiProvider, len(ids))
for _, id := range ids {
+11
View File
@@ -83,6 +83,7 @@ func callChatCompletions(ctx context.Context, up *models.ChatUpstream, body *jso
if up == nil || body == nil {
return nil, fmt.Errorf("nil upstream or body")
}
forceNonStreamChatBody(body)
req := &upstream.Request{
URL: openai.ChatCompletionsURL(ChatBaseURL(up.BaseURL)),
APIKey: up.APIKey,
@@ -111,6 +112,16 @@ func ChatBaseURL(baseURL string) string {
return base
}
// forceNonStreamChatBody ensures visual orchestration always calls upstream chat/completions
// without stream=true (Codex Responses requests carry stream=true in the converted body).
func forceNonStreamChatBody(body *jsonutils.JSONDict) {
if body == nil {
return
}
body.Remove("stream")
body.Remove("stream_options")
}
func finishReasonFromChatResponse(body []byte) string {
var resp struct {
Choices []struct {
+13 -4
View File
@@ -14,7 +14,12 @@
package visual
import api "yunion.io/x/onecloud/pkg/apis/aiproxy"
import (
"strings"
"yunion.io/x/onecloud/pkg/aiproxy/models"
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
)
// RuntimeConfig is the resolved visual extension settings for one request.
type RuntimeConfig struct {
@@ -40,7 +45,11 @@ func RuntimeConfigFromModel(cfg *api.SAiModelConfig) (RuntimeConfig, *api.SAiMod
return out, vis
}
// Enabled reports whether visual extension is enabled on the model config.
func Enabled(cfg *api.SAiModelConfig) bool {
return cfg != nil && cfg.VisualEnabled()
// Enabled reports whether visual extension is active on the resolved upstream
// (config.enabled plus visual_provider_id / visual_model_key columns).
func Enabled(up *models.ChatUpstream) bool {
if up == nil || !up.ModelConfig.VisualEnabled() {
return false
}
return strings.TrimSpace(up.VisualProviderId) != "" && strings.TrimSpace(up.VisualModelKey) != ""
}
+4 -4
View File
@@ -73,7 +73,7 @@ func anthropicContentHasImage(raw json.RawMessage) bool {
// ShouldHandleMessages reports whether the Messages visual path should run (non-stream only).
func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
if isStream || dict == nil || up == nil || !Enabled(up) {
return false
}
return AnthropicMessagesHasImage(dict)
@@ -81,7 +81,7 @@ func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isS
// ShouldRejectMessagesStreaming reports stream+visual+image (unsupported).
func ShouldRejectMessagesStreaming(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
if !isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
if !isStream || dict == nil || up == nil || !Enabled(up) {
return false
}
return AnthropicMessagesHasImage(dict)
@@ -94,7 +94,7 @@ func HandleMessagesCreate(
textUp *models.ChatUpstream,
) ([]byte, error) {
runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig)
if visCfg == nil {
if visCfg == nil || !visCfg.Enabled {
return nil, fmt.Errorf("visual config is missing")
}
userCred := auth.AdminCredential()
@@ -102,7 +102,7 @@ func HandleMessagesCreate(
if err != nil {
return nil, err
}
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg)
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, textUp.VisualProviderId, textUp.VisualModelKey)
if err != nil {
return nil, err
}
+21 -5
View File
@@ -26,21 +26,23 @@ import (
)
// ShouldHandle reports whether the Responses visual orchestration path should run.
// Streaming requests with images use non-streaming orchestration and synthetic SSE (moon-bridge pattern).
func ShouldHandle(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
_ = isStream
if dict == nil || up == nil || !Enabled(up) {
return false
}
return openai.ResponsesInputHasImage(dict)
}
// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request.
func HandleResponsesCreate(
// HandleResponsesCreateChat runs visual orchestration and returns the upstream chat completion body.
func HandleResponsesCreateChat(
ctx context.Context,
dict *jsonutils.JSONDict,
textUp *models.ChatUpstream,
) ([]byte, *openai.ResponsesConvertState, error) {
runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig)
if visCfg == nil {
if visCfg == nil || !visCfg.Enabled {
return nil, nil, fmt.Errorf("visual config is missing")
}
userCred := auth.AdminCredential()
@@ -48,7 +50,7 @@ func HandleResponsesCreate(
if err != nil {
return nil, nil, err
}
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg)
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, textUp.VisualProviderId, textUp.VisualModelKey)
if err != nil {
return nil, nil, err
}
@@ -57,6 +59,7 @@ func HandleResponsesCreate(
return nil, nil, err
}
chatClone := cloneDict(chatBody)
forceNonStreamChatBody(chatClone)
textUpChat := *textUp
textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL)
visUpChat := *visUp
@@ -65,6 +68,19 @@ func HandleResponsesCreate(
if err != nil {
return nil, nil, err
}
return respBody, state, nil
}
// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request.
func HandleResponsesCreate(
ctx context.Context,
dict *jsonutils.JSONDict,
textUp *models.ChatUpstream,
) ([]byte, *openai.ResponsesConvertState, error) {
respBody, state, err := HandleResponsesCreateChat(ctx, dict, textUp)
if err != nil {
return nil, nil, err
}
out, err := openai.ChatCompletionToResponses(respBody, state)
if err != nil {
return nil, nil, err
+33 -5
View File
@@ -66,9 +66,18 @@ func TestEnabled(t *testing.T) {
Visual: &api.SAiModelVisualConfig{Enabled: true},
},
}
if !Enabled(cfg) {
up := &models.ChatUpstream{
ModelConfig: cfg,
VisualProviderId: "prov-1",
VisualModelKey: "vision-model",
}
if !Enabled(up) {
t.Fatal("expected enabled")
}
up.VisualProviderId = ""
if Enabled(up) {
t.Fatal("expected disabled without visual_provider_id")
}
}
func TestInjectChatTools(t *testing.T) {
@@ -119,6 +128,19 @@ func TestAnthropicMessagesHasImage(t *testing.T) {
}
}
func TestForceNonStreamChatBody(t *testing.T) {
body := jsonutils.NewDict()
body.Set("stream", jsonutils.JSONTrue)
body.Set("stream_options", jsonutils.NewDict())
forceNonStreamChatBody(body)
if stream, _ := body.Bool("stream"); stream {
t.Fatal("expected stream removed")
}
if _, err := body.Get("stream_options"); err == nil {
t.Fatal("expected stream_options removed")
}
}
func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) {
cfg := &api.SAiModelConfig{
Extensions: &api.SAiModelExtensions{
@@ -126,14 +148,20 @@ func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) {
},
}
up := &models.ChatUpstream{
ModelConfig: cfg,
APIMode: "anthropic",
ModelConfig: cfg,
VisualProviderId: "prov-1",
VisualModelKey: "vision-model",
APIMode: "anthropic",
}
body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`))
if !ShouldHandle(body.(*jsonutils.JSONDict), up, false) {
t.Fatal("expected visual handle even with anthropic api_mode")
}
if ShouldHandle(body.(*jsonutils.JSONDict), up, true) {
t.Fatal("stream should not use visual orchestrator")
if !ShouldHandle(body.(*jsonutils.JSONDict), up, true) {
t.Fatal("stream with image should use visual orchestrator")
}
bodyText, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_text","text":"hello"}]}]}`))
if ShouldHandle(bodyText.(*jsonutils.JSONDict), up, true) {
t.Fatal("stream without image should not use visual orchestrator")
}
}
+74 -7
View File
@@ -106,19 +106,42 @@ func handleResponsesCreate(ctx context.Context, w http.ResponseWriter, r *http.R
}
if visual.ShouldHandle(dict, up, isStream) {
bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up)
if !isStream {
bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up)
if err != nil {
dbg.Error("visual orchestration: %v", err)
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err)
return
}
dbg.ClientResponse(bodyOut)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(bodyOut)
if up.AiKeyId != "" {
models.RecordAiKeySuccess(up.AiKeyId)
}
return
}
chatBody, _, err := visual.HandleResponsesCreateChat(ctx, dict, up)
if err != nil {
dbg.Error("visual orchestration: %v", err)
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err)
return
}
dbg.ClientResponse(bodyOut)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(bodyOut)
if up.AiKeyId != "" {
models.RecordAiKeySuccess(up.AiKeyId)
payloads, err := openai.NonStreamChatCompletionToStreamPayloads(chatBody)
if err != nil {
dbg.Error("visual synthetic stream: %v", err)
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual synthetic stream: %v", err)
return
}
adapter, err := responses.GetAdapter(up.ProviderKey, up.APIMode)
if err != nil {
dbg.Error("adapter: %v", err)
writeResponsesError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err)
return
}
writeResponsesSyntheticStream(ctx, w, adapter, up.UpstreamModel, dict, payloads, up.AiKeyId, dbg)
return
}
@@ -428,6 +451,50 @@ func writeResponsesPassthroughStream(ctx context.Context, w http.ResponseWriter,
}
}
func writeResponsesSyntheticStream(
ctx context.Context,
w http.ResponseWriter,
adapter providerapi.ResponsesAdapter,
requestModel string,
requestBody *jsonutils.JSONDict,
payloads [][]byte,
aiKeyId string,
dbg *ProxyDebugSession,
) {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.WriteHeader(http.StatusOK)
flushIf(w)
state := adapter.NewStreamState(requestModel, requestBody)
streamOK := true
outSeq := 0
for _, payload := range payloads {
events, err := adapter.ConvertStreamPayload(state, payload, false)
if err != nil {
dbg.Error("visual synthetic stream convert error: %v", err)
streamOK = false
break
}
outSeq++
writeResponsesSSEEvents(w, events, dbg, outSeq)
}
if streamOK {
events, err := adapter.ConvertStreamPayload(state, nil, true)
if err != nil {
dbg.Error("visual synthetic stream end error: %v", err)
streamOK = false
} else {
outSeq++
writeResponsesSSEEvents(w, events, dbg, outSeq)
}
}
if streamOK && aiKeyId != "" {
models.RecordAiKeySuccess(aiKeyId)
}
}
func writeResponsesTranslatedStream(
ctx context.Context,
w http.ResponseWriter,
+96
View File
@@ -25,6 +25,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/stringutils2"
)
@@ -36,10 +37,22 @@ type SAiModel struct {
AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
// ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo).
ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"`
// VisualProviderId is the ai_provider used for tool-delegated image analysis.
VisualProviderId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
// VisualModelKey is the upstream model id for visual analysis.
VisualModelKey string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"`
// Config stores per-model extension settings (e.g. visual delegation).
Config *api.SAiModelConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"`
}
// VisualActive reports whether visual extension is enabled and columns are set.
func (m *SAiModel) VisualActive() bool {
if m == nil || !m.Config.VisualEnabled() {
return false
}
return strings.TrimSpace(m.VisualProviderId) != "" && strings.TrimSpace(m.VisualModelKey) != ""
}
type SAiModelManager struct {
db.SEnabledStatusStandaloneResourceBaseManager
}
@@ -128,20 +141,43 @@ func (manager *SAiModelManager) FetchCustomizeColumns(
rows := make([]api.AiModelDetails, len(objs))
baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
providerIds := make([]string, len(objs))
visualProviderIds := make([]string, 0, len(objs))
for i := range objs {
rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i]
m := objs[i].(*SAiModel)
providerIds[i] = m.AiProviderId
if vid := strings.TrimSpace(m.VisualProviderId); vid != "" {
visualProviderIds = append(visualProviderIds, vid)
}
}
providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds)
if err != nil {
log.Errorf("FetchIdNameMap2 ai_provider: %v", err)
return rows
}
visualProviderNames, err := db.FetchIdNameMap2(AiProviderManager, visualProviderIds)
if err != nil {
log.Errorf("FetchIdNameMap2 visual ai_provider: %v", err)
visualProviderNames = nil
}
visualProviderKeys, err := db.FetchIdFieldMap2(AiProviderManager, "provider_key", visualProviderIds)
if err != nil {
log.Errorf("FetchIdFieldMap2 visual ai_provider provider_key: %v", err)
visualProviderKeys = nil
}
for i := range rows {
rows[i].AiProviderName, _ = providerNames[providerIds[i]]
m := objs[i].(*SAiModel)
rows[i].VisualProviderId = m.VisualProviderId
rows[i].VisualModelKey = m.VisualModelKey
rows[i].VisualActive = m.VisualActive()
rows[i].Config = m.Config
if visualProviderNames != nil {
rows[i].VisualProviderName, _ = visualProviderNames[m.VisualProviderId]
}
if visualProviderKeys != nil {
rows[i].VisualProviderKey, _ = visualProviderKeys[m.VisualProviderId]
}
}
return rows
}
@@ -179,6 +215,16 @@ func (manager *SAiModelManager) ValidateCreateData(
input.Name = defaultAiModelName(prov.Name, mk)
}
vpid, vmk, err := normalizeAiModelVisualFields(ctx, userCred, input.VisualProviderId, input.VisualModelKey)
if err != nil {
return input, err
}
input.VisualProviderId = vpid
input.VisualModelKey = vmk
if err := validateAiModelVisualSettings(input.Config, vpid, vmk); err != nil {
return input, err
}
return input, nil
}
@@ -219,5 +265,55 @@ func (m *SAiModel) ValidateUpdateData(
}
}
cfg := m.Config
if input.Config != nil {
cfg = input.Config
}
vpid := m.VisualProviderId
vmk := m.VisualModelKey
if strings.TrimSpace(input.VisualProviderId) != "" || strings.TrimSpace(input.VisualModelKey) != "" {
nvpid, nvmk, err := normalizeAiModelVisualFields(ctx, userCred, input.VisualProviderId, input.VisualModelKey)
if err != nil {
return input, err
}
if strings.TrimSpace(input.VisualProviderId) != "" {
vpid = nvpid
input.VisualProviderId = nvpid
}
if strings.TrimSpace(input.VisualModelKey) != "" {
vmk = nvmk
input.VisualModelKey = nvmk
}
}
if err := validateAiModelVisualSettings(cfg, vpid, vmk); err != nil {
return input, err
}
return input, nil
}
func normalizeAiModelVisualFields(
ctx context.Context,
userCred mcclient.TokenCredential,
visualProviderId, visualModelKey string,
) (string, string, error) {
vpid := strings.TrimSpace(visualProviderId)
vmk := strings.TrimSpace(visualModelKey)
if vpid != "" {
prov, err := fetchEnabledAiProvider(ctx, userCred, vpid)
if err != nil {
return "", "", errors.Wrap(err, "visual_provider_id")
}
vpid = prov.Id
}
return vpid, vmk, nil
}
func validateAiModelVisualSettings(cfg *api.SAiModelConfig, visualProviderId, visualModelKey string) error {
vpid := strings.TrimSpace(visualProviderId)
vmk := strings.TrimSpace(visualModelKey)
if cfg.VisualEnabled() && (vpid == "" || vmk == "") {
return errors.Wrap(httperrors.ErrInputParameter, "visual_provider_id and visual_model_key are required when visual is enabled")
}
return nil
}
+22 -20
View File
@@ -72,6 +72,9 @@ type ChatUpstream struct {
AiKeyId string
APIMode string
ModelConfig *api.SAiModelConfig
// VisualProviderId / VisualModelKey come from the resolved text ai_model row.
VisualProviderId string
VisualModelKey string
// VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row.
VirtualKeyId string
@@ -346,19 +349,21 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
}
up := &ChatUpstream{
BaseURL: baseURL,
APIKey: keyRes.Secret,
UpstreamModel: upstreamModel,
ProviderKey: prov.ProviderKey,
AiProviderId: prov.Id,
AiModelId: mdl.Id,
AiKeyId: keyRes.AiKeyId,
VirtualKeyId: vk.Id,
APIMode: apiMode,
ModelConfig: mdl.Config,
ProjectId: vk.ProjectId,
DomainId: vk.DomainId,
RoutingLog: resolved.routingLog,
BaseURL: baseURL,
APIKey: keyRes.Secret,
UpstreamModel: upstreamModel,
ProviderKey: prov.ProviderKey,
AiProviderId: prov.Id,
AiModelId: mdl.Id,
AiKeyId: keyRes.AiKeyId,
VirtualKeyId: vk.Id,
APIMode: apiMode,
ModelConfig: mdl.Config,
VisualProviderId: mdl.VisualProviderId,
VisualModelKey: mdl.VisualModelKey,
ProjectId: vk.ProjectId,
DomainId: vk.DomainId,
RoutingLog: resolved.routingLog,
}
if vk.Limits != nil {
up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest
@@ -368,14 +373,11 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
}
// ResolveVisualUpstream resolves the visual provider upstream for tool-delegated image analysis.
func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, cfg *api.SAiModelVisualConfig) (*ChatUpstream, error) {
if cfg == nil || !cfg.Enabled {
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual extension is not enabled")
}
providerID := strings.TrimSpace(cfg.VisualAiProviderId)
modelKey := strings.TrimSpace(cfg.VisualModelKey)
func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, visualProviderId, visualModelKey string) (*ChatUpstream, error) {
providerID := strings.TrimSpace(visualProviderId)
modelKey := strings.TrimSpace(visualModelKey)
if providerID == "" || modelKey == "" {
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_ai_provider_id and visual_model_key are required")
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_provider_id and visual_model_key are required")
}
pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerID)
if err != nil {
+36 -4
View File
@@ -210,6 +210,40 @@ func ClientFacingModelIDsForRouting(
return out
}
// VisualActiveClientModelIDsForRouting returns client-facing model ids whose
// bound ai_model has VisualActive() (enabled visual extension + provider/model columns).
func VisualActiveClientModelIDsForRouting(
routing *SAiRouting,
bindings []SAiRoutingModel,
modelsById map[string]*SAiModel,
providers map[string]*SAiProvider,
) map[string]struct{} {
out := make(map[string]struct{})
if routing == nil {
return out
}
routeKey := strings.TrimSpace(routing.ModelKey)
for i := range bindings {
e := &bindings[i]
prov := providers[e.AiProviderId]
mdl := modelsById[e.AiModelId]
if prov == nil || mdl == nil || !mdl.VisualActive() {
continue
}
var id string
if routeKey != "" {
id = hierarchicalClientModelID(routing, e, mdl)
} else {
id = clientFacingModelID(routing, e, mdl)
}
if id == "" {
continue
}
out[id] = struct{}{}
}
return out
}
func hierarchicalClientModelID(routing *SAiRouting, entry *SAiRoutingModel, mdl *SAiModel) string {
if routing == nil {
return ""
@@ -300,10 +334,8 @@ func modelsListEntryFromModel(id, ownedBy string, created int64, mdl *SAiModel)
Created: created,
OwnedBy: ownedBy,
}
if mdl != nil && mdl.Config != nil {
if mods := mdl.Config.InputModalitiesForCatalog(); len(mods) > 0 {
entry.InputModalities = mods
}
if mdl != nil && mdl.VisualActive() {
entry.InputModalities = []string{"text", "image"}
}
return entry
}
+44 -1
View File
@@ -14,7 +14,11 @@
package models
import "testing"
import (
"testing"
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
)
func TestClientFacingModelID(t *testing.T) {
mdl := &SAiModel{ModelKey: "gpt-4o-mini"}
@@ -71,6 +75,45 @@ func TestClientFacingModelIDsForRouting(t *testing.T) {
}
}
func TestVisualActiveClientModelIDsForRouting(t *testing.T) {
routing := &SAiRouting{ModelKey: "test-model"}
active := &SAiModel{
ModelKey: "deepseek-v4-flash",
VisualProviderId: "moonshot-id",
VisualModelKey: "moonshot-v1-8k-vision-preview",
Config: &api.SAiModelConfig{
Extensions: &api.SAiModelExtensions{
Visual: &api.SAiModelVisualConfig{Enabled: true},
},
},
}
inactive := &SAiModel{ModelKey: "deepseek-v4-pro"}
prov := &SAiProvider{ProviderKey: "deepseek"}
bindings := []SAiRoutingModel{
{AiProviderId: "p1", AiModelId: "m-active"},
{AiProviderId: "p1", AiModelId: "m-inactive"},
}
modelsById := map[string]*SAiModel{
"m-active": active,
"m-inactive": inactive,
}
providers := map[string]*SAiProvider{"p1": prov}
got := VisualActiveClientModelIDsForRouting(routing, bindings, modelsById, providers)
if len(got) != 1 {
t.Fatalf("len = %d, want 1: %#v", len(got), got)
}
if _, ok := got["test-model/deepseek-v4-flash"]; !ok {
t.Fatalf("missing hierarchical visual id: %#v", got)
}
if _, ok := got["test-model/deepseek-v4-pro"]; ok {
t.Fatal("inactive visual model should not be included")
}
if len(VisualActiveClientModelIDsForRouting(nil, bindings, modelsById, providers)) != 0 {
t.Fatal("nil routing should yield empty set")
}
}
func TestHierarchicalClientModelID(t *testing.T) {
routing := &SAiRouting{ModelKey: "claude"}
mdl := &SAiModel{ModelKey: "claude-sonnet-4-6"}
@@ -234,3 +234,58 @@ func TestResponsesStreamConverterText(t *testing.T) {
t.Fatalf("events = %+v", end)
}
}
func TestNonStreamChatCompletionToStreamPayloads(t *testing.T) {
raw := []byte(`{
"id":"chatcmpl-visual-1",
"model":"deepseek-v4-flash",
"choices":[{
"message":{
"role":"assistant",
"reasoning_content":"thinking",
"content":"Hello from visual"
},
"finish_reason":"stop"
}],
"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}
}`)
payloads, err := NonStreamChatCompletionToStreamPayloads(raw)
if err != nil {
t.Fatal(err)
}
if len(payloads) < 3 {
t.Fatalf("payloads = %d, want at least 3", len(payloads))
}
conv := NewResponsesStreamConverter("deepseek-v4-flash", nil)
var events []ResponsesStreamEvent
for _, payload := range payloads {
chunkEvents, err := conv.Feed(payload, false)
if err != nil {
t.Fatal(err)
}
events = append(events, chunkEvents...)
}
endEvents, err := conv.Feed(nil, true)
if err != nil {
t.Fatal(err)
}
events = append(events, endEvents...)
foundCreated := false
foundText := false
foundCompleted := false
for _, e := range events {
switch e.Event {
case "response.created":
foundCreated = true
case "response.output_text.delta":
foundText = true
case "response.completed":
foundCompleted = true
}
}
if !foundCreated || !foundText || !foundCompleted {
t.Fatalf("events missing lifecycle: created=%v text=%v completed=%v all=%+v", foundCreated, foundText, foundCompleted, events)
}
}
@@ -434,3 +434,135 @@ func ToProviderChunks(events []ResponsesStreamEvent) []providerapi.ResponsesStre
}
return out
}
type syntheticChatChunkDelta struct {
Content *string `json:"content,omitempty"`
ReasoningContent *string `json:"reasoning_content,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
}
type syntheticChatChunkChoice struct {
Delta syntheticChatChunkDelta `json:"delta,omitempty"`
FinishReason string `json:"finish_reason,omitempty"`
}
type syntheticChatChunkPayload struct {
ID string `json:"id,omitempty"`
Model string `json:"model,omitempty"`
Choices []syntheticChatChunkChoice `json:"choices,omitempty"`
Usage *struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
} `json:"usage,omitempty"`
}
// NonStreamChatCompletionToStreamPayloads converts a chat.completion JSON body into
// synthetic chat.completion.chunk payloads for ResponsesStreamConverter.
func NonStreamChatCompletionToStreamPayloads(body []byte) ([][]byte, error) {
var resp struct {
ID string `json:"id"`
Model string `json:"model"`
Choices []struct {
Message struct {
Content json.RawMessage `json:"content"`
ReasoningContent string `json:"reasoning_content"`
ToolCalls []ToolCall `json:"tool_calls"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage *struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(body, &resp); err != nil {
return nil, fmt.Errorf("invalid chat completion: %w", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("empty chat completion choices")
}
choice := resp.Choices[0]
payloads := make([][]byte, 0, 4)
appendChunk := func(chunk syntheticChatChunkPayload) error {
if chunk.ID == "" {
chunk.ID = resp.ID
}
if chunk.Model == "" {
chunk.Model = resp.Model
}
b, err := json.Marshal(chunk)
if err != nil {
return err
}
payloads = append(payloads, b)
return nil
}
if err := appendChunk(syntheticChatChunkPayload{
ID: resp.ID,
Model: resp.Model,
Choices: []syntheticChatChunkChoice{{}},
}); err != nil {
return nil, err
}
if rc := strings.TrimSpace(choice.Message.ReasoningContent); rc != "" {
reason := rc
if err := appendChunk(syntheticChatChunkPayload{
Choices: []syntheticChatChunkChoice{{
Delta: syntheticChatChunkDelta{ReasoningContent: &reason},
}},
}); err != nil {
return nil, err
}
}
if text := strings.TrimSpace(MessageTextContent(choice.Message.Content)); text != "" {
content := text
if err := appendChunk(syntheticChatChunkPayload{
Choices: []syntheticChatChunkChoice{{
Delta: syntheticChatChunkDelta{Content: &content},
}},
}); err != nil {
return nil, err
}
}
for i, tc := range choice.Message.ToolCalls {
call := tc
if call.Index == 0 && i > 0 {
call.Index = i
}
if call.Type == "" {
call.Type = "function"
}
if err := appendChunk(syntheticChatChunkPayload{
Choices: []syntheticChatChunkChoice{{
Delta: syntheticChatChunkDelta{ToolCalls: []ToolCall{call}},
}},
}); err != nil {
return nil, err
}
}
if resp.Usage != nil {
if err := appendChunk(syntheticChatChunkPayload{Usage: resp.Usage}); err != nil {
return nil, err
}
}
if choice.FinishReason == "length" {
if err := appendChunk(syntheticChatChunkPayload{
Choices: []syntheticChatChunkChoice{{
FinishReason: "length",
}},
}); err != nil {
return nil, err
}
}
return payloads, nil
}
+24 -16
View File
@@ -31,27 +31,36 @@ type AiModelListInput struct {
type AiModelCreateInput struct {
apis.EnabledStatusStandaloneResourceCreateInput
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
Config *SAiModelConfig `json:"config"`
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
VisualProviderId string `json:"visual_provider_id"`
VisualModelKey string `json:"visual_model_key"`
Config *SAiModelConfig `json:"config"`
}
type AiModelUpdateInput struct {
apis.EnabledStatusStandaloneResourceBaseUpdateInput
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
Enabled *bool `json:"enabled"`
Config *SAiModelConfig `json:"config"`
AiProviderId string `json:"ai_provider_id"`
ModelKey string `json:"model_key"`
VisualProviderId string `json:"visual_provider_id"`
VisualModelKey string `json:"visual_model_key"`
Enabled *bool `json:"enabled"`
Config *SAiModelConfig `json:"config"`
}
type AiModelDetails struct {
apis.EnabledStatusStandaloneResourceDetails
AiProviderId string `json:"ai_provider_id"`
AiProviderName string `json:"ai_provider_name"`
ModelKey string `json:"model_key"`
Config *SAiModelConfig `json:"config"`
AiProviderId string `json:"ai_provider_id"`
AiProviderName string `json:"ai_provider_name"`
ModelKey string `json:"model_key"`
VisualProviderId string `json:"visual_provider_id"`
VisualProviderName string `json:"visual_provider_name"`
VisualProviderKey string `json:"visual_provider_key"`
VisualModelKey string `json:"visual_model_key"`
VisualActive bool `json:"visual_active"`
Config *SAiModelConfig `json:"config"`
}
// SAiModelConfig stores per-model extension settings.
@@ -64,12 +73,11 @@ type SAiModelExtensions struct {
}
// SAiModelVisualConfig enables tool-delegated vision for text-only upstream models.
// visual_provider_id / visual_model_key live on ai_model columns, not in this JSON.
type SAiModelVisualConfig struct {
Enabled bool `json:"enabled"`
VisualAiProviderId string `json:"visual_ai_provider_id"`
VisualModelKey string `json:"visual_model_key"`
MaxRounds int `json:"max_rounds,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
Enabled bool `json:"enabled"`
MaxRounds int `json:"max_rounds,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
}
// String implements gotypes.ISerializable for sqlchemy JSON/compound columns.
+69 -11
View File
@@ -137,22 +137,42 @@ type AiModelShowOptions struct {
type AiModelCreateOptions struct {
options.BaseCreateOptions
AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"`
ModelKey string `help:"model routing key" json:"model_key"`
Enabled *bool `json:"enabled,omitempty"`
AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"`
ModelKey string `help:"model routing key" json:"model_key"`
VisualProviderId string `help:"ai_provider for tool-delegated vision" json:"visual_provider_id"`
VisualModelKey string `help:"upstream model id for visual analysis" json:"visual_model_key"`
VisualEnabled *bool `help:"enable visual extension" json:"-"`
VisualMaxRounds int `help:"visual orchestration max rounds" json:"-"`
VisualMaxTokens int `help:"visual analysis max tokens" json:"-"`
Enabled *bool `json:"enabled,omitempty"`
}
func (o *AiModelCreateOptions) Params() (jsonutils.JSONObject, error) {
return options.StructToParams(o)
params, err := options.StructToParams(o)
if err != nil {
return nil, err
}
params.Remove("visual_enabled")
params.Remove("visual_max_rounds")
params.Remove("visual_max_tokens")
if err := mergeAiModelVisualConfig(params, o.VisualEnabled, o.VisualMaxRounds, o.VisualMaxTokens, false); err != nil {
return nil, err
}
return params, nil
}
type AiModelUpdateOptions struct {
ID string `help:"ID or name" json:"-"`
Name string `json:"name,omitempty"`
Desc string `json:"description,omitempty"`
AiProviderId string `json:"ai_provider_id,omitempty"`
ModelKey string `json:"model_key,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
ID string `help:"ID or name" json:"-"`
Name string `json:"name,omitempty"`
Desc string `json:"description,omitempty"`
AiProviderId string `json:"ai_provider_id,omitempty"`
ModelKey string `json:"model_key,omitempty"`
VisualProviderId string `json:"visual_provider_id,omitempty"`
VisualModelKey string `json:"visual_model_key,omitempty"`
VisualEnabled *bool `help:"enable visual extension" json:"-"`
VisualMaxRounds int `help:"visual orchestration max rounds" json:"-"`
VisualMaxTokens int `help:"visual analysis max tokens" json:"-"`
Enabled *bool `json:"enabled,omitempty"`
}
func (o *AiModelUpdateOptions) GetId() string {
@@ -160,7 +180,45 @@ func (o *AiModelUpdateOptions) GetId() string {
}
func (o *AiModelUpdateOptions) Params() (jsonutils.JSONObject, error) {
return options.StructToParams(o)
d, err := options.StructToParams(o)
if err != nil {
return nil, err
}
d.Remove("visual_enabled")
d.Remove("visual_max_rounds")
d.Remove("visual_max_tokens")
if err := mergeAiModelVisualConfig(d, o.VisualEnabled, o.VisualMaxRounds, o.VisualMaxTokens, true); err != nil {
return nil, err
}
return d, nil
}
// mergeAiModelVisualConfig builds config.extensions.visual from flat CLI flags.
// requireEnabled=true for update: max_* alone would replace the whole config and drop enabled.
func mergeAiModelVisualConfig(params *jsonutils.JSONDict, enabled *bool, maxRounds, maxTokens int, requireEnabled bool) error {
hasMax := maxRounds > 0 || maxTokens > 0
if enabled == nil && !hasMax {
return nil
}
if requireEnabled && enabled == nil && hasMax {
return errors.Errorf("set --visual-enabled when using --visual-max-rounds or --visual-max-tokens (config is replaced as a whole)")
}
visual := jsonutils.NewDict()
if enabled != nil {
visual.Set("enabled", jsonutils.NewBool(*enabled))
}
if maxRounds > 0 {
visual.Set("max_rounds", jsonutils.NewInt(int64(maxRounds)))
}
if maxTokens > 0 {
visual.Set("max_tokens", jsonutils.NewInt(int64(maxTokens)))
}
extensions := jsonutils.NewDict()
extensions.Set("visual", visual)
config := jsonutils.NewDict()
config.Set("extensions", extensions)
params.Set("config", config)
return nil
}
type AiModelDeleteOptions struct {