mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-08-28 19:45:07 +08:00
feat(aiproxy): move visual provider to model columns and support streaming responses (#25144)
Promote visual_provider_id/visual_model_key to dedicated ai_model columns, force non-stream upstream orchestration for visual paths, and synthesize SSE chunks for streaming Responses requests that include images.
This commit is contained in:
@@ -301,7 +301,7 @@ startup_timeout_sec = 3600
|
||||
tool_timeout_sec = 3600
|
||||
```
|
||||
|
||||
**models_catalog.json**(节选):包含该 virtual key 可见模型的 Codex 元数据(`base_instructions`、`truncation_policy`、`shell_type` 等),避免 Codex 回退到内置 preset。
|
||||
**models_catalog.json**(节选):包含该 virtual key 可见模型的 Codex 元数据(`base_instructions`、`truncation_policy`、`shell_type`、`input_modalities` 等),避免 Codex 回退到内置 preset。若绑定 `ai_model` 已启用 Visual(`visual_active`:`config.extensions.visual.enabled` + `visual_provider_id` + `visual_model_key`),对应条目会写入 `input_modalities: ["text","image"]`,使 Codex 按多模态发图。
|
||||
|
||||
**aiproxy.env**
|
||||
|
||||
|
||||
@@ -207,8 +207,50 @@ func newModelInfo(slug, displayName, description string, contextWindow int, inpu
|
||||
}
|
||||
}
|
||||
|
||||
// ensureImageModality appends "image" when missing, defaulting empty lists to ["text"].
|
||||
func ensureImageModality(modalities []string) []string {
|
||||
hasImage := false
|
||||
hasText := false
|
||||
for _, m := range modalities {
|
||||
switch strings.TrimSpace(m) {
|
||||
case "image":
|
||||
hasImage = true
|
||||
case "text":
|
||||
hasText = true
|
||||
}
|
||||
}
|
||||
if hasImage {
|
||||
return modalities
|
||||
}
|
||||
out := append([]string(nil), modalities...)
|
||||
if !hasText && len(out) == 0 {
|
||||
out = []string{"text"}
|
||||
} else if !hasText {
|
||||
out = append([]string{"text"}, out...)
|
||||
}
|
||||
return append(out, "image")
|
||||
}
|
||||
|
||||
// ApplyVisualModalities sets input_modalities to include image for Visual-active slugs.
|
||||
func ApplyVisualModalities(catalog []ModelInfo, visualActiveSlugs map[string]struct{}) []ModelInfo {
|
||||
if len(catalog) == 0 || len(visualActiveSlugs) == 0 {
|
||||
return catalog
|
||||
}
|
||||
out := make([]ModelInfo, len(catalog))
|
||||
copy(out, catalog)
|
||||
for i := range out {
|
||||
if _, ok := visualActiveSlugs[out[i].Slug]; !ok {
|
||||
continue
|
||||
}
|
||||
out[i].InputModalities = ensureImageModality(out[i].InputModalities)
|
||||
out[i].SupportsImageDetailOriginal = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// EnsureModelInCatalog appends a fallback catalog entry when model is not listed.
|
||||
func EnsureModelInCatalog(catalog []ModelInfo, model string) []ModelInfo {
|
||||
// When visualActiveSlugs contains the model, the fallback includes image modality.
|
||||
func EnsureModelInCatalog(catalog []ModelInfo, model string, visualActiveSlugs map[string]struct{}) []ModelInfo {
|
||||
model = strings.TrimSpace(model)
|
||||
if model == "" {
|
||||
return catalog
|
||||
@@ -218,7 +260,15 @@ func EnsureModelInCatalog(catalog []ModelInfo, model string) []ModelInfo {
|
||||
return catalog
|
||||
}
|
||||
}
|
||||
return append(catalog, buildModelInfoFromEntry(ModelListEntry{ID: model}))
|
||||
entry := ModelListEntry{ID: model}
|
||||
if _, ok := visualActiveSlugs[model]; ok {
|
||||
entry.InputModalities = []string{"text", "image"}
|
||||
}
|
||||
info := buildModelInfoFromEntry(entry)
|
||||
if _, ok := visualActiveSlugs[model]; ok {
|
||||
info.SupportsImageDetailOriginal = true
|
||||
}
|
||||
return append(catalog, info)
|
||||
}
|
||||
|
||||
// CatalogContextWindow returns context_window for model when present in catalog.
|
||||
|
||||
@@ -29,3 +29,50 @@ func TestBuildCatalogVisualModalities(t *testing.T) {
|
||||
t.Fatalf("modalities = %#v", catalog[0].InputModalities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyVisualModalities(t *testing.T) {
|
||||
catalog := BuildCatalogFromIDs([]ModelListEntry{
|
||||
{ID: "text-only", OwnedBy: "deepseek"},
|
||||
{ID: "route/flash", OwnedBy: "deepseek", InputModalities: []string{"text"}},
|
||||
{ID: "already-image", OwnedBy: "deepseek", InputModalities: []string{"text", "image"}},
|
||||
})
|
||||
visual := map[string]struct{}{
|
||||
"route/flash": {},
|
||||
"already-image": {},
|
||||
"missing-slug": {},
|
||||
}
|
||||
out := ApplyVisualModalities(catalog, visual)
|
||||
if len(out) != 3 {
|
||||
t.Fatalf("len = %d", len(out))
|
||||
}
|
||||
bySlug := map[string]ModelInfo{}
|
||||
for _, m := range out {
|
||||
bySlug[m.Slug] = m
|
||||
}
|
||||
if len(bySlug["text-only"].InputModalities) != 1 || bySlug["text-only"].InputModalities[0] != "text" {
|
||||
t.Fatalf("text-only = %#v", bySlug["text-only"].InputModalities)
|
||||
}
|
||||
if !bySlug["route/flash"].SupportsImageDetailOriginal {
|
||||
t.Fatal("route/flash should support image detail original")
|
||||
}
|
||||
mods := bySlug["route/flash"].InputModalities
|
||||
if len(mods) != 2 || mods[0] != "text" || mods[1] != "image" {
|
||||
t.Fatalf("route/flash modalities = %#v", mods)
|
||||
}
|
||||
mods = bySlug["already-image"].InputModalities
|
||||
if len(mods) != 2 || mods[1] != "image" {
|
||||
t.Fatalf("already-image modalities = %#v", mods)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureImageModality(t *testing.T) {
|
||||
if got := ensureImageModality(nil); len(got) != 2 || got[0] != "text" || got[1] != "image" {
|
||||
t.Fatalf("nil = %#v", got)
|
||||
}
|
||||
if got := ensureImageModality([]string{"text", "image"}); len(got) != 2 {
|
||||
t.Fatalf("already has image = %#v", got)
|
||||
}
|
||||
if got := ensureImageModality([]string{"audio"}); len(got) != 3 || got[0] != "text" || got[2] != "image" {
|
||||
t.Fatalf("preserve audio = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,7 +180,12 @@ func Run(session *mcclient.ClientSession, opts *Options) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
catalog := EnsureModelInCatalog(BuildCatalogFromIDs(entries), model)
|
||||
visualSlugs, err := resolveVisualActiveClientModelIDs(session, opts.Routing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
catalog := ApplyVisualModalities(BuildCatalogFromIDs(entries), visualSlugs)
|
||||
catalog = EnsureModelInCatalog(catalog, model, visualSlugs)
|
||||
|
||||
providerName := strings.TrimSpace(opts.ProviderName)
|
||||
if providerName == "" {
|
||||
|
||||
@@ -157,14 +157,28 @@ func TestBuildCatalogFromIDs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnsureModelInCatalog(t *testing.T) {
|
||||
catalog := EnsureModelInCatalog(nil, "custom-model")
|
||||
catalog := EnsureModelInCatalog(nil, "custom-model", nil)
|
||||
if len(catalog) != 1 || catalog[0].Slug != "custom-model" {
|
||||
t.Fatalf("catalog = %+v", catalog)
|
||||
}
|
||||
catalog = EnsureModelInCatalog(catalog, "custom-model")
|
||||
if len(catalog[0].InputModalities) != 1 || catalog[0].InputModalities[0] != "text" {
|
||||
t.Fatalf("default modalities = %#v", catalog[0].InputModalities)
|
||||
}
|
||||
catalog = EnsureModelInCatalog(catalog, "custom-model", nil)
|
||||
if len(catalog) != 1 {
|
||||
t.Fatalf("duplicate append: %+v", catalog)
|
||||
}
|
||||
visual := map[string]struct{}{"visual-model": {}}
|
||||
catalog = EnsureModelInCatalog(nil, "visual-model", visual)
|
||||
if len(catalog) != 1 || catalog[0].Slug != "visual-model" {
|
||||
t.Fatalf("visual catalog = %+v", catalog)
|
||||
}
|
||||
if len(catalog[0].InputModalities) != 2 || catalog[0].InputModalities[1] != "image" {
|
||||
t.Fatalf("visual modalities = %#v", catalog[0].InputModalities)
|
||||
}
|
||||
if !catalog[0].SupportsImageDetailOriginal {
|
||||
t.Fatal("expected SupportsImageDetailOriginal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteModelsCatalogJSON(t *testing.T) {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
apmodels "yunion.io/x/onecloud/pkg/aiproxy/models"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
|
||||
)
|
||||
@@ -38,9 +39,41 @@ func filterModelEntriesForRouting(session *mcclient.ClientSession, routingNameOr
|
||||
}
|
||||
|
||||
func allowedClientModelIDsForRouting(session *mcclient.ClientSession, routingNameOrID string) (map[string]struct{}, error) {
|
||||
routing, bindings, modelsById, providers, err := loadRoutingCatalogContext(session, routingNameOrID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := apmodels.ClientFacingModelIDsForRouting(routing, bindings, modelsById, providers)
|
||||
allowed := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
allowed[id] = struct{}{}
|
||||
}
|
||||
return allowed, nil
|
||||
}
|
||||
|
||||
// resolveVisualActiveClientModelIDs returns client model ids with VisualActive() when --routing is set.
|
||||
func resolveVisualActiveClientModelIDs(session *mcclient.ClientSession, routingNameOrID string) (map[string]struct{}, error) {
|
||||
routingNameOrID = strings.TrimSpace(routingNameOrID)
|
||||
if routingNameOrID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
routing, bindings, modelsById, providers, err := loadRoutingCatalogContext(session, routingNameOrID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return apmodels.VisualActiveClientModelIDsForRouting(routing, bindings, modelsById, providers), nil
|
||||
}
|
||||
|
||||
func loadRoutingCatalogContext(session *mcclient.ClientSession, routingNameOrID string) (
|
||||
*apmodels.SAiRouting,
|
||||
[]apmodels.SAiRoutingModel,
|
||||
map[string]*apmodels.SAiModel,
|
||||
map[string]*apmodels.SAiProvider,
|
||||
error,
|
||||
) {
|
||||
routingObj, err := apmodules.AiRoutings.Get(session, routingNameOrID, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "ai-routing-show %s", routingNameOrID)
|
||||
return nil, nil, nil, nil, errors.Wrapf(err, "ai-routing-show %s", routingNameOrID)
|
||||
}
|
||||
routingID, _ := routingObj.GetString("id")
|
||||
routing := &apmodels.SAiRouting{
|
||||
@@ -50,23 +83,17 @@ func allowedClientModelIDsForRouting(session *mcclient.ClientSession, routingNam
|
||||
|
||||
bindings, providerIDs, err := fetchRoutingBindings(session, routingID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
modelsById, err := fetchCatalogModelsForRouting(session, routingID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
providers, err := fetchProvidersByIDs(session, providerIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
ids := apmodels.ClientFacingModelIDsForRouting(routing, bindings, modelsById, providers)
|
||||
allowed := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
allowed[id] = struct{}{}
|
||||
}
|
||||
return allowed, nil
|
||||
return routing, bindings, modelsById, providers, nil
|
||||
}
|
||||
|
||||
func fetchRoutingBindings(session *mcclient.ClientSession, routingID string) ([]apmodels.SAiRoutingModel, []string, error) {
|
||||
@@ -100,6 +127,7 @@ func fetchCatalogModelsForRouting(session *mcclient.ClientSession, routingID str
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("ai_routing_id", jsonutils.NewString(routingID))
|
||||
params.Set("enabled", jsonutils.JSONTrue)
|
||||
params.Set("details", jsonutils.JSONTrue)
|
||||
params.Set("limit", jsonutils.NewInt(500))
|
||||
result, err := apmodules.AiModels.List(session, params)
|
||||
if err != nil {
|
||||
@@ -111,14 +139,33 @@ func fetchCatalogModelsForRouting(session *mcclient.ClientSession, routingID str
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
out[id] = &apmodels.SAiModel{
|
||||
AiProviderId: strings.TrimSpace(mustString(item, "ai_provider_id")),
|
||||
ModelKey: strings.TrimSpace(mustString(item, "model_key")),
|
||||
mdl := &apmodels.SAiModel{
|
||||
AiProviderId: strings.TrimSpace(mustString(item, "ai_provider_id")),
|
||||
ModelKey: strings.TrimSpace(mustString(item, "model_key")),
|
||||
VisualProviderId: strings.TrimSpace(mustString(item, "visual_provider_id")),
|
||||
VisualModelKey: strings.TrimSpace(mustString(item, "visual_model_key")),
|
||||
Config: parseAiModelConfig(item),
|
||||
}
|
||||
out[id] = mdl
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseAiModelConfig(obj jsonutils.JSONObject) *api.SAiModelConfig {
|
||||
if obj == nil {
|
||||
return nil
|
||||
}
|
||||
cfgObj, err := obj.Get("config")
|
||||
if err != nil || cfgObj == nil || cfgObj == jsonutils.JSONNull {
|
||||
return nil
|
||||
}
|
||||
cfg := &api.SAiModelConfig{}
|
||||
if err := cfgObj.Unmarshal(cfg); err != nil {
|
||||
return nil
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func fetchProvidersByIDs(session *mcclient.ClientSession, ids []string) (map[string]*apmodels.SAiProvider, error) {
|
||||
out := make(map[string]*apmodels.SAiProvider, len(ids))
|
||||
for _, id := range ids {
|
||||
|
||||
@@ -83,6 +83,7 @@ func callChatCompletions(ctx context.Context, up *models.ChatUpstream, body *jso
|
||||
if up == nil || body == nil {
|
||||
return nil, fmt.Errorf("nil upstream or body")
|
||||
}
|
||||
forceNonStreamChatBody(body)
|
||||
req := &upstream.Request{
|
||||
URL: openai.ChatCompletionsURL(ChatBaseURL(up.BaseURL)),
|
||||
APIKey: up.APIKey,
|
||||
@@ -111,6 +112,16 @@ func ChatBaseURL(baseURL string) string {
|
||||
return base
|
||||
}
|
||||
|
||||
// forceNonStreamChatBody ensures visual orchestration always calls upstream chat/completions
|
||||
// without stream=true (Codex Responses requests carry stream=true in the converted body).
|
||||
func forceNonStreamChatBody(body *jsonutils.JSONDict) {
|
||||
if body == nil {
|
||||
return
|
||||
}
|
||||
body.Remove("stream")
|
||||
body.Remove("stream_options")
|
||||
}
|
||||
|
||||
func finishReasonFromChatResponse(body []byte) string {
|
||||
var resp struct {
|
||||
Choices []struct {
|
||||
|
||||
@@ -14,7 +14,12 @@
|
||||
|
||||
package visual
|
||||
|
||||
import api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/models"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
// RuntimeConfig is the resolved visual extension settings for one request.
|
||||
type RuntimeConfig struct {
|
||||
@@ -40,7 +45,11 @@ func RuntimeConfigFromModel(cfg *api.SAiModelConfig) (RuntimeConfig, *api.SAiMod
|
||||
return out, vis
|
||||
}
|
||||
|
||||
// Enabled reports whether visual extension is enabled on the model config.
|
||||
func Enabled(cfg *api.SAiModelConfig) bool {
|
||||
return cfg != nil && cfg.VisualEnabled()
|
||||
// Enabled reports whether visual extension is active on the resolved upstream
|
||||
// (config.enabled plus visual_provider_id / visual_model_key columns).
|
||||
func Enabled(up *models.ChatUpstream) bool {
|
||||
if up == nil || !up.ModelConfig.VisualEnabled() {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(up.VisualProviderId) != "" && strings.TrimSpace(up.VisualModelKey) != ""
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func anthropicContentHasImage(raw json.RawMessage) bool {
|
||||
|
||||
// ShouldHandleMessages reports whether the Messages visual path should run (non-stream only).
|
||||
func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
|
||||
if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
|
||||
if isStream || dict == nil || up == nil || !Enabled(up) {
|
||||
return false
|
||||
}
|
||||
return AnthropicMessagesHasImage(dict)
|
||||
@@ -81,7 +81,7 @@ func ShouldHandleMessages(dict *jsonutils.JSONDict, up *models.ChatUpstream, isS
|
||||
|
||||
// ShouldRejectMessagesStreaming reports stream+visual+image (unsupported).
|
||||
func ShouldRejectMessagesStreaming(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
|
||||
if !isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
|
||||
if !isStream || dict == nil || up == nil || !Enabled(up) {
|
||||
return false
|
||||
}
|
||||
return AnthropicMessagesHasImage(dict)
|
||||
@@ -94,7 +94,7 @@ func HandleMessagesCreate(
|
||||
textUp *models.ChatUpstream,
|
||||
) ([]byte, error) {
|
||||
runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig)
|
||||
if visCfg == nil {
|
||||
if visCfg == nil || !visCfg.Enabled {
|
||||
return nil, fmt.Errorf("visual config is missing")
|
||||
}
|
||||
userCred := auth.AdminCredential()
|
||||
@@ -102,7 +102,7 @@ func HandleMessagesCreate(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg)
|
||||
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, textUp.VisualProviderId, textUp.VisualModelKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -26,21 +26,23 @@ import (
|
||||
)
|
||||
|
||||
// ShouldHandle reports whether the Responses visual orchestration path should run.
|
||||
// Streaming requests with images use non-streaming orchestration and synthetic SSE (moon-bridge pattern).
|
||||
func ShouldHandle(dict *jsonutils.JSONDict, up *models.ChatUpstream, isStream bool) bool {
|
||||
if isStream || dict == nil || up == nil || !Enabled(up.ModelConfig) {
|
||||
_ = isStream
|
||||
if dict == nil || up == nil || !Enabled(up) {
|
||||
return false
|
||||
}
|
||||
return openai.ResponsesInputHasImage(dict)
|
||||
}
|
||||
|
||||
// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request.
|
||||
func HandleResponsesCreate(
|
||||
// HandleResponsesCreateChat runs visual orchestration and returns the upstream chat completion body.
|
||||
func HandleResponsesCreateChat(
|
||||
ctx context.Context,
|
||||
dict *jsonutils.JSONDict,
|
||||
textUp *models.ChatUpstream,
|
||||
) ([]byte, *openai.ResponsesConvertState, error) {
|
||||
runtime, visCfg := RuntimeConfigFromModel(textUp.ModelConfig)
|
||||
if visCfg == nil {
|
||||
if visCfg == nil || !visCfg.Enabled {
|
||||
return nil, nil, fmt.Errorf("visual config is missing")
|
||||
}
|
||||
userCred := auth.AdminCredential()
|
||||
@@ -48,7 +50,7 @@ func HandleResponsesCreate(
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, visCfg)
|
||||
visUp, err := models.ResolveVisualUpstream(ctx, userCred, vk, textUp.VisualProviderId, textUp.VisualModelKey)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -57,6 +59,7 @@ func HandleResponsesCreate(
|
||||
return nil, nil, err
|
||||
}
|
||||
chatClone := cloneDict(chatBody)
|
||||
forceNonStreamChatBody(chatClone)
|
||||
textUpChat := *textUp
|
||||
textUpChat.BaseURL = ChatBaseURL(textUp.BaseURL)
|
||||
visUpChat := *visUp
|
||||
@@ -65,6 +68,19 @@ func HandleResponsesCreate(
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return respBody, state, nil
|
||||
}
|
||||
|
||||
// HandleResponsesCreate runs the visual orchestration loop for a non-streaming Responses request.
|
||||
func HandleResponsesCreate(
|
||||
ctx context.Context,
|
||||
dict *jsonutils.JSONDict,
|
||||
textUp *models.ChatUpstream,
|
||||
) ([]byte, *openai.ResponsesConvertState, error) {
|
||||
respBody, state, err := HandleResponsesCreateChat(ctx, dict, textUp)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out, err := openai.ChatCompletionToResponses(respBody, state)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
@@ -66,9 +66,18 @@ func TestEnabled(t *testing.T) {
|
||||
Visual: &api.SAiModelVisualConfig{Enabled: true},
|
||||
},
|
||||
}
|
||||
if !Enabled(cfg) {
|
||||
up := &models.ChatUpstream{
|
||||
ModelConfig: cfg,
|
||||
VisualProviderId: "prov-1",
|
||||
VisualModelKey: "vision-model",
|
||||
}
|
||||
if !Enabled(up) {
|
||||
t.Fatal("expected enabled")
|
||||
}
|
||||
up.VisualProviderId = ""
|
||||
if Enabled(up) {
|
||||
t.Fatal("expected disabled without visual_provider_id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectChatTools(t *testing.T) {
|
||||
@@ -119,6 +128,19 @@ func TestAnthropicMessagesHasImage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestForceNonStreamChatBody(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("stream", jsonutils.JSONTrue)
|
||||
body.Set("stream_options", jsonutils.NewDict())
|
||||
forceNonStreamChatBody(body)
|
||||
if stream, _ := body.Bool("stream"); stream {
|
||||
t.Fatal("expected stream removed")
|
||||
}
|
||||
if _, err := body.Get("stream_options"); err == nil {
|
||||
t.Fatal("expected stream_options removed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) {
|
||||
cfg := &api.SAiModelConfig{
|
||||
Extensions: &api.SAiModelExtensions{
|
||||
@@ -126,14 +148,20 @@ func TestShouldHandleResponsesIgnoresAPIMode(t *testing.T) {
|
||||
},
|
||||
}
|
||||
up := &models.ChatUpstream{
|
||||
ModelConfig: cfg,
|
||||
APIMode: "anthropic",
|
||||
ModelConfig: cfg,
|
||||
VisualProviderId: "prov-1",
|
||||
VisualModelKey: "vision-model",
|
||||
APIMode: "anthropic",
|
||||
}
|
||||
body, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_image","image_url":"data:image/png;base64,x"}]}]}`))
|
||||
if !ShouldHandle(body.(*jsonutils.JSONDict), up, false) {
|
||||
t.Fatal("expected visual handle even with anthropic api_mode")
|
||||
}
|
||||
if ShouldHandle(body.(*jsonutils.JSONDict), up, true) {
|
||||
t.Fatal("stream should not use visual orchestrator")
|
||||
if !ShouldHandle(body.(*jsonutils.JSONDict), up, true) {
|
||||
t.Fatal("stream with image should use visual orchestrator")
|
||||
}
|
||||
bodyText, _ := jsonutils.Parse([]byte(`{"input":[{"role":"user","content":[{"type":"input_text","text":"hello"}]}]}`))
|
||||
if ShouldHandle(bodyText.(*jsonutils.JSONDict), up, true) {
|
||||
t.Fatal("stream without image should not use visual orchestrator")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,19 +106,42 @@ func handleResponsesCreate(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if visual.ShouldHandle(dict, up, isStream) {
|
||||
bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up)
|
||||
if !isStream {
|
||||
bodyOut, _, err := visual.HandleResponsesCreate(ctx, dict, up)
|
||||
if err != nil {
|
||||
dbg.Error("visual orchestration: %v", err)
|
||||
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err)
|
||||
return
|
||||
}
|
||||
dbg.ClientResponse(bodyOut)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(bodyOut)
|
||||
if up.AiKeyId != "" {
|
||||
models.RecordAiKeySuccess(up.AiKeyId)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
chatBody, _, err := visual.HandleResponsesCreateChat(ctx, dict, up)
|
||||
if err != nil {
|
||||
dbg.Error("visual orchestration: %v", err)
|
||||
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual orchestration: %v", err)
|
||||
return
|
||||
}
|
||||
dbg.ClientResponse(bodyOut)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(bodyOut)
|
||||
if up.AiKeyId != "" {
|
||||
models.RecordAiKeySuccess(up.AiKeyId)
|
||||
payloads, err := openai.NonStreamChatCompletionToStreamPayloads(chatBody)
|
||||
if err != nil {
|
||||
dbg.Error("visual synthetic stream: %v", err)
|
||||
writeResponsesError(ctx, w, http.StatusBadGateway, "api_error", "visual synthetic stream: %v", err)
|
||||
return
|
||||
}
|
||||
adapter, err := responses.GetAdapter(up.ProviderKey, up.APIMode)
|
||||
if err != nil {
|
||||
dbg.Error("adapter: %v", err)
|
||||
writeResponsesError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err)
|
||||
return
|
||||
}
|
||||
writeResponsesSyntheticStream(ctx, w, adapter, up.UpstreamModel, dict, payloads, up.AiKeyId, dbg)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -428,6 +451,50 @@ func writeResponsesPassthroughStream(ctx context.Context, w http.ResponseWriter,
|
||||
}
|
||||
}
|
||||
|
||||
func writeResponsesSyntheticStream(
|
||||
ctx context.Context,
|
||||
w http.ResponseWriter,
|
||||
adapter providerapi.ResponsesAdapter,
|
||||
requestModel string,
|
||||
requestBody *jsonutils.JSONDict,
|
||||
payloads [][]byte,
|
||||
aiKeyId string,
|
||||
dbg *ProxyDebugSession,
|
||||
) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flushIf(w)
|
||||
|
||||
state := adapter.NewStreamState(requestModel, requestBody)
|
||||
streamOK := true
|
||||
outSeq := 0
|
||||
for _, payload := range payloads {
|
||||
events, err := adapter.ConvertStreamPayload(state, payload, false)
|
||||
if err != nil {
|
||||
dbg.Error("visual synthetic stream convert error: %v", err)
|
||||
streamOK = false
|
||||
break
|
||||
}
|
||||
outSeq++
|
||||
writeResponsesSSEEvents(w, events, dbg, outSeq)
|
||||
}
|
||||
if streamOK {
|
||||
events, err := adapter.ConvertStreamPayload(state, nil, true)
|
||||
if err != nil {
|
||||
dbg.Error("visual synthetic stream end error: %v", err)
|
||||
streamOK = false
|
||||
} else {
|
||||
outSeq++
|
||||
writeResponsesSSEEvents(w, events, dbg, outSeq)
|
||||
}
|
||||
}
|
||||
if streamOK && aiKeyId != "" {
|
||||
models.RecordAiKeySuccess(aiKeyId)
|
||||
}
|
||||
}
|
||||
|
||||
func writeResponsesTranslatedStream(
|
||||
ctx context.Context,
|
||||
w http.ResponseWriter,
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/cloudcommon/db"
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
"yunion.io/x/onecloud/pkg/util/stringutils2"
|
||||
)
|
||||
@@ -36,10 +37,22 @@ type SAiModel struct {
|
||||
AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
|
||||
// ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo).
|
||||
ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"`
|
||||
// VisualProviderId is the ai_provider used for tool-delegated image analysis.
|
||||
VisualProviderId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
|
||||
// VisualModelKey is the upstream model id for visual analysis.
|
||||
VisualModelKey string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"`
|
||||
// Config stores per-model extension settings (e.g. visual delegation).
|
||||
Config *api.SAiModelConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"`
|
||||
}
|
||||
|
||||
// VisualActive reports whether visual extension is enabled and columns are set.
|
||||
func (m *SAiModel) VisualActive() bool {
|
||||
if m == nil || !m.Config.VisualEnabled() {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(m.VisualProviderId) != "" && strings.TrimSpace(m.VisualModelKey) != ""
|
||||
}
|
||||
|
||||
type SAiModelManager struct {
|
||||
db.SEnabledStatusStandaloneResourceBaseManager
|
||||
}
|
||||
@@ -128,20 +141,43 @@ func (manager *SAiModelManager) FetchCustomizeColumns(
|
||||
rows := make([]api.AiModelDetails, len(objs))
|
||||
baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
|
||||
providerIds := make([]string, len(objs))
|
||||
visualProviderIds := make([]string, 0, len(objs))
|
||||
for i := range objs {
|
||||
rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i]
|
||||
m := objs[i].(*SAiModel)
|
||||
providerIds[i] = m.AiProviderId
|
||||
if vid := strings.TrimSpace(m.VisualProviderId); vid != "" {
|
||||
visualProviderIds = append(visualProviderIds, vid)
|
||||
}
|
||||
}
|
||||
providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds)
|
||||
if err != nil {
|
||||
log.Errorf("FetchIdNameMap2 ai_provider: %v", err)
|
||||
return rows
|
||||
}
|
||||
visualProviderNames, err := db.FetchIdNameMap2(AiProviderManager, visualProviderIds)
|
||||
if err != nil {
|
||||
log.Errorf("FetchIdNameMap2 visual ai_provider: %v", err)
|
||||
visualProviderNames = nil
|
||||
}
|
||||
visualProviderKeys, err := db.FetchIdFieldMap2(AiProviderManager, "provider_key", visualProviderIds)
|
||||
if err != nil {
|
||||
log.Errorf("FetchIdFieldMap2 visual ai_provider provider_key: %v", err)
|
||||
visualProviderKeys = nil
|
||||
}
|
||||
for i := range rows {
|
||||
rows[i].AiProviderName, _ = providerNames[providerIds[i]]
|
||||
m := objs[i].(*SAiModel)
|
||||
rows[i].VisualProviderId = m.VisualProviderId
|
||||
rows[i].VisualModelKey = m.VisualModelKey
|
||||
rows[i].VisualActive = m.VisualActive()
|
||||
rows[i].Config = m.Config
|
||||
if visualProviderNames != nil {
|
||||
rows[i].VisualProviderName, _ = visualProviderNames[m.VisualProviderId]
|
||||
}
|
||||
if visualProviderKeys != nil {
|
||||
rows[i].VisualProviderKey, _ = visualProviderKeys[m.VisualProviderId]
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
@@ -179,6 +215,16 @@ func (manager *SAiModelManager) ValidateCreateData(
|
||||
input.Name = defaultAiModelName(prov.Name, mk)
|
||||
}
|
||||
|
||||
vpid, vmk, err := normalizeAiModelVisualFields(ctx, userCred, input.VisualProviderId, input.VisualModelKey)
|
||||
if err != nil {
|
||||
return input, err
|
||||
}
|
||||
input.VisualProviderId = vpid
|
||||
input.VisualModelKey = vmk
|
||||
if err := validateAiModelVisualSettings(input.Config, vpid, vmk); err != nil {
|
||||
return input, err
|
||||
}
|
||||
|
||||
return input, nil
|
||||
}
|
||||
|
||||
@@ -219,5 +265,55 @@ func (m *SAiModel) ValidateUpdateData(
|
||||
}
|
||||
}
|
||||
|
||||
cfg := m.Config
|
||||
if input.Config != nil {
|
||||
cfg = input.Config
|
||||
}
|
||||
vpid := m.VisualProviderId
|
||||
vmk := m.VisualModelKey
|
||||
if strings.TrimSpace(input.VisualProviderId) != "" || strings.TrimSpace(input.VisualModelKey) != "" {
|
||||
nvpid, nvmk, err := normalizeAiModelVisualFields(ctx, userCred, input.VisualProviderId, input.VisualModelKey)
|
||||
if err != nil {
|
||||
return input, err
|
||||
}
|
||||
if strings.TrimSpace(input.VisualProviderId) != "" {
|
||||
vpid = nvpid
|
||||
input.VisualProviderId = nvpid
|
||||
}
|
||||
if strings.TrimSpace(input.VisualModelKey) != "" {
|
||||
vmk = nvmk
|
||||
input.VisualModelKey = nvmk
|
||||
}
|
||||
}
|
||||
if err := validateAiModelVisualSettings(cfg, vpid, vmk); err != nil {
|
||||
return input, err
|
||||
}
|
||||
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func normalizeAiModelVisualFields(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
visualProviderId, visualModelKey string,
|
||||
) (string, string, error) {
|
||||
vpid := strings.TrimSpace(visualProviderId)
|
||||
vmk := strings.TrimSpace(visualModelKey)
|
||||
if vpid != "" {
|
||||
prov, err := fetchEnabledAiProvider(ctx, userCred, vpid)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "visual_provider_id")
|
||||
}
|
||||
vpid = prov.Id
|
||||
}
|
||||
return vpid, vmk, nil
|
||||
}
|
||||
|
||||
func validateAiModelVisualSettings(cfg *api.SAiModelConfig, visualProviderId, visualModelKey string) error {
|
||||
vpid := strings.TrimSpace(visualProviderId)
|
||||
vmk := strings.TrimSpace(visualModelKey)
|
||||
if cfg.VisualEnabled() && (vpid == "" || vmk == "") {
|
||||
return errors.Wrap(httperrors.ErrInputParameter, "visual_provider_id and visual_model_key are required when visual is enabled")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -72,6 +72,9 @@ type ChatUpstream struct {
|
||||
AiKeyId string
|
||||
APIMode string
|
||||
ModelConfig *api.SAiModelConfig
|
||||
// VisualProviderId / VisualModelKey come from the resolved text ai_model row.
|
||||
VisualProviderId string
|
||||
VisualModelKey string
|
||||
|
||||
// VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row.
|
||||
VirtualKeyId string
|
||||
@@ -346,19 +349,21 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
|
||||
}
|
||||
|
||||
up := &ChatUpstream{
|
||||
BaseURL: baseURL,
|
||||
APIKey: keyRes.Secret,
|
||||
UpstreamModel: upstreamModel,
|
||||
ProviderKey: prov.ProviderKey,
|
||||
AiProviderId: prov.Id,
|
||||
AiModelId: mdl.Id,
|
||||
AiKeyId: keyRes.AiKeyId,
|
||||
VirtualKeyId: vk.Id,
|
||||
APIMode: apiMode,
|
||||
ModelConfig: mdl.Config,
|
||||
ProjectId: vk.ProjectId,
|
||||
DomainId: vk.DomainId,
|
||||
RoutingLog: resolved.routingLog,
|
||||
BaseURL: baseURL,
|
||||
APIKey: keyRes.Secret,
|
||||
UpstreamModel: upstreamModel,
|
||||
ProviderKey: prov.ProviderKey,
|
||||
AiProviderId: prov.Id,
|
||||
AiModelId: mdl.Id,
|
||||
AiKeyId: keyRes.AiKeyId,
|
||||
VirtualKeyId: vk.Id,
|
||||
APIMode: apiMode,
|
||||
ModelConfig: mdl.Config,
|
||||
VisualProviderId: mdl.VisualProviderId,
|
||||
VisualModelKey: mdl.VisualModelKey,
|
||||
ProjectId: vk.ProjectId,
|
||||
DomainId: vk.DomainId,
|
||||
RoutingLog: resolved.routingLog,
|
||||
}
|
||||
if vk.Limits != nil {
|
||||
up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest
|
||||
@@ -368,14 +373,11 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
|
||||
}
|
||||
|
||||
// ResolveVisualUpstream resolves the visual provider upstream for tool-delegated image analysis.
|
||||
func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, cfg *api.SAiModelVisualConfig) (*ChatUpstream, error) {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual extension is not enabled")
|
||||
}
|
||||
providerID := strings.TrimSpace(cfg.VisualAiProviderId)
|
||||
modelKey := strings.TrimSpace(cfg.VisualModelKey)
|
||||
func ResolveVisualUpstream(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey, visualProviderId, visualModelKey string) (*ChatUpstream, error) {
|
||||
providerID := strings.TrimSpace(visualProviderId)
|
||||
modelKey := strings.TrimSpace(visualModelKey)
|
||||
if providerID == "" || modelKey == "" {
|
||||
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_ai_provider_id and visual_model_key are required")
|
||||
return nil, errors.Wrap(httperrors.ErrInputParameter, "visual_provider_id and visual_model_key are required")
|
||||
}
|
||||
pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerID)
|
||||
if err != nil {
|
||||
|
||||
@@ -210,6 +210,40 @@ func ClientFacingModelIDsForRouting(
|
||||
return out
|
||||
}
|
||||
|
||||
// VisualActiveClientModelIDsForRouting returns client-facing model ids whose
|
||||
// bound ai_model has VisualActive() (enabled visual extension + provider/model columns).
|
||||
func VisualActiveClientModelIDsForRouting(
|
||||
routing *SAiRouting,
|
||||
bindings []SAiRoutingModel,
|
||||
modelsById map[string]*SAiModel,
|
||||
providers map[string]*SAiProvider,
|
||||
) map[string]struct{} {
|
||||
out := make(map[string]struct{})
|
||||
if routing == nil {
|
||||
return out
|
||||
}
|
||||
routeKey := strings.TrimSpace(routing.ModelKey)
|
||||
for i := range bindings {
|
||||
e := &bindings[i]
|
||||
prov := providers[e.AiProviderId]
|
||||
mdl := modelsById[e.AiModelId]
|
||||
if prov == nil || mdl == nil || !mdl.VisualActive() {
|
||||
continue
|
||||
}
|
||||
var id string
|
||||
if routeKey != "" {
|
||||
id = hierarchicalClientModelID(routing, e, mdl)
|
||||
} else {
|
||||
id = clientFacingModelID(routing, e, mdl)
|
||||
}
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
out[id] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func hierarchicalClientModelID(routing *SAiRouting, entry *SAiRoutingModel, mdl *SAiModel) string {
|
||||
if routing == nil {
|
||||
return ""
|
||||
@@ -300,10 +334,8 @@ func modelsListEntryFromModel(id, ownedBy string, created int64, mdl *SAiModel)
|
||||
Created: created,
|
||||
OwnedBy: ownedBy,
|
||||
}
|
||||
if mdl != nil && mdl.Config != nil {
|
||||
if mods := mdl.Config.InputModalitiesForCatalog(); len(mods) > 0 {
|
||||
entry.InputModalities = mods
|
||||
}
|
||||
if mdl != nil && mdl.VisualActive() {
|
||||
entry.InputModalities = []string{"text", "image"}
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
@@ -14,7 +14,11 @@
|
||||
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"testing"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
func TestClientFacingModelID(t *testing.T) {
|
||||
mdl := &SAiModel{ModelKey: "gpt-4o-mini"}
|
||||
@@ -71,6 +75,45 @@ func TestClientFacingModelIDsForRouting(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVisualActiveClientModelIDsForRouting(t *testing.T) {
|
||||
routing := &SAiRouting{ModelKey: "test-model"}
|
||||
active := &SAiModel{
|
||||
ModelKey: "deepseek-v4-flash",
|
||||
VisualProviderId: "moonshot-id",
|
||||
VisualModelKey: "moonshot-v1-8k-vision-preview",
|
||||
Config: &api.SAiModelConfig{
|
||||
Extensions: &api.SAiModelExtensions{
|
||||
Visual: &api.SAiModelVisualConfig{Enabled: true},
|
||||
},
|
||||
},
|
||||
}
|
||||
inactive := &SAiModel{ModelKey: "deepseek-v4-pro"}
|
||||
prov := &SAiProvider{ProviderKey: "deepseek"}
|
||||
bindings := []SAiRoutingModel{
|
||||
{AiProviderId: "p1", AiModelId: "m-active"},
|
||||
{AiProviderId: "p1", AiModelId: "m-inactive"},
|
||||
}
|
||||
modelsById := map[string]*SAiModel{
|
||||
"m-active": active,
|
||||
"m-inactive": inactive,
|
||||
}
|
||||
providers := map[string]*SAiProvider{"p1": prov}
|
||||
|
||||
got := VisualActiveClientModelIDsForRouting(routing, bindings, modelsById, providers)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("len = %d, want 1: %#v", len(got), got)
|
||||
}
|
||||
if _, ok := got["test-model/deepseek-v4-flash"]; !ok {
|
||||
t.Fatalf("missing hierarchical visual id: %#v", got)
|
||||
}
|
||||
if _, ok := got["test-model/deepseek-v4-pro"]; ok {
|
||||
t.Fatal("inactive visual model should not be included")
|
||||
}
|
||||
if len(VisualActiveClientModelIDsForRouting(nil, bindings, modelsById, providers)) != 0 {
|
||||
t.Fatal("nil routing should yield empty set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHierarchicalClientModelID(t *testing.T) {
|
||||
routing := &SAiRouting{ModelKey: "claude"}
|
||||
mdl := &SAiModel{ModelKey: "claude-sonnet-4-6"}
|
||||
|
||||
@@ -234,3 +234,58 @@ func TestResponsesStreamConverterText(t *testing.T) {
|
||||
t.Fatalf("events = %+v", end)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonStreamChatCompletionToStreamPayloads(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"id":"chatcmpl-visual-1",
|
||||
"model":"deepseek-v4-flash",
|
||||
"choices":[{
|
||||
"message":{
|
||||
"role":"assistant",
|
||||
"reasoning_content":"thinking",
|
||||
"content":"Hello from visual"
|
||||
},
|
||||
"finish_reason":"stop"
|
||||
}],
|
||||
"usage":{"prompt_tokens":10,"completion_tokens":5,"total_tokens":15}
|
||||
}`)
|
||||
payloads, err := NonStreamChatCompletionToStreamPayloads(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payloads) < 3 {
|
||||
t.Fatalf("payloads = %d, want at least 3", len(payloads))
|
||||
}
|
||||
|
||||
conv := NewResponsesStreamConverter("deepseek-v4-flash", nil)
|
||||
var events []ResponsesStreamEvent
|
||||
for _, payload := range payloads {
|
||||
chunkEvents, err := conv.Feed(payload, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events = append(events, chunkEvents...)
|
||||
}
|
||||
endEvents, err := conv.Feed(nil, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
events = append(events, endEvents...)
|
||||
|
||||
foundCreated := false
|
||||
foundText := false
|
||||
foundCompleted := false
|
||||
for _, e := range events {
|
||||
switch e.Event {
|
||||
case "response.created":
|
||||
foundCreated = true
|
||||
case "response.output_text.delta":
|
||||
foundText = true
|
||||
case "response.completed":
|
||||
foundCompleted = true
|
||||
}
|
||||
}
|
||||
if !foundCreated || !foundText || !foundCompleted {
|
||||
t.Fatalf("events missing lifecycle: created=%v text=%v completed=%v all=%+v", foundCreated, foundText, foundCompleted, events)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,3 +434,135 @@ func ToProviderChunks(events []ResponsesStreamEvent) []providerapi.ResponsesStre
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type syntheticChatChunkDelta struct {
|
||||
Content *string `json:"content,omitempty"`
|
||||
ReasoningContent *string `json:"reasoning_content,omitempty"`
|
||||
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
||||
}
|
||||
|
||||
type syntheticChatChunkChoice struct {
|
||||
Delta syntheticChatChunkDelta `json:"delta,omitempty"`
|
||||
FinishReason string `json:"finish_reason,omitempty"`
|
||||
}
|
||||
|
||||
type syntheticChatChunkPayload struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Choices []syntheticChatChunkChoice `json:"choices,omitempty"`
|
||||
Usage *struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// NonStreamChatCompletionToStreamPayloads converts a chat.completion JSON body into
|
||||
// synthetic chat.completion.chunk payloads for ResponsesStreamConverter.
|
||||
func NonStreamChatCompletionToStreamPayloads(body []byte) ([][]byte, error) {
|
||||
var resp struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
ReasoningContent string `json:"reasoning_content"`
|
||||
ToolCalls []ToolCall `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage *struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil, fmt.Errorf("invalid chat completion: %w", err)
|
||||
}
|
||||
if len(resp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("empty chat completion choices")
|
||||
}
|
||||
choice := resp.Choices[0]
|
||||
payloads := make([][]byte, 0, 4)
|
||||
|
||||
appendChunk := func(chunk syntheticChatChunkPayload) error {
|
||||
if chunk.ID == "" {
|
||||
chunk.ID = resp.ID
|
||||
}
|
||||
if chunk.Model == "" {
|
||||
chunk.Model = resp.Model
|
||||
}
|
||||
b, err := json.Marshal(chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payloads = append(payloads, b)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := appendChunk(syntheticChatChunkPayload{
|
||||
ID: resp.ID,
|
||||
Model: resp.Model,
|
||||
Choices: []syntheticChatChunkChoice{{}},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if rc := strings.TrimSpace(choice.Message.ReasoningContent); rc != "" {
|
||||
reason := rc
|
||||
if err := appendChunk(syntheticChatChunkPayload{
|
||||
Choices: []syntheticChatChunkChoice{{
|
||||
Delta: syntheticChatChunkDelta{ReasoningContent: &reason},
|
||||
}},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if text := strings.TrimSpace(MessageTextContent(choice.Message.Content)); text != "" {
|
||||
content := text
|
||||
if err := appendChunk(syntheticChatChunkPayload{
|
||||
Choices: []syntheticChatChunkChoice{{
|
||||
Delta: syntheticChatChunkDelta{Content: &content},
|
||||
}},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for i, tc := range choice.Message.ToolCalls {
|
||||
call := tc
|
||||
if call.Index == 0 && i > 0 {
|
||||
call.Index = i
|
||||
}
|
||||
if call.Type == "" {
|
||||
call.Type = "function"
|
||||
}
|
||||
if err := appendChunk(syntheticChatChunkPayload{
|
||||
Choices: []syntheticChatChunkChoice{{
|
||||
Delta: syntheticChatChunkDelta{ToolCalls: []ToolCall{call}},
|
||||
}},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if resp.Usage != nil {
|
||||
if err := appendChunk(syntheticChatChunkPayload{Usage: resp.Usage}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if choice.FinishReason == "length" {
|
||||
if err := appendChunk(syntheticChatChunkPayload{
|
||||
Choices: []syntheticChatChunkChoice{{
|
||||
FinishReason: "length",
|
||||
}},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return payloads, nil
|
||||
}
|
||||
|
||||
@@ -31,27 +31,36 @@ type AiModelListInput struct {
|
||||
type AiModelCreateInput struct {
|
||||
apis.EnabledStatusStandaloneResourceCreateInput
|
||||
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
ModelKey string `json:"model_key"`
|
||||
Config *SAiModelConfig `json:"config"`
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
ModelKey string `json:"model_key"`
|
||||
VisualProviderId string `json:"visual_provider_id"`
|
||||
VisualModelKey string `json:"visual_model_key"`
|
||||
Config *SAiModelConfig `json:"config"`
|
||||
}
|
||||
|
||||
type AiModelUpdateInput struct {
|
||||
apis.EnabledStatusStandaloneResourceBaseUpdateInput
|
||||
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
ModelKey string `json:"model_key"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
Config *SAiModelConfig `json:"config"`
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
ModelKey string `json:"model_key"`
|
||||
VisualProviderId string `json:"visual_provider_id"`
|
||||
VisualModelKey string `json:"visual_model_key"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
Config *SAiModelConfig `json:"config"`
|
||||
}
|
||||
|
||||
type AiModelDetails struct {
|
||||
apis.EnabledStatusStandaloneResourceDetails
|
||||
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
AiProviderName string `json:"ai_provider_name"`
|
||||
ModelKey string `json:"model_key"`
|
||||
Config *SAiModelConfig `json:"config"`
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
AiProviderName string `json:"ai_provider_name"`
|
||||
ModelKey string `json:"model_key"`
|
||||
VisualProviderId string `json:"visual_provider_id"`
|
||||
VisualProviderName string `json:"visual_provider_name"`
|
||||
VisualProviderKey string `json:"visual_provider_key"`
|
||||
VisualModelKey string `json:"visual_model_key"`
|
||||
VisualActive bool `json:"visual_active"`
|
||||
Config *SAiModelConfig `json:"config"`
|
||||
}
|
||||
|
||||
// SAiModelConfig stores per-model extension settings.
|
||||
@@ -64,12 +73,11 @@ type SAiModelExtensions struct {
|
||||
}
|
||||
|
||||
// SAiModelVisualConfig enables tool-delegated vision for text-only upstream models.
|
||||
// visual_provider_id / visual_model_key live on ai_model columns, not in this JSON.
|
||||
type SAiModelVisualConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
VisualAiProviderId string `json:"visual_ai_provider_id"`
|
||||
VisualModelKey string `json:"visual_model_key"`
|
||||
MaxRounds int `json:"max_rounds,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
MaxRounds int `json:"max_rounds,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
}
|
||||
|
||||
// String implements gotypes.ISerializable for sqlchemy JSON/compound columns.
|
||||
|
||||
@@ -137,22 +137,42 @@ type AiModelShowOptions struct {
|
||||
type AiModelCreateOptions struct {
|
||||
options.BaseCreateOptions
|
||||
|
||||
AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"`
|
||||
ModelKey string `help:"model routing key" json:"model_key"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"`
|
||||
ModelKey string `help:"model routing key" json:"model_key"`
|
||||
VisualProviderId string `help:"ai_provider for tool-delegated vision" json:"visual_provider_id"`
|
||||
VisualModelKey string `help:"upstream model id for visual analysis" json:"visual_model_key"`
|
||||
VisualEnabled *bool `help:"enable visual extension" json:"-"`
|
||||
VisualMaxRounds int `help:"visual orchestration max rounds" json:"-"`
|
||||
VisualMaxTokens int `help:"visual analysis max tokens" json:"-"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
func (o *AiModelCreateOptions) Params() (jsonutils.JSONObject, error) {
|
||||
return options.StructToParams(o)
|
||||
params, err := options.StructToParams(o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params.Remove("visual_enabled")
|
||||
params.Remove("visual_max_rounds")
|
||||
params.Remove("visual_max_tokens")
|
||||
if err := mergeAiModelVisualConfig(params, o.VisualEnabled, o.VisualMaxRounds, o.VisualMaxTokens, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return params, nil
|
||||
}
|
||||
|
||||
type AiModelUpdateOptions struct {
|
||||
ID string `help:"ID or name" json:"-"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Desc string `json:"description,omitempty"`
|
||||
AiProviderId string `json:"ai_provider_id,omitempty"`
|
||||
ModelKey string `json:"model_key,omitempty"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
ID string `help:"ID or name" json:"-"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Desc string `json:"description,omitempty"`
|
||||
AiProviderId string `json:"ai_provider_id,omitempty"`
|
||||
ModelKey string `json:"model_key,omitempty"`
|
||||
VisualProviderId string `json:"visual_provider_id,omitempty"`
|
||||
VisualModelKey string `json:"visual_model_key,omitempty"`
|
||||
VisualEnabled *bool `help:"enable visual extension" json:"-"`
|
||||
VisualMaxRounds int `help:"visual orchestration max rounds" json:"-"`
|
||||
VisualMaxTokens int `help:"visual analysis max tokens" json:"-"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
func (o *AiModelUpdateOptions) GetId() string {
|
||||
@@ -160,7 +180,45 @@ func (o *AiModelUpdateOptions) GetId() string {
|
||||
}
|
||||
|
||||
func (o *AiModelUpdateOptions) Params() (jsonutils.JSONObject, error) {
|
||||
return options.StructToParams(o)
|
||||
d, err := options.StructToParams(o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Remove("visual_enabled")
|
||||
d.Remove("visual_max_rounds")
|
||||
d.Remove("visual_max_tokens")
|
||||
if err := mergeAiModelVisualConfig(d, o.VisualEnabled, o.VisualMaxRounds, o.VisualMaxTokens, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// mergeAiModelVisualConfig builds config.extensions.visual from flat CLI flags.
|
||||
// requireEnabled=true for update: max_* alone would replace the whole config and drop enabled.
|
||||
func mergeAiModelVisualConfig(params *jsonutils.JSONDict, enabled *bool, maxRounds, maxTokens int, requireEnabled bool) error {
|
||||
hasMax := maxRounds > 0 || maxTokens > 0
|
||||
if enabled == nil && !hasMax {
|
||||
return nil
|
||||
}
|
||||
if requireEnabled && enabled == nil && hasMax {
|
||||
return errors.Errorf("set --visual-enabled when using --visual-max-rounds or --visual-max-tokens (config is replaced as a whole)")
|
||||
}
|
||||
visual := jsonutils.NewDict()
|
||||
if enabled != nil {
|
||||
visual.Set("enabled", jsonutils.NewBool(*enabled))
|
||||
}
|
||||
if maxRounds > 0 {
|
||||
visual.Set("max_rounds", jsonutils.NewInt(int64(maxRounds)))
|
||||
}
|
||||
if maxTokens > 0 {
|
||||
visual.Set("max_tokens", jsonutils.NewInt(int64(maxTokens)))
|
||||
}
|
||||
extensions := jsonutils.NewDict()
|
||||
extensions.Set("visual", visual)
|
||||
config := jsonutils.NewDict()
|
||||
config.Set("extensions", extensions)
|
||||
params.Set("config", config)
|
||||
return nil
|
||||
}
|
||||
|
||||
type AiModelDeleteOptions struct {
|
||||
|
||||
Reference in New Issue
Block a user