From 7cff45fb5beb8990e0eb4da781077787cfd8aaa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 9 Jun 2026 18:54:54 +0800 Subject: [PATCH] feat(aiproxy): add aiproxy (#25003) --- build/aiproxy/vars | 1 + build/docker/Dockerfile.aiproxy | 3 + cmd/aiproxy/main.go | 26 + cmd/climc/main.go | 1 + cmd/climc/shell/aiproxy/ai_key.go | 31 + cmd/climc/shell/aiproxy/ai_model.go | 31 + cmd/climc/shell/aiproxy/ai_provider.go | 31 + cmd/climc/shell/aiproxy/ai_proxy_node.go | 32 ++ cmd/climc/shell/aiproxy/ai_routing.go | 32 ++ cmd/climc/shell/aiproxy/ai_routing_model.go | 30 + cmd/climc/shell/aiproxy/ai_virtual_key.go | 31 + cmd/climc/shell/aiproxy/doc.go | 16 + cmd/climc/shell/aiproxy/enabled.go | 25 + docs/aiproxy/functional-test-climc-mimo.md | 114 ++++ docs/aiproxy/functional-test-climc.md | 303 ++++++++++ pkg/aiproxy/handlers/chat_completions.go | 368 ++++++++++++ pkg/aiproxy/handlers/completions.go | 258 +++++++++ pkg/aiproxy/handlers/doc.go | 15 + pkg/aiproxy/handlers/embeddings.go | 146 +++++ pkg/aiproxy/handlers/handlers.go | 75 +++ pkg/aiproxy/handlers/images_generations.go | 146 +++++ pkg/aiproxy/handlers/models.go | 84 +++ pkg/aiproxy/models/ai_key_health.go | 135 +++++ pkg/aiproxy/models/ai_key_resolve.go | 205 +++++++ pkg/aiproxy/models/ai_keys.go | 165 ++++++ pkg/aiproxy/models/ai_models.go | 179 ++++++ pkg/aiproxy/models/ai_providers.go | 155 +++++ pkg/aiproxy/models/ai_proxy_nodes.go | 354 ++++++++++++ pkg/aiproxy/models/ai_routing_models.go | 337 +++++++++++ pkg/aiproxy/models/ai_routings.go | 236 ++++++++ pkg/aiproxy/models/ai_virtual_keys.go | 314 +++++++++++ .../models/aiproxy_catalog_validate.go | 181 ++++++ pkg/aiproxy/models/catalog_seed.go | 229 ++++++++ pkg/aiproxy/models/catalog_seed_models.go | 286 ++++++++++ pkg/aiproxy/models/chat_upstream.go | 255 +++++++++ pkg/aiproxy/models/doc.go | 1 + pkg/aiproxy/models/initdb.go | 46 ++ pkg/aiproxy/models/list_models.go | 190 +++++++ pkg/aiproxy/models/list_models_test.go | 37 ++ pkg/aiproxy/models/proxy_node_local.go | 71 +++ pkg/aiproxy/models/virtual_key_guard.go | 63 +++ pkg/aiproxy/options/doc.go | 15 + pkg/aiproxy/options/options.go | 45 ++ pkg/aiproxy/policy/defaults.go | 58 ++ pkg/aiproxy/policy/doc.go | 15 + pkg/aiproxy/policy/resources.go | 34 ++ pkg/aiproxy/providerapi/doc.go | 1 + pkg/aiproxy/providerapi/stream.go | 20 + pkg/aiproxy/providerapi/types.go | 84 +++ pkg/aiproxy/providers/aliyun/aliyun.go | 37 ++ pkg/aiproxy/providers/aliyun/doc.go | 1 + pkg/aiproxy/providers/anthropic/anthropic.go | 285 ++++++++++ pkg/aiproxy/providers/anthropic/doc.go | 1 + pkg/aiproxy/providers/azure/azure.go | 117 ++++ pkg/aiproxy/providers/azure/doc.go | 1 + pkg/aiproxy/providers/baidu/baidu.go | 103 ++++ pkg/aiproxy/providers/baidu/baidu_test.go | 116 ++++ pkg/aiproxy/providers/baidu/doc.go | 1 + pkg/aiproxy/providers/baidu/token.go | 146 +++++ pkg/aiproxy/providers/baidu/wenxin_v1.go | 235 ++++++++ pkg/aiproxy/providers/bridge.go | 33 ++ pkg/aiproxy/providers/cohere/cohere.go | 94 ++++ pkg/aiproxy/providers/cohere/doc.go | 1 + pkg/aiproxy/providers/completions.go | 28 + pkg/aiproxy/providers/doc.go | 5 + pkg/aiproxy/providers/embeddings.go | 30 + pkg/aiproxy/providers/embeddings_test.go | 121 ++++ pkg/aiproxy/providers/gemini/doc.go | 1 + pkg/aiproxy/providers/gemini/gemini.go | 367 ++++++++++++ pkg/aiproxy/providers/images.go | 30 + pkg/aiproxy/providers/images_test.go | 121 ++++ pkg/aiproxy/providers/openai/compat.go | 102 ++++ pkg/aiproxy/providers/openai/completions.go | 63 +++ pkg/aiproxy/providers/openai/doc.go | 1 + pkg/aiproxy/providers/openai/embeddings.go | 49 ++ pkg/aiproxy/providers/openai/images.go | 49 ++ pkg/aiproxy/providers/openai/schema.go | 414 ++++++++++++++ pkg/aiproxy/providers/openai/tools.go | 528 ++++++++++++++++++ pkg/aiproxy/providers/openai/tools_test.go | 136 +++++ pkg/aiproxy/providers/providers_test.go | 196 +++++++ pkg/aiproxy/providers/registry.go | 105 ++++ pkg/aiproxy/providers/stream.go | 25 + pkg/aiproxy/providers/types.go | 32 ++ pkg/aiproxy/providers/vllm/doc.go | 2 + pkg/aiproxy/providers/vllm/vllm.go | 57 ++ pkg/aiproxy/providers/vllm/vllm_test.go | 77 +++ pkg/aiproxy/service/doc.go | 15 + pkg/aiproxy/service/service.go | 92 +++ pkg/aiproxy/service/slave_register.go | 67 +++ pkg/aiproxy/upstream/doc.go | 1 + pkg/aiproxy/upstream/openai_compat.go | 281 ++++++++++ pkg/apis/aiproxy/ai_key.go | 91 +++ pkg/apis/aiproxy/ai_model.go | 49 ++ pkg/apis/aiproxy/ai_provider.go | 92 +++ pkg/apis/aiproxy/ai_proxy_node.go | 65 +++ pkg/apis/aiproxy/ai_routing.go | 72 +++ pkg/apis/aiproxy/ai_routing_model.go | 61 ++ pkg/apis/aiproxy/ai_virtual_key.go | 96 ++++ pkg/apis/aiproxy/consts.go | 24 + pkg/apis/aiproxy/doc.go | 15 + pkg/apis/aiproxy/serialize_register.go | 33 ++ pkg/apis/compute/guestnetwork.go | 2 +- pkg/apis/const.go | 4 +- pkg/compute/hostdrivers/proxmox.go | 5 +- .../models/guestnetwork_traffic_log.go | 1 + pkg/compute/models/guestnetworksecgroups.go | 1 + pkg/compute/regiondrivers/ecloud.go | 3 +- pkg/esxi/handler/proxmox.go | 5 +- pkg/hostman/storageman/storage_proxmox.go | 7 +- pkg/llm/service/handler.go | 1 + pkg/mcclient/modules/aiproxy/doc.go | 16 + pkg/mcclient/modules/aiproxy/mod_ai_keys.go | 35 ++ pkg/mcclient/modules/aiproxy/mod_ai_models.go | 35 ++ .../modules/aiproxy/mod_ai_providers.go | 35 ++ .../modules/aiproxy/mod_ai_proxy_nodes.go | 35 ++ .../modules/aiproxy/mod_ai_routing_models.go | 35 ++ .../modules/aiproxy/mod_ai_routings.go | 35 ++ .../modules/aiproxy/mod_ai_virtual_keys.go | 35 ++ pkg/mcclient/modules/llm/mod_llm_model_set.go | 1 + pkg/mcclient/modules/loader/loader.go | 1 + pkg/mcclient/modules/managers.go | 6 + pkg/mcclient/options/aiproxy/doc.go | 15 + pkg/mcclient/options/aiproxy/resources.go | 480 ++++++++++++++++ .../options/compute/servernetworks.go | 1 + .../aiproxy-ai-provider-create-test.sh | 171 ++++++ .../aiproxy/aiproxy-functional-test-common.sh | 396 +++++++++++++ .../aiproxy/aiproxy-functional-test-mimo.sh | 11 + .../aiproxy/aiproxy-functional-test-qwen.sh | 11 + .../test/aiproxy/aiproxy-functional-test.sh | 46 ++ 129 files changed, 11693 insertions(+), 10 deletions(-) create mode 100644 build/aiproxy/vars create mode 100644 build/docker/Dockerfile.aiproxy create mode 100644 cmd/aiproxy/main.go create mode 100644 cmd/climc/shell/aiproxy/ai_key.go create mode 100644 cmd/climc/shell/aiproxy/ai_model.go create mode 100644 cmd/climc/shell/aiproxy/ai_provider.go create mode 100644 cmd/climc/shell/aiproxy/ai_proxy_node.go create mode 100644 cmd/climc/shell/aiproxy/ai_routing.go create mode 100644 cmd/climc/shell/aiproxy/ai_routing_model.go create mode 100644 cmd/climc/shell/aiproxy/ai_virtual_key.go create mode 100644 cmd/climc/shell/aiproxy/doc.go create mode 100644 cmd/climc/shell/aiproxy/enabled.go create mode 100644 docs/aiproxy/functional-test-climc-mimo.md create mode 100644 docs/aiproxy/functional-test-climc.md create mode 100644 pkg/aiproxy/handlers/chat_completions.go create mode 100644 pkg/aiproxy/handlers/completions.go create mode 100644 pkg/aiproxy/handlers/doc.go create mode 100644 pkg/aiproxy/handlers/embeddings.go create mode 100644 pkg/aiproxy/handlers/handlers.go create mode 100644 pkg/aiproxy/handlers/images_generations.go create mode 100644 pkg/aiproxy/handlers/models.go create mode 100644 pkg/aiproxy/models/ai_key_health.go create mode 100644 pkg/aiproxy/models/ai_key_resolve.go create mode 100644 pkg/aiproxy/models/ai_keys.go create mode 100644 pkg/aiproxy/models/ai_models.go create mode 100644 pkg/aiproxy/models/ai_providers.go create mode 100644 pkg/aiproxy/models/ai_proxy_nodes.go create mode 100644 pkg/aiproxy/models/ai_routing_models.go create mode 100644 pkg/aiproxy/models/ai_routings.go create mode 100644 pkg/aiproxy/models/ai_virtual_keys.go create mode 100644 pkg/aiproxy/models/aiproxy_catalog_validate.go create mode 100644 pkg/aiproxy/models/catalog_seed.go create mode 100644 pkg/aiproxy/models/catalog_seed_models.go create mode 100644 pkg/aiproxy/models/chat_upstream.go create mode 100644 pkg/aiproxy/models/doc.go create mode 100644 pkg/aiproxy/models/initdb.go create mode 100644 pkg/aiproxy/models/list_models.go create mode 100644 pkg/aiproxy/models/list_models_test.go create mode 100644 pkg/aiproxy/models/proxy_node_local.go create mode 100644 pkg/aiproxy/models/virtual_key_guard.go create mode 100644 pkg/aiproxy/options/doc.go create mode 100644 pkg/aiproxy/options/options.go create mode 100644 pkg/aiproxy/policy/defaults.go create mode 100644 pkg/aiproxy/policy/doc.go create mode 100644 pkg/aiproxy/policy/resources.go create mode 100644 pkg/aiproxy/providerapi/doc.go create mode 100644 pkg/aiproxy/providerapi/stream.go create mode 100644 pkg/aiproxy/providerapi/types.go create mode 100644 pkg/aiproxy/providers/aliyun/aliyun.go create mode 100644 pkg/aiproxy/providers/aliyun/doc.go create mode 100644 pkg/aiproxy/providers/anthropic/anthropic.go create mode 100644 pkg/aiproxy/providers/anthropic/doc.go create mode 100644 pkg/aiproxy/providers/azure/azure.go create mode 100644 pkg/aiproxy/providers/azure/doc.go create mode 100644 pkg/aiproxy/providers/baidu/baidu.go create mode 100644 pkg/aiproxy/providers/baidu/baidu_test.go create mode 100644 pkg/aiproxy/providers/baidu/doc.go create mode 100644 pkg/aiproxy/providers/baidu/token.go create mode 100644 pkg/aiproxy/providers/baidu/wenxin_v1.go create mode 100644 pkg/aiproxy/providers/bridge.go create mode 100644 pkg/aiproxy/providers/cohere/cohere.go create mode 100644 pkg/aiproxy/providers/cohere/doc.go create mode 100644 pkg/aiproxy/providers/completions.go create mode 100644 pkg/aiproxy/providers/doc.go create mode 100644 pkg/aiproxy/providers/embeddings.go create mode 100644 pkg/aiproxy/providers/embeddings_test.go create mode 100644 pkg/aiproxy/providers/gemini/doc.go create mode 100644 pkg/aiproxy/providers/gemini/gemini.go create mode 100644 pkg/aiproxy/providers/images.go create mode 100644 pkg/aiproxy/providers/images_test.go create mode 100644 pkg/aiproxy/providers/openai/compat.go create mode 100644 pkg/aiproxy/providers/openai/completions.go create mode 100644 pkg/aiproxy/providers/openai/doc.go create mode 100644 pkg/aiproxy/providers/openai/embeddings.go create mode 100644 pkg/aiproxy/providers/openai/images.go create mode 100644 pkg/aiproxy/providers/openai/schema.go create mode 100644 pkg/aiproxy/providers/openai/tools.go create mode 100644 pkg/aiproxy/providers/openai/tools_test.go create mode 100644 pkg/aiproxy/providers/providers_test.go create mode 100644 pkg/aiproxy/providers/registry.go create mode 100644 pkg/aiproxy/providers/stream.go create mode 100644 pkg/aiproxy/providers/types.go create mode 100644 pkg/aiproxy/providers/vllm/doc.go create mode 100644 pkg/aiproxy/providers/vllm/vllm.go create mode 100644 pkg/aiproxy/providers/vllm/vllm_test.go create mode 100644 pkg/aiproxy/service/doc.go create mode 100644 pkg/aiproxy/service/service.go create mode 100644 pkg/aiproxy/service/slave_register.go create mode 100644 pkg/aiproxy/upstream/doc.go create mode 100644 pkg/aiproxy/upstream/openai_compat.go create mode 100644 pkg/apis/aiproxy/ai_key.go create mode 100644 pkg/apis/aiproxy/ai_model.go create mode 100644 pkg/apis/aiproxy/ai_provider.go create mode 100644 pkg/apis/aiproxy/ai_proxy_node.go create mode 100644 pkg/apis/aiproxy/ai_routing.go create mode 100644 pkg/apis/aiproxy/ai_routing_model.go create mode 100644 pkg/apis/aiproxy/ai_virtual_key.go create mode 100644 pkg/apis/aiproxy/consts.go create mode 100644 pkg/apis/aiproxy/doc.go create mode 100644 pkg/apis/aiproxy/serialize_register.go create mode 100644 pkg/mcclient/modules/aiproxy/doc.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_keys.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_models.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_providers.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_proxy_nodes.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_routing_models.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_routings.go create mode 100644 pkg/mcclient/modules/aiproxy/mod_ai_virtual_keys.go create mode 100644 pkg/mcclient/options/aiproxy/doc.go create mode 100644 pkg/mcclient/options/aiproxy/resources.go create mode 100755 scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh create mode 100755 scripts/test/aiproxy/aiproxy-functional-test-common.sh create mode 100755 scripts/test/aiproxy/aiproxy-functional-test-mimo.sh create mode 100755 scripts/test/aiproxy/aiproxy-functional-test-qwen.sh create mode 100755 scripts/test/aiproxy/aiproxy-functional-test.sh diff --git a/build/aiproxy/vars b/build/aiproxy/vars new file mode 100644 index 0000000000..7cc889f1f5 --- /dev/null +++ b/build/aiproxy/vars @@ -0,0 +1 @@ +DESCRIPTION="Yunion Cloud AI Proxy Service" diff --git a/build/docker/Dockerfile.aiproxy b/build/docker/Dockerfile.aiproxy new file mode 100644 index 0000000000..6c0c99bf6d --- /dev/null +++ b/build/docker/Dockerfile.aiproxy @@ -0,0 +1,3 @@ +FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v3.22.2-0 + +ADD ./_output/alpine-build/bin/aiproxy /opt/yunion/bin/aiproxy diff --git a/cmd/aiproxy/main.go b/cmd/aiproxy/main.go new file mode 100644 index 0000000000..ba4a2432cb --- /dev/null +++ b/cmd/aiproxy/main.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/service" + "yunion.io/x/onecloud/pkg/util/atexit" +) + +func main() { + defer atexit.Handle() + + service.StartService() +} diff --git a/cmd/climc/main.go b/cmd/climc/main.go index 3933024778..0f0b3d3400 100644 --- a/cmd/climc/main.go +++ b/cmd/climc/main.go @@ -17,6 +17,7 @@ package main import ( "yunion.io/x/onecloud/cmd/climc/entry" _ "yunion.io/x/onecloud/cmd/climc/shell" + _ "yunion.io/x/onecloud/cmd/climc/shell/aiproxy" _ "yunion.io/x/onecloud/cmd/climc/shell/ansible" _ "yunion.io/x/onecloud/cmd/climc/shell/apimap" _ "yunion.io/x/onecloud/cmd/climc/shell/cloudevent" diff --git a/cmd/climc/shell/aiproxy/ai_key.go b/cmd/climc/shell/aiproxy/ai_key.go new file mode 100644 index 0000000000..0754a93c5a --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_key.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiKeys) + cmd.Create(new(apoptions.AiKeyCreateOptions)) + cmd.List(new(apoptions.AiKeyListOptions)) + cmd.Show(new(apoptions.AiKeyShowOptions)) + cmd.Update(new(apoptions.AiKeyUpdateOptions)) + cmd.Delete(new(apoptions.AiKeyDeleteOptions)) + registerEnableDisable(cmd) +} diff --git a/cmd/climc/shell/aiproxy/ai_model.go b/cmd/climc/shell/aiproxy/ai_model.go new file mode 100644 index 0000000000..812cb05944 --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_model.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiModels) + cmd.Create(new(apoptions.AiModelCreateOptions)) + cmd.List(new(apoptions.AiModelListOptions)) + cmd.Show(new(apoptions.AiModelShowOptions)) + cmd.Update(new(apoptions.AiModelUpdateOptions)) + cmd.Delete(new(apoptions.AiModelDeleteOptions)) + registerEnableDisable(cmd) +} diff --git a/cmd/climc/shell/aiproxy/ai_provider.go b/cmd/climc/shell/aiproxy/ai_provider.go new file mode 100644 index 0000000000..b18e17fccb --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_provider.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiProviders) + cmd.Create(new(apoptions.AiProviderCreateOptions)) + cmd.List(new(apoptions.AiProviderListOptions)) + cmd.Show(new(apoptions.AiProviderShowOptions)) + cmd.Update(new(apoptions.AiProviderUpdateOptions)) + cmd.Delete(new(apoptions.AiProviderDeleteOptions)) + registerEnableDisable(cmd) +} diff --git a/cmd/climc/shell/aiproxy/ai_proxy_node.go b/cmd/climc/shell/aiproxy/ai_proxy_node.go new file mode 100644 index 0000000000..88aca18c5d --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_proxy_node.go @@ -0,0 +1,32 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiProxyNodes) + cmd.Create(new(apoptions.AiProxyNodeCreateOptions)) + cmd.List(new(apoptions.AiProxyNodeListOptions)) + cmd.Show(new(apoptions.AiProxyNodeShowOptions)) + cmd.Update(new(apoptions.AiProxyNodeUpdateOptions)) + cmd.Delete(new(apoptions.AiProxyNodeDeleteOptions)) + registerEnableDisable(cmd) + cmd.PerformClass("register", new(apoptions.AiProxyNodeRegisterOptions)) +} diff --git a/cmd/climc/shell/aiproxy/ai_routing.go b/cmd/climc/shell/aiproxy/ai_routing.go new file mode 100644 index 0000000000..b69fc3ff97 --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_routing.go @@ -0,0 +1,32 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiRoutings) + cmd.Create(new(apoptions.AiRoutingCreateOptions)) + cmd.List(new(apoptions.AiRoutingListOptions)) + cmd.Show(new(apoptions.AiRoutingShowOptions)) + cmd.Update(new(apoptions.AiRoutingUpdateOptions)) + cmd.Delete(new(apoptions.AiRoutingDeleteOptions)) + cmd.Perform("set-models", new(apoptions.AiRoutingSetModelsOptions)) + registerEnableDisable(cmd) +} diff --git a/cmd/climc/shell/aiproxy/ai_routing_model.go b/cmd/climc/shell/aiproxy/ai_routing_model.go new file mode 100644 index 0000000000..bf0f336f57 --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_routing_model.go @@ -0,0 +1,30 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiRoutingModels) + cmd.Create(new(apoptions.AiRoutingModelCreateOptions)) + cmd.List(new(apoptions.AiRoutingModelListOptions)) + cmd.Show(new(apoptions.AiRoutingModelShowOptions)) + cmd.Update(new(apoptions.AiRoutingModelUpdateOptions)) + cmd.Delete(new(apoptions.AiRoutingModelDeleteOptions)) +} diff --git a/cmd/climc/shell/aiproxy/ai_virtual_key.go b/cmd/climc/shell/aiproxy/ai_virtual_key.go new file mode 100644 index 0000000000..f09e72037f --- /dev/null +++ b/cmd/climc/shell/aiproxy/ai_virtual_key.go @@ -0,0 +1,31 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" + apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" +) + +func init() { + cmd := shell.NewResourceCmd(&apmodules.AiVirtualKeys) + cmd.Create(new(apoptions.AiVirtualKeyCreateOptions)) + cmd.List(new(apoptions.AiVirtualKeyListOptions)) + cmd.Show(new(apoptions.AiVirtualKeyShowOptions)) + cmd.Update(new(apoptions.AiVirtualKeyUpdateOptions)) + cmd.Delete(new(apoptions.AiVirtualKeyDeleteOptions)) + registerEnableDisable(cmd) +} diff --git a/cmd/climc/shell/aiproxy/doc.go b/cmd/climc/shell/aiproxy/doc.go new file mode 100644 index 0000000000..e915aa93ca --- /dev/null +++ b/cmd/climc/shell/aiproxy/doc.go @@ -0,0 +1,16 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package aiproxy registers climc commands for the aiproxy service. +package aiproxy // import "yunion.io/x/onecloud/cmd/climc/shell/aiproxy" diff --git a/cmd/climc/shell/aiproxy/enabled.go b/cmd/climc/shell/aiproxy/enabled.go new file mode 100644 index 0000000000..5f74ef0259 --- /dev/null +++ b/cmd/climc/shell/aiproxy/enabled.go @@ -0,0 +1,25 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" + baseoptions "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func registerEnableDisable(cmd *shell.ResourceCmd) { + cmd.Perform("enable", new(baseoptions.BaseIdOptions)) + cmd.Perform("disable", new(baseoptions.BaseIdOptions)) +} diff --git a/docs/aiproxy/functional-test-climc-mimo.md b/docs/aiproxy/functional-test-climc-mimo.md new file mode 100644 index 0000000000..1a2a356260 --- /dev/null +++ b/docs/aiproxy/functional-test-climc-mimo.md @@ -0,0 +1,114 @@ +# aiproxy 功能测试（climc + 小米 MiMo） + +本文用 **climc** 配置 aiproxy 资源，并通过 **curl** 调用 `POST /v1/chat/completions` 验证 **xiaomi** catalog（`api.xiaomimimo.com`）。 + +> **安全**：请勿将 MiMo API Key 写入脚本或提交到 Git。使用环境变量 `MIMO_API_KEY`。若 Key 曾泄露，请到小米开放平台轮换。 + +通义千问（DashScope）测试见 [functional-test-climc.md](./functional-test-climc.md)。 + +## 前置条件 + +| 项 | 说明 | +|----|------| +| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` public endpoint | +| 数据库 | 主节点已执行 `InitDB`，catalog 含 `xiaomi` 及 `mimo-*` 模型 | +| 客户端 | 已 `source /etc/yunion/rcadmin`，`climc` 可用 | +| 工具 | `jq` | +| 网络 | aiproxy 节点能访问 `https://api.xiaomimimo.com` | + +```bash +source /etc/yunion/rcadmin +export CLIMC_OUTPUT_FORMAT=json +bash scripts/test/aiproxy/aiproxy-functional-test.sh +# 交互菜单中选择 xiaomi，输入 MiMo API Key +``` + +或快捷入口（默认选中 `xiaomi`）： + +```bash +export MIMO_API_KEY='你的 MiMo API Key' +bash scripts/test/aiproxy/aiproxy-functional-test-mimo.sh +``` + +预置模型：`export AIPROXY_FT_PROVIDER=xiaomi AIPROXY_FT_MODEL=mimo-v2.5-pro` + +按 provider 自动命名的资源（可覆盖 `AIPROXY_FT_KEY_NAME` 等）： + +| 变量 | 默认（xiaomi） | +|------|----------------| +| `AIPROXY_FT_KEY_NAME` | `aiproxy-ft-xiaomi` | +| `AIPROXY_FT_VK_NAME` | `aiproxy-ft-xiaomi-vk` | +| `AIPROXY_FT_ROUTING_NAME` | `aiproxy-ft-xiaomi-routing` | +| `AIPROXY_FT_MODEL` | 交互默认 `mimo-v2-flash` | + +## 手动步骤摘要 + +### Catalog + +```bash +climc ai-provider-show xiaomi +climc ai-model-show xiaomi-mimo-v2-flash +``` + +`config.base_url` 应为 `https://api.xiaomimimo.com`。 + +### ai_key + +```bash +climc ai-key-create mimo-ft \ + --ai-provider-id xiaomi \ + --secret "${MIMO_API_KEY}" \ + --weight 10 \ + --enabled +``` + +### 路由与 chat + +```bash +climc ai-virtual-key-create aiproxy-mimo-ft-vk +climc ai-routing-create aiproxy-mimo-ft-routing \ + --priority 10 \ + --models '[{"ai_provider_id":"xiaomi","ai_model_id":"mimo-v2-flash","priority":1}]' + +AIPROXY_URL="$(climc endpoint-list --service aiproxy --interface public --limit 1 \ + --output-format json | jq -r '.data[0].url')" +VK="$(climc ai-virtual-key-show aiproxy-mimo-ft-vk --output-format json | jq -r '.virtual_key')" + +curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \ + -H "Authorization: Bearer ${VK}" \ + -H "Content-Type: application/json" \ + -d '{"model":"mimo-v2-flash","messages":[{"role":"user","content":"hi"}],"max_tokens":64}' | jq . +``` + +### 流式 + +`scripts/test/aiproxy/aiproxy-functional-test-mimo.sh` 在非流式通过后默认执行 step 7（`stream: true`）。跳过：`export AIPROXY_FT_SKIP_STREAM=1`。 + +```bash +curl -k -sS -N -o /tmp/aiproxy-mimo-stream.sse \ + "${AIPROXY_URL%/}/v1/chat/completions" \ + -H "Authorization: Bearer ${VK}" \ + -H "Content-Type: application/json" \ + -d '{"model":"mimo-v2-flash","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' +``` + +catalog 中其它模型：`mimo-v2.5-pro`、`mimo-v2-pro`、`mimo-v2.5`、`mimo-v2-omni`（id 形如 `xiaomi-mimo-v2.5-pro`）。 + +## 常见问题 + +**上游 401** +检查 `MIMO_API_KEY` 是否有效；确认 `ai_key` 已 `--enabled` 且 `ai_provider_id=xiaomi`。 + +**`no ai_routing matched`** +virtual key 与 routing 须在同一 climc 项目下创建。 + +**与 DashScope 脚本冲突** +MiMo 脚本使用独立的 vk/routing/key 名称；勿与 `aiproxy-ft-vk` 混用同一 routing 的 model 列表。 + +## 清理 + +```bash +climc ai-routing-delete aiproxy-mimo-ft-routing +climc ai-virtual-key-delete aiproxy-mimo-ft-vk +climc ai-key-delete mimo-ft +``` diff --git a/docs/aiproxy/functional-test-climc.md b/docs/aiproxy/functional-test-climc.md new file mode 100644 index 0000000000..82c864e12c --- /dev/null +++ b/docs/aiproxy/functional-test-climc.md @@ -0,0 +1,303 @@ +# aiproxy 功能测试（climc + 通义千问 DashScope） + +本文用 **climc** 配置 aiproxy 资源，并用 **curl** 调用 OpenAI 兼容接口 `POST /v1/chat/completions` 做端到端验证。 + +> **安全**：请勿把 DashScope API Key 写进脚本、文档或提交到 Git。在 shell 里用环境变量 `DASHSCOPE_API_KEY` 传入。若 Key 曾在聊天/工单中泄露，请到阿里云控制台轮换。 + +## 前置条件 + +| 项 | 说明 | +|----|------| +| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` 服务及 public endpoint | +| 数据库 | 主节点已执行 `InitDB`，catalog 中已有 `aliyun` provider 及 `qwen-*` 模型（首次启动 master 会自动 seed） | +| 客户端 | 已 `source /etc/yunion/rcadmin`（或等价 rc 文件），`climc` 能正常 list | +| 工具 | `jq`（脚本与下文 curl 示例用于解析 JSON） | +| 网络 | aiproxy 节点能访问 `https://dashscope.aliyuncs.com` | + +### 一键脚本（交互式，推荐） + +从 catalog 选择 **模型提供商** 与 **model_key**，终端输入 API Key（或使用环境变量跳过输入），自动完成非流式 + 流式 chat： + +```bash +source /etc/yunion/rcadmin +export CLIMC_OUTPUT_FORMAT=json +bash scripts/test/aiproxy/aiproxy-functional-test.sh +``` + +也可预置后减少交互（仍会选择模型、是否流式，除非全部用环境变量）： + +```bash +export DASHSCOPE_API_KEY='你的 DashScope API Key' # 或 AIPROXY_FT_API_KEY +export AIPROXY_FT_PROVIDER=aliyun +export AIPROXY_FT_MODEL=qwen-turbo +bash scripts/test/aiproxy/aiproxy-functional-test.sh +``` + +通义快捷入口：`bash scripts/test/aiproxy/aiproxy-functional-test-qwen.sh`（默认 `aliyun`）。 +小米 MiMo 见 [functional-test-climc-mimo.md](./functional-test-climc-mimo.md)（`aiproxy-functional-test-mimo.sh`）。 + +非交互（CI）： + +```bash +export AIPROXY_FT_NONINTERACTIVE=1 +export AIPROXY_FT_PROVIDER=aliyun +export AIPROXY_FT_MODEL=qwen-turbo +export AIPROXY_FT_API_KEY='...' +export AIPROXY_FT_SKIP_STREAM=1 # 可选，跳过流式 +bash scripts/test/aiproxy/aiproxy-functional-test.sh +``` + +| 环境变量 | 说明 | +|----------|------| +| `AIPROXY_FT_PROVIDER` | `provider_key`（如 `aliyun`、`xiaomi`） | +| `AIPROXY_FT_MODEL` | `model_key`（如 `qwen-turbo`） | +| `AIPROXY_FT_API_KEY` | 上游 API Key（通用） | +| `DASHSCOPE_API_KEY` / `MIMO_API_KEY` | 按提供商兼容的旧变量名 | +| `AIPROXY_FT_SKIP_STREAM` | `1` 跳过流式；`0` 强制流式 | +| `AIPROXY_URL` | 留空则从 endpoint-list 解析 | + +## 测试流程概览 + +```mermaid +flowchart LR + VK[ai_virtual_key] --> RT[ai_routing] + RT --> RM[ai_routing_model] + RM --> P[ai_provider aliyun] + RM --> M[ai_model qwen-turbo] + P --> K[ai_key secret] + K --> DS[DashScope API] +``` + +## 0a. ai_provider 创建测试脚本 + +自定义 provider（非 catalog seed）创建与校验： + +```bash +source /etc/yunion/rcadmin +bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh +``` + +交互输入：资源名、`provider_key`、`base_url`、是否 `--enabled`。非交互示例： + +```bash +export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1 +export AIPROXY_PROVIDER_FT_NAME=my-vllm +export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-vllm +export AIPROXY_PROVIDER_FT_BASE_URL=http://127.0.0.1:8000/v1 +bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh +``` + +`provider_key` 须全局唯一；与 InitDB catalog（如 `aliyun`）重复会失败。完整 config 可用 `AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"..."}'`。 + +## 0. ai_proxy_node（多副本 / 路由绑定） + +列出 aiproxy 实例节点（InitDB 后默认有 `primary`）： + +```bash +climc ai-proxy-node-list +climc ai-proxy-node-show primary +``` + +注册 standby 节点（与进程内 `register` 心跳相同，一般由 standby 自动调用；手工测试可用）： + +```bash +climc ai-proxy-node-register --address https://standby-host:30938 --hb-timeout 120 +``` + +手工创建/更新节点（需具备写权限策略）： + +```bash +climc ai-proxy-node-create standby-1 \ + --address https://standby-host:30938 \ + --domain aiproxy-standby.example.com \ + --hb-timeout 120 \ + --enabled + +climc ai-proxy-node-update primary --address https://primary-host:30938 --domain aiproxy.example.com +climc ai-proxy-node-enable primary +climc ai-proxy-node-disable +``` + +将 `ai_routing` 绑定到指定节点（chat 须走该节点 public endpoint）： + +```bash +climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary +``` + +## 1. 检查 Keystone endpoint + +```bash +climc endpoint-list --service aiproxy --interface public +``` + +应能看到当前 region 的 public URL（脚本会取第一条用于 curl）。 + +## 2. 检查 catalog（InitDB seed） + +```bash +climc ai-provider-list --provider-key aliyun +climc ai-provider-show aliyun +climc ai-model-list --ai-provider-id aliyun --model-key qwen-turbo +``` + +确认 `provider_key=aliyun`，`config.base_url` 含 `https://dashscope.aliyuncs.com/compatible-mode`，且存在模型 `model_key=qwen-turbo`（catalog 固定 id / name：`aliyun-qwen-turbo`）。 + +## 3. 注册上游 API Key（ai_key） + +将 DashScope Key 存为 `ai_key`，供 chat 时按 provider 加权选取： + +```bash +climc ai-key-create qwen-dashscope-ft \ + --ai-provider-id aliyun \ + --secret "${DASHSCOPE_API_KEY}" \ + --weight 10 \ + --enabled +``` + +校验： + +```bash +climc ai-key-list --ai-provider-id aliyun +climc ai-key-show qwen-dashscope-ft +``` + +确认 `ai_provider_id` 为 `aliyun`，且 **`enabled=true`**（`ai_key` 默认 disabled，创建时需 `--enabled`；若已存在但被禁用，执行 `climc ai-key-enable qwen-dashscope-ft`）。若曾用错误参数创建过同名 key，可更新： + +```bash +climc ai-key-update qwen-dashscope-ft \ + --ai-provider-id aliyun \ + --secret "${DASHSCOPE_API_KEY}" \ + --weight 10 +climc ai-key-enable qwen-dashscope-ft +``` + +（`secret` 在 API 中通常不回显，仅用于上游调用。） + +## 4. 创建 Virtual Key（客户端鉴权） + +```bash +climc ai-virtual-key-create aiproxy-ft-vk +``` + +记下返回的 `virtual_key`（形如 `sk-...`）。查看： + +```bash +climc ai-virtual-key-list +climc ai-virtual-key-show aiproxy-ft-vk +``` + +Virtual key 归属当前 climc 用户的 **项目**；后续 `ai_routing` 须在同一项目（或共享到该项目）下。 + +## 5. 创建项目路由（ai_routing + models） + +将项目内请求 `model=qwen-turbo` 指到 catalog 的 aliyun/qwen-turbo。 + +`models` 里 `ai_model_id` 使用 catalog 固定 id（与 name 相同，如 `aliyun-qwen-turbo`），或在指定 `ai_provider_id` 时也可填 **model_key**（如 `qwen-turbo`）： + +```bash +climc ai-routing-create aiproxy-ft-routing \ + --priority 10 \ + --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-turbo","priority":1}]' +``` + +或手工指定 name： + +```bash +climc ai-routing-create aiproxy-ft-routing \ + --priority 10 \ + --models '[{"ai_provider_id":"aliyun","ai_model_id":"aliyun-qwen-turbo","priority":1}]' +``` + +查看绑定模型： + +```bash +climc ai-routing-show aiproxy-ft-routing +``` + +也可事后调整： + +```bash +climc ai-routing-set-models aiproxy-ft-routing \ + --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-plus","priority":1}]' +``` + +可选：将规则绑定到指定 aiproxy 实例（多副本时）： + +```bash +# 仅当需要固定到 primary 等节点时 +climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary +``` + +## 6. Chat completions（curl） + +climc 暂无 chat 子命令，用 public endpoint + virtual key 调用（`-k` 跳过 TLS 证书校验，适用于自签或内网 HTTPS）： + +```bash +AIPROXY_URL="${AIPROXY_URL:-$(climc endpoint-list --service aiproxy --interface public --limit 1 \ + --output-format json | jq -r '.data[0].url // empty')}" + +VK="$(climc ai-virtual-key-show aiproxy-ft-vk --output-format json \ + | jq -r '.virtual_key')" + +curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \ + -H "Authorization: Bearer ${VK}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen-turbo", + "messages": [{"role": "user", "content": "用一句话介绍通义千问"}], + "max_tokens": 128 + }' | jq . +``` + +**期望**：HTTP 200，JSON 含 `choices[0].message.content` 及 `usage`。 + +## 6b. 流式 Chat（curl / 脚本 step 7） + +一键脚本在步骤 6 非流式成功后，默认继续执行流式校验（聚合 `choices[0].delta.content`）。跳过流式： + +```bash +export AIPROXY_FT_SKIP_STREAM=1 +``` + +手动 curl（SSE，`data: [DONE]` 结束）： + +```bash +curl -k -sS -N -o /tmp/aiproxy-ft-stream.sse \ + "${AIPROXY_URL%/}/v1/chat/completions" \ + -H "Authorization: Bearer ${VK}" \ + -H "Content-Type: application/json" \ + -d '{"model":"qwen-turbo","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' +``` + +期望：HTTP 200，响应体含 `data: {...}` 行且至少一条 `delta.content` 非空；最后为 `data: [DONE]`。 + +## 7. 负向用例（可选） + +| 场景 | 操作 | 期望 | +|------|------|------| +| 错误 virtual key | `Authorization: Bearer sk-invalid` | 4xx，virtual key 无效 | +| 无路由 | `climc ai-routing-disable aiproxy-ft-routing` 或删除后再 chat | 404，无匹配 routing | +| 禁用 virtual key | `climc ai-virtual-key-disable aiproxy-ft-vk` | 4xx | +| provider 限制 | create vk 时 `--limits '{"allowed_ai_provider_ids":["openai"]}'` | 4xx，provider 不允许 | + +## 8. 清理（可选） + +```bash +climc ai-routing-delete aiproxy-ft-routing +climc ai-virtual-key-delete aiproxy-ft-vk +climc ai-key-delete qwen-dashscope-ft +``` + +## 常见问题 + +**`no ai_routing matched for virtual key project`** +Virtual key 的 `project_id` 与 routing 所在项目不一致，或 routing 未 `enabled`、未共享到该项目。用同一 `climc` 项目上下文创建两者。 + +**`no api_key for ai_provider`** +未创建 `ai_key`，且 `ai_provider.config` 里也没有 `api_key`。按步骤 3 创建 `ai_key`。 + +**DashScope 401/403** +检查 `DASHSCOPE_API_KEY` 是否有效、是否开通对应模型。 + +**多副本 `ai_routing` 绑定其它节点** +若 routing 指定了 `ai_proxy_node_id`，须访问该节点的 public endpoint，或去掉绑定。 diff --git a/pkg/aiproxy/handlers/chat_completions.go b/pkg/aiproxy/handlers/chat_completions.go new file mode 100644 index 0000000000..80725c0778 --- /dev/null +++ b/pkg/aiproxy/handlers/chat_completions.go @@ -0,0 +1,368 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +const headerAiVirtualKey = "X-Ai-Virtual-Key" + +func extractVirtualKey(r *http.Request) string { + if v := strings.TrimSpace(r.Header.Get(headerAiVirtualKey)); v != "" { + return v + } + authz := strings.TrimSpace(r.Header.Get("Authorization")) + parts := strings.SplitN(authz, " ", 2) + if len(parts) == 2 && strings.EqualFold(parts[0], "Bearer") { + return strings.TrimSpace(parts[1]) + } + return "" +} + +func upstreamErrorStatusCode(uerr *upstream.Error) int { + if uerr == nil || uerr.StatusCode <= 0 { + return 0 + } + return uerr.StatusCode +} + +func writeUpstreamError(w http.ResponseWriter, uerr *upstream.Error) { + status := http.StatusBadGateway + if uerr != nil && uerr.StatusCode > 0 { + status = uerr.StatusCode + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if uerr != nil && len(uerr.Body) > 0 { + _, _ = w.Write(uerr.Body) + return + } + msg := "upstream request failed" + if uerr != nil { + msg = uerr.Error() + } + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "error": map[string]interface{}{ + "message": msg, + }, + }) +} + +func flushIf(w http.ResponseWriter) { + if f, ok := w.(http.Flusher); ok { + f.Flush() + } +} + +// chatCompletionsHandler implements OpenAI-compatible POST /openai/v1/chat/completions. +// Auth is the ai_virtual_key only (Authorization: Bearer or X-Ai-Virtual-Key). +// Upstream is resolved: ai_virtual_key -> project ai_routing -> ai_routing_model -> ai_key (by catalog model_key). +func chatCompletionsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + httperrors.InvalidInputError(ctx, w, "only POST is supported") + return + } + + defer r.Body.Close() + raw, err := io.ReadAll(r.Body) + if err != nil { + httperrors.InvalidInputError(ctx, w, "read body: %v", err) + return + } + + body, err := jsonutils.Parse(raw) + if err != nil { + httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err) + return + } + dict, ok := body.(*jsonutils.JSONDict) + if !ok { + httperrors.InvalidInputError(ctx, w, "body must be a JSON object") + return + } + + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + var vkLim *api.SAiVirtualKeyLimits + if up.MaxTokensPerRequest > 0 { + vkLim = &api.SAiVirtualKeyLimits{ + MaxTokensPerRequest: up.MaxTokensPerRequest, + } + } + if err := models.EnforceVirtualKeyMaxTokens(dict, vkLim); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + isStream, _ := dict.Bool("stream") + prov := providers.Get(up.ProviderKey) + if _, err := prov.BuildUpstreamRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict, isStream); err != nil { + httperrors.InvalidInputError(ctx, w, "provider request: %v", err) + return + } + timeout := 120 * time.Second + if isStream { + timeout = 2 * time.Hour + } + + if !isStream { + resp, uerr := chatCompletionWithKeyFailover(ctx, up, dict, isStream, timeout) + if uerr != nil { + writeUpstreamError(w, uerr) + return + } + body := resp.Body + if norm, nerr := prov.NormalizeResponse(body); nerr == nil && len(norm) > 0 { + body = norm + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + return + } + + ch, uerr := chatCompletionStreamWithKeyFailover(ctx, up, dict, isStream, prov, timeout) + if uerr != nil { + writeUpstreamError(w, uerr) + return + } + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flushIf(w) + + streamOK := true + for chunk := range ch { + if chunk.Done { + break + } + if len(chunk.Data) == 0 { + continue + } + if isUpstreamErrorChunk(chunk.Data) { + streamOK = false + models.RecordAiKeyFailure(up.AiKeyId, parseUpstreamErrorStatus(chunk.Data)) + _, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data) + flushIf(w) + break + } + _, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data) + flushIf(w) + } + _, _ = fmt.Fprintf(w, "data: [DONE]\n\n") + flushIf(w) + if streamOK { + models.RecordAiKeySuccess(up.AiKeyId) + } +} + +func isUpstreamErrorChunk(data []byte) bool { + var wrap struct { + Error interface{} `json:"error"` + } + return json.Unmarshal(data, &wrap) == nil && wrap.Error != nil +} + +func parseUpstreamErrorStatus(data []byte) int { + var wrap struct { + Error struct { + Code interface{} `json:"code"` + } `json:"error"` + } + if json.Unmarshal(data, &wrap) != nil { + return 0 + } + switch c := wrap.Error.Code.(type) { + case float64: + return int(c) + case int: + return c + default: + return 0 + } +} + +func chatCompletionWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + dict *jsonutils.JSONDict, + stream bool, + timeout time.Duration, +) (*upstream.Response, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := buildProviderUpstream(up, dict, stream) + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + resp, uerr := upstream.ChatCompletion(reqCtx, upReq) + cancel() + if uerr == nil { + models.RecordAiKeySuccess(up.AiKeyId) + return resp, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func chatCompletionStreamWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + dict *jsonutils.JSONDict, + stream bool, + prov providers.Provider, + timeout time.Duration, +) (<-chan upstream.StreamChunk, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := buildProviderUpstream(up, dict, stream) + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + ch, uerr := providerStreamChunks(reqCtx, up, upReq, prov) + cancel() + if uerr == nil { + return ch, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func buildProviderUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict, isStream bool) (*upstream.Request, error) { + prov := providers.Get(up.ProviderKey) + httpReq, err := prov.BuildUpstreamRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict, isStream) + if err != nil { + return nil, err + } + return providers.ToUpstreamRequest(httpReq, up.APIKey), nil +} + +func providerStreamChunks( + ctx context.Context, + up *models.ChatUpstream, + upReq *upstream.Request, + prov providers.Provider, +) (<-chan upstream.StreamChunk, *upstream.Error) { + chatCtx := &providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + } + if providers.OpenAIStreamPassthrough(prov, chatCtx) { + return upstream.ChatCompletionStream(ctx, upReq) + } + rawCh, uerr := upstream.ChatCompletionStreamRaw(ctx, upReq) + if uerr != nil { + return nil, uerr + } + out := make(chan upstream.StreamChunk, 16) + go func() { + defer close(out) + state := &providers.StreamState{Model: up.UpstreamModel} + for evt := range rawCh { + chunks, err := prov.ConvertStreamEvent(evt.Event, evt.Data, state) + if err != nil { + msg, _ := json.Marshal(map[string]interface{}{ + "error": map[string]interface{}{"message": err.Error()}, + }) + out <- upstream.StreamChunk{Data: msg} + return + } + for _, c := range chunks { + if len(c.Data) > 0 { + out <- upstream.StreamChunk{Data: c.Data} + } + if c.Done { + out <- upstream.StreamChunk{Done: true} + return + } + } + } + }() + return out, nil +} diff --git a/pkg/aiproxy/handlers/completions.go b/pkg/aiproxy/handlers/completions.go new file mode 100644 index 0000000000..a1087b20f2 --- /dev/null +++ b/pkg/aiproxy/handlers/completions.go @@ -0,0 +1,258 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +// completionsHandler implements OpenAI-compatible POST /openai/v1/completions. +func completionsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + httperrors.InvalidInputError(ctx, w, "only POST is supported") + return + } + + defer r.Body.Close() + raw, err := io.ReadAll(r.Body) + if err != nil { + httperrors.InvalidInputError(ctx, w, "read body: %v", err) + return + } + + body, err := jsonutils.Parse(raw) + if err != nil { + httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err) + return + } + dict, ok := body.(*jsonutils.JSONDict) + if !ok { + httperrors.InvalidInputError(ctx, w, "body must be a JSON object") + return + } + + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + var vkLim *api.SAiVirtualKeyLimits + if up.MaxTokensPerRequest > 0 { + vkLim = &api.SAiVirtualKeyLimits{ + MaxTokensPerRequest: up.MaxTokensPerRequest, + } + } + if err := models.EnforceVirtualKeyMaxTokens(dict, vkLim); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + compProv, err := providers.GetCompletions(up.ProviderKey) + if err != nil { + httperrors.InvalidInputError(ctx, w, "%v", err) + return + } + + isStream, _ := dict.Bool("stream") + if _, err := compProv.BuildCompletionsRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict, isStream); err != nil { + httperrors.InvalidInputError(ctx, w, "provider request: %v", err) + return + } + + timeout := 120 * time.Second + if isStream { + timeout = 2 * time.Hour + } + + if !isStream { + resp, uerr := completionsWithKeyFailover(ctx, up, dict, isStream, timeout) + if uerr != nil { + writeUpstreamError(w, uerr) + return + } + out := resp.Body + if norm, nerr := compProv.NormalizeCompletionsResponse(out); nerr == nil && len(norm) > 0 { + out = norm + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(out) + return + } + + ch, uerr := completionsStreamWithKeyFailover(ctx, up, dict, isStream, compProv, timeout) + if uerr != nil { + writeUpstreamError(w, uerr) + return + } + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flushIf(w) + + streamOK := true + for chunk := range ch { + if chunk.Done { + break + } + if len(chunk.Data) == 0 { + continue + } + if isUpstreamErrorChunk(chunk.Data) { + streamOK = false + models.RecordAiKeyFailure(up.AiKeyId, parseUpstreamErrorStatus(chunk.Data)) + _, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data) + flushIf(w) + break + } + _, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data) + flushIf(w) + } + _, _ = fmt.Fprintf(w, "data: [DONE]\n\n") + flushIf(w) + if streamOK { + models.RecordAiKeySuccess(up.AiKeyId) + } +} + +func completionsWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + dict *jsonutils.JSONDict, + stream bool, + timeout time.Duration, +) (*upstream.Response, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := buildCompletionsUpstream(up, dict, stream) + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + resp, uerr := upstream.ChatCompletion(reqCtx, upReq) + cancel() + if uerr == nil { + models.RecordAiKeySuccess(up.AiKeyId) + return resp, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func completionsStreamWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + dict *jsonutils.JSONDict, + stream bool, + compProv providers.CompletionsProvider, + timeout time.Duration, +) (<-chan upstream.StreamChunk, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := buildCompletionsUpstream(up, dict, stream) + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + var ch <-chan upstream.StreamChunk + var uerr *upstream.Error + if compProv.OpenAICompletionsStreamPassthrough() { + ch, uerr = upstream.ChatCompletionStream(reqCtx, upReq) + } else { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: "streaming completions not supported for provider"} + } + cancel() + if uerr == nil { + return ch, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func buildCompletionsUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict, isStream bool) (*upstream.Request, error) { + compProv, err := providers.GetCompletions(up.ProviderKey) + if err != nil { + return nil, err + } + httpReq, err := compProv.BuildCompletionsRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict, isStream) + if err != nil { + return nil, err + } + return providers.ToUpstreamRequest(httpReq, up.APIKey), nil +} diff --git a/pkg/aiproxy/handlers/doc.go b/pkg/aiproxy/handlers/doc.go new file mode 100644 index 0000000000..fa66ecf7f9 --- /dev/null +++ b/pkg/aiproxy/handlers/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers // import "yunion.io/x/onecloud/pkg/aiproxy/handlers" diff --git a/pkg/aiproxy/handlers/embeddings.go b/pkg/aiproxy/handlers/embeddings.go new file mode 100644 index 0000000000..9ab470949f --- /dev/null +++ b/pkg/aiproxy/handlers/embeddings.go @@ -0,0 +1,146 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "io" + "net/http" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +// embeddingsHandler implements OpenAI-compatible POST /openai/v1/embeddings. +func embeddingsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + httperrors.InvalidInputError(ctx, w, "only POST is supported") + return + } + + defer r.Body.Close() + raw, err := io.ReadAll(r.Body) + if err != nil { + httperrors.InvalidInputError(ctx, w, "read body: %v", err) + return + } + + body, err := jsonutils.Parse(raw) + if err != nil { + httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err) + return + } + dict, ok := body.(*jsonutils.JSONDict) + if !ok { + httperrors.InvalidInputError(ctx, w, "body must be a JSON object") + return + } + + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + embProv := providers.GetEmbeddings(up.ProviderKey) + if _, err := embProv.BuildEmbeddingsRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict); err != nil { + httperrors.InvalidInputError(ctx, w, "provider request: %v", err) + return + } + + resp, uerr := embeddingsWithKeyFailover(ctx, up, dict, 60*time.Second) + if uerr != nil { + writeUpstreamError(w, uerr) + return + } + out := resp.Body + if norm, nerr := embProv.NormalizeEmbeddingsResponse(out); nerr == nil && len(norm) > 0 { + out = norm + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(out) +} + +func embeddingsWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + dict *jsonutils.JSONDict, + timeout time.Duration, +) (*upstream.Response, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := buildEmbeddingsUpstream(up, dict) + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + resp, uerr := upstream.ChatCompletion(reqCtx, upReq) + cancel() + if uerr == nil { + models.RecordAiKeySuccess(up.AiKeyId) + return resp, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func buildEmbeddingsUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict) (*upstream.Request, error) { + embProv := providers.GetEmbeddings(up.ProviderKey) + httpReq, err := embProv.BuildEmbeddingsRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict) + if err != nil { + return nil, err + } + return providers.ToUpstreamRequest(httpReq, up.APIKey), nil +} diff --git a/pkg/aiproxy/handlers/handlers.go b/pkg/aiproxy/handlers/handlers.go new file mode 100644 index 0000000000..6826309d58 --- /dev/null +++ b/pkg/aiproxy/handlers/handlers.go @@ -0,0 +1,75 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/options" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/appsrv/dispatcher" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" +) + +const openaiCompatAPIPrefix = "/ai/openai/v1" + +func InitHandlers(app *appsrv.Application, isSlave bool) { + db.InitAllManagers() + db.RegistUserCredCacheUpdater() + + app_common.ExportOptionsHandler(app, &options.Options) + + taskman.AddTaskHandler("", app, isSlave) + + db.AddScopeResourceCountHandler("", app) + + app.AddHandler2("POST", openaiCompatAPIPrefix+"/chat/completions", chatCompletionsHandler, nil, "aiproxy_openai_v1_chat_completions", nil) + app.AddHandler2("POST", openaiCompatAPIPrefix+"/completions", completionsHandler, nil, "aiproxy_openai_v1_completions", nil) + app.AddHandler2("POST", openaiCompatAPIPrefix+"/embeddings", embeddingsHandler, nil, "aiproxy_openai_v1_embeddings", nil) + app.AddHandler2("POST", openaiCompatAPIPrefix+"/images/generations", imagesGenerationsHandler, nil, "aiproxy_openai_v1_images_generations", nil) + app.AddHandler2("GET", openaiCompatAPIPrefix+"/models", modelsHandler, nil, "aiproxy_openai_v1_models", nil) + app.AddHandler2("GET", openaiCompatAPIPrefix+"/models/", modelRetrieveHandler, nil, "aiproxy_openai_v1_models_retrieve", nil) + + for _, manager := range []db.IModelManager{ + taskman.TaskManager, + taskman.SubTaskManager, + taskman.TaskObjectManager, + taskman.ArchivedTaskManager, + + db.SharedResourceManager, + db.UserCacheManager, + db.TenantCacheManager, + } { + db.RegisterModelManager(manager) + } + + for _, manager := range []db.IModelManager{ + db.OpsLog, + db.Metadata, + + models.AiProviderManager, + models.AiModelManager, + models.AiKeyManager, + models.AiVirtualKeyManager, + models.AiRoutingManager, + models.AiRoutingModelManager, + models.AiProxyNodeManager, + } { + db.RegisterModelManager(manager) + handler := db.NewModelHandler(manager) + dispatcher.AddModelDispatcher("", app, handler, isSlave) + } +} diff --git a/pkg/aiproxy/handlers/images_generations.go b/pkg/aiproxy/handlers/images_generations.go new file mode 100644 index 0000000000..f6371bd541 --- /dev/null +++ b/pkg/aiproxy/handlers/images_generations.go @@ -0,0 +1,146 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "io" + "net/http" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +// imagesGenerationsHandler implements OpenAI-compatible POST /openai/v1/images/generations. +func imagesGenerationsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + httperrors.InvalidInputError(ctx, w, "only POST is supported") + return + } + + defer r.Body.Close() + raw, err := io.ReadAll(r.Body) + if err != nil { + httperrors.InvalidInputError(ctx, w, "read body: %v", err) + return + } + + body, err := jsonutils.Parse(raw) + if err != nil { + httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err) + return + } + dict, ok := body.(*jsonutils.JSONDict) + if !ok { + httperrors.InvalidInputError(ctx, w, "body must be a JSON object") + return + } + + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + imgProv := providers.GetImages(up.ProviderKey) + if _, err := imgProv.BuildImagesGenerationsRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict); err != nil { + httperrors.InvalidInputError(ctx, w, "provider request: %v", err) + return + } + + resp, uerr := imagesGenerationsWithKeyFailover(ctx, up, dict, 180*time.Second) + if uerr != nil { + writeUpstreamError(w, uerr) + return + } + out := resp.Body + if norm, nerr := imgProv.NormalizeImagesGenerationsResponse(out); nerr == nil && len(norm) > 0 { + out = norm + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(out) +} + +func imagesGenerationsWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + dict *jsonutils.JSONDict, + timeout time.Duration, +) (*upstream.Response, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := buildImagesGenerationsUpstream(up, dict) + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + resp, uerr := upstream.ChatCompletion(reqCtx, upReq) + cancel() + if uerr == nil { + models.RecordAiKeySuccess(up.AiKeyId) + return resp, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func buildImagesGenerationsUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict) (*upstream.Request, error) { + imgProv := providers.GetImages(up.ProviderKey) + httpReq, err := imgProv.BuildImagesGenerationsRequest(&providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + }, dict) + if err != nil { + return nil, err + } + return providers.ToUpstreamRequest(httpReq, up.APIKey), nil +} diff --git a/pkg/aiproxy/handlers/models.go b/pkg/aiproxy/handlers/models.go new file mode 100644 index 0000000000..72e7bf509a --- /dev/null +++ b/pkg/aiproxy/handlers/models.go @@ -0,0 +1,84 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "strings" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +// modelsHandler implements OpenAI-compatible GET /openai/v1/models. +func modelsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + httperrors.InvalidInputError(ctx, w, "only GET is supported") + return + } + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + items, err := models.ListModelsForVirtualKey(ctx, userCred, vk) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + if items == nil { + items = []models.ModelsListEntry{} + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "object": "list", + "data": items, + }) +} + +// modelRetrieveHandler implements OpenAI-compatible GET /openai/v1/models/{model}. +func modelRetrieveHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + httperrors.InvalidInputError(ctx, w, "only GET is supported") + return + } + params := appsrv.AppContextGetParams(ctx) + modelID := "" + if params != nil { + modelID = strings.TrimSpace(params.Params[""]) + } + if modelID == "" { + httperrors.InvalidInputError(ctx, w, "missing model id") + return + } + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + items, err := models.ListModelsForVirtualKey(ctx, userCred, vk) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + for _, item := range items { + if item.ID == modelID { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(item) + return + } + } + httperrors.NotFoundError(ctx, w, "model %q not found", modelID) +} diff --git a/pkg/aiproxy/models/ai_key_health.go b/pkg/aiproxy/models/ai_key_health.go new file mode 100644 index 0000000000..ffd8728623 --- /dev/null +++ b/pkg/aiproxy/models/ai_key_health.go @@ -0,0 +1,135 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "sync" + "time" +) + +const ( + aiKeyHealthMaxScore = 100 + aiKeyHealthFailPenalty = 25 + aiKeyHealthSuccessBoost = 10 + aiKeyHealthCooldownAfter = 3 + aiKeyHealthCooldownPeriod = 60 * time.Second +) + +type aiKeyHealthState struct { + score int + consecutiveFails int + cooldownUntil time.Time +} + +var ( + aiKeyHealthMu sync.RWMutex + aiKeyHealth = map[string]*aiKeyHealthState{} +) + +func getAiKeyHealth(keyId string) *aiKeyHealthState { + if keyId == "" { + return nil + } + aiKeyHealthMu.RLock() + st := aiKeyHealth[keyId] + aiKeyHealthMu.RUnlock() + if st != nil { + return st + } + aiKeyHealthMu.Lock() + defer aiKeyHealthMu.Unlock() + if st = aiKeyHealth[keyId]; st == nil { + st = &aiKeyHealthState{score: aiKeyHealthMaxScore} + aiKeyHealth[keyId] = st + } + return st +} + +// dynamicAiKeyWeightMultiplier returns 0-100 applied to configured ai_key.weight (100 = full weight). +func dynamicAiKeyWeightMultiplier(keyId string) int { + if keyId == "" { + return aiKeyHealthMaxScore + } + st := getAiKeyHealth(keyId) + now := time.Now() + aiKeyHealthMu.Lock() + defer aiKeyHealthMu.Unlock() + if !st.cooldownUntil.IsZero() && now.Before(st.cooldownUntil) { + return 0 + } + if !st.cooldownUntil.IsZero() && !now.Before(st.cooldownUntil) { + st.cooldownUntil = time.Time{} + if st.score < aiKeyHealthMaxScore/2 { + st.score = aiKeyHealthMaxScore / 2 + } + } + if st.score <= 0 { + return 0 + } + if st.score > aiKeyHealthMaxScore { + return aiKeyHealthMaxScore + } + return st.score +} + +// RecordAiKeySuccess boosts dynamic weight after a successful upstream call. +func RecordAiKeySuccess(keyId string) { + if keyId == "" { + return + } + st := getAiKeyHealth(keyId) + aiKeyHealthMu.Lock() + defer aiKeyHealthMu.Unlock() + st.consecutiveFails = 0 + st.cooldownUntil = time.Time{} + st.score += aiKeyHealthSuccessBoost + if st.score > aiKeyHealthMaxScore { + st.score = aiKeyHealthMaxScore + } +} + +// RecordAiKeyFailure reduces dynamic weight when upstream rejects a key (429/401/5xx etc.). +func RecordAiKeyFailure(keyId string, statusCode int) { + if keyId == "" || !IsRetryableAiKeyUpstreamStatus(statusCode) { + return + } + st := getAiKeyHealth(keyId) + aiKeyHealthMu.Lock() + defer aiKeyHealthMu.Unlock() + st.consecutiveFails++ + st.score -= aiKeyHealthFailPenalty + if st.score < 0 { + st.score = 0 + } + if st.consecutiveFails >= aiKeyHealthCooldownAfter { + st.cooldownUntil = time.Now().Add(aiKeyHealthCooldownPeriod) + st.score = 0 + } +} + +// IsRetryableAiKeyUpstreamStatus reports HTTP statuses that imply the api_key may be bad or overloaded. +func IsRetryableAiKeyUpstreamStatus(statusCode int) bool { + if statusCode <= 0 { + return true + } + switch { + case statusCode == 401, statusCode == 403, statusCode == 429: + return true + case statusCode >= 500 && statusCode <= 599: + return true + default: + return false + } +} diff --git a/pkg/aiproxy/models/ai_key_resolve.go b/pkg/aiproxy/models/ai_key_resolve.go new file mode 100644 index 0000000000..04a690a255 --- /dev/null +++ b/pkg/aiproxy/models/ai_key_resolve.go @@ -0,0 +1,205 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "crypto/rand" + "math/big" + "strings" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" +) + +func effectiveAiKeyRoutingWeight(r *api.SAiKeyRouting) int { + if r == nil || r.Weight <= 0 { + return 0 + } + return r.Weight +} + +// baseAiKeyWeight returns configured weight (column, else routing.weight, else 1). +func baseAiKeyWeight(k *SAiKey) int { + if k == nil { + return 1 + } + if k.Weight > 0 { + return k.Weight + } + if w := effectiveAiKeyRoutingWeight(k.Routing); w > 0 { + return w + } + return 1 +} + +// effectiveAiKeyWeight returns load-balance weight including dynamic penalty (差 key 降权). +func effectiveAiKeyWeight(k *SAiKey) int { + base := baseAiKeyWeight(k) + if k == nil || base <= 0 { + return 0 + } + mul := dynamicAiKeyWeightMultiplier(k.Id) + if mul <= 0 { + return 0 + } + return base * mul / aiKeyHealthMaxScore +} + +func aiKeyRoutingAcceptsModel(r *api.SAiKeyRouting, reqModel string) bool { + rm := strings.TrimSpace(reqModel) + if r == nil { + return true + } + for _, block := range r.BlockedModelKeys { + if modelPatternMatches(block, rm) { + return false + } + } + if len(r.AllowedModelKeys) > 0 { + ok := false + for _, allow := range r.AllowedModelKeys { + if modelPatternMatches(allow, rm) { + ok = true + break + } + } + if !ok { + return false + } + } + return true +} + +func pickWeightedAiKey(candidates []*SAiKey) *SAiKey { + if len(candidates) == 0 { + return nil + } + if len(candidates) == 1 { + return candidates[0] + } + total := 0 + for _, k := range candidates { + total += effectiveAiKeyWeight(k) + } + if total <= 0 { + return candidates[0] + } + n, err := rand.Int(rand.Reader, big.NewInt(int64(total))) + if err != nil { + return candidates[0] + } + threshold := int(n.Int64()) + 1 + acc := 0 + for _, k := range candidates { + acc += effectiveAiKeyWeight(k) + if acc >= threshold { + return k + } + } + return candidates[len(candidates)-1] +} + +type resolvedUpstreamAPIKey struct { + Secret string + AiKeyId string + FromRows bool +} + +// MaxAiKeyFailoverAttempts is how many alternate ai_key rows to try per chat request. +const MaxAiKeyFailoverAttempts = 8 + +// resolveUpstreamAPIKey picks an ai_key (weighted + dynamic penalty) or provider.config api_key. +func resolveUpstreamAPIKey(prov *SAiProvider, modelKey string) (*resolvedUpstreamAPIKey, error) { + return resolveUpstreamAPIKeyExcluding(prov, modelKey, nil) +} + +func resolveUpstreamAPIKeyExcluding(prov *SAiProvider, modelKey string, exclude map[string]bool) (*resolvedUpstreamAPIKey, error) { + if prov == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider is nil") + } + pid := strings.TrimSpace(prov.Id) + if pid == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider id is empty") + } + + keys := make([]SAiKey, 0, 32) + q := AiKeyManager.Query().Equals("ai_provider_id", pid).Equals("enabled", true) + err := q.All(&keys) + if err != nil { + return nil, errors.Wrap(err, "list ai_key for provider") + } + + candidates := make([]*SAiKey, 0, len(keys)) + hasSecretKey := false + for i := range keys { + k := &keys[i] + if strings.TrimSpace(k.Secret) == "" { + continue + } + hasSecretKey = true + if exclude != nil && exclude[k.Id] { + continue + } + if effectiveAiKeyWeight(k) <= 0 { + continue + } + if aiKeyRoutingAcceptsModel(k.Routing, modelKey) { + candidates = append(candidates, k) + } + } + if len(candidates) > 0 { + chosen := pickWeightedAiKey(candidates) + if chosen == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "failed to pick ai_key") + } + return &resolvedUpstreamAPIKey{ + Secret: strings.TrimSpace(chosen.Secret), + AiKeyId: chosen.Id, + FromRows: true, + }, nil + } + if hasSecretKey { + return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "no available ai_key for catalog model %q (check weight, cooldown, allowed_model_keys)", modelKey) + } + if prov.Config == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty") + } + apiKey := strings.TrimSpace(prov.Config.ResolvedAPIKey()) + if apiKey == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "set api_key on ai_provider or add an enabled ai_key with secret for this provider") + } + return &resolvedUpstreamAPIKey{Secret: apiKey}, nil +} + +// RepickUpstreamAPIKey selects another ai_key for the same provider/model, excluding already tried ids. +func RepickUpstreamAPIKey(up *ChatUpstream, tried map[string]bool) error { + if up == nil || strings.TrimSpace(up.AiProviderId) == "" { + return errors.Wrap(httperrors.ErrInvalidStatus, "missing ai_provider on upstream") + } + provObj, err := AiProviderManager.FetchById(up.AiProviderId) + if err != nil { + return errors.Wrap(err, "fetch ai_provider for key repick") + } + prov := provObj.(*SAiProvider) + resolved, err := resolveUpstreamAPIKeyExcluding(prov, up.UpstreamModel, tried) + if err != nil { + return err + } + up.APIKey = resolved.Secret + up.AiKeyId = resolved.AiKeyId + return nil +} diff --git a/pkg/aiproxy/models/ai_keys.go b/pkg/aiproxy/models/ai_keys.go new file mode 100644 index 0000000000..74bc79f32c --- /dev/null +++ b/pkg/aiproxy/models/ai_keys.go @@ -0,0 +1,165 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// SAiKey stores a named upstream API key (or other secret material) for reuse by routing or providers. +type SAiKey struct { + db.SEnabledStatusStandaloneResourceBase + + // AiProviderId optionally associates this key with a catalog provider row. + AiProviderId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"` + // Secret holds raw key material; only privileged scopes should list it. + Secret string `width:"4096" charset:"ascii" nullable:"false" create:"required"` + // Weight is used for weighted random load balancing among matching keys (default 1). + Weight int `default:"1" nullable:"false" list:"user" create:"optional" update:"user"` + // Routing limits which request "model" values may use this key. + Routing *api.SAiKeyRouting `length:"medium" charset:"utf8" list:"user" create:"optional" update:"user"` +} + +type SAiKeyManager struct { + db.SEnabledStatusStandaloneResourceBaseManager +} + +var AiKeyManager *SAiKeyManager + +func init() { + AiKeyManager = &SAiKeyManager{ + SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager( + SAiKey{}, + "ai_keys_tbl", + "ai_key", + "ai_keys", + ), + } + AiKeyManager.SetVirtualObject(AiKeyManager) +} + +func (manager *SAiKeyManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiKeyListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter") + } + if id := strings.TrimSpace(query.AiProviderId); id != "" { + q = q.Equals("ai_provider_id", id) + } + return q, nil +} + +func (manager *SAiKeyManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiKeyDetails { + rows := make([]api.AiKeyDetails, len(objs)) + baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + providerIds := make([]string, len(objs)) + for i := range objs { + rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i] + k := objs[i].(*SAiKey) + providerIds[i] = k.AiProviderId + } + providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds) + if err != nil { + log.Errorf("FetchIdNameMap2 ai_provider: %v", err) + return rows + } + for i := range rows { + rows[i].AiProviderName, _ = providerNames[providerIds[i]] + } + return rows +} + +func (manager *SAiKeyManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiKeyCreateInput, +) (api.AiKeyCreateInput, error) { + var err error + input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData") + } + if input.Weight < 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "weight must be >= 0") + } + if input.Weight == 0 { + input.Weight = 1 + } + if strings.TrimSpace(input.Secret) == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "secret is required") + } + if strings.TrimSpace(input.AiProviderId) == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "ai_provider_id is required") + } + prov, err := fetchEnabledAiProvider(ctx, userCred, input.AiProviderId) + if err != nil { + return input, err + } + input.AiProviderId = prov.Id + if input.Enabled == nil && input.Disabled == nil { + input.SetEnabled() + } + return input, nil +} + +func (k *SAiKey) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.AiKeyUpdateInput, +) (*api.AiKeyUpdateInput, error) { + var err error + input.EnabledStatusStandaloneResourceBaseUpdateInput, err = k.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData") + } + if input.Weight < 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "weight must be >= 0") + } + if pid := strings.TrimSpace(input.AiProviderId); pid != "" { + prov, err := fetchEnabledAiProvider(ctx, userCred, pid) + if err != nil { + return input, err + } + input.AiProviderId = prov.Id + } + return input, nil +} diff --git a/pkg/aiproxy/models/ai_models.go b/pkg/aiproxy/models/ai_models.go new file mode 100644 index 0000000000..68fe2e4b23 --- /dev/null +++ b/pkg/aiproxy/models/ai_models.go @@ -0,0 +1,179 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// SAiModel stores a model catalog row associated with an SAiProvider. +type SAiModel struct { + db.SEnabledStatusStandaloneResourceBase + + AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` + // ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo). + ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"` +} + +type SAiModelManager struct { + db.SEnabledStatusStandaloneResourceBaseManager +} + +var AiModelManager *SAiModelManager + +func init() { + AiModelManager = &SAiModelManager{ + SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager( + SAiModel{}, + "ai_models_tbl", + "ai_model", + "ai_models", + ), + } + AiModelManager.SetVirtualObject(AiModelManager) +} + +func (manager *SAiModelManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiModelListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter") + } + if id := strings.TrimSpace(query.AiProviderId); id != "" { + q = q.Equals("ai_provider_id", id) + } + if key := strings.TrimSpace(query.ModelKey); key != "" { + q = q.Equals("model_key", key) + } + return q, nil +} + +func (manager *SAiModelManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiModelDetails { + rows := make([]api.AiModelDetails, len(objs)) + baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + providerIds := make([]string, len(objs)) + for i := range objs { + rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i] + m := objs[i].(*SAiModel) + providerIds[i] = m.AiProviderId + } + providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds) + if err != nil { + log.Errorf("FetchIdNameMap2 ai_provider: %v", err) + return rows + } + for i := range rows { + rows[i].AiProviderName, _ = providerNames[providerIds[i]] + } + return rows +} + +func (manager *SAiModelManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiModelCreateInput, +) (api.AiModelCreateInput, error) { + var err error + input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData") + } + + prov, err := fetchEnabledAiProvider(ctx, userCred, input.AiProviderId) + if err != nil { + return input, err + } + input.AiProviderId = prov.Id + + mk, err := validateAiModelKey(input.ModelKey) + if err != nil { + return input, err + } + input.ModelKey = mk + + if err := ensureAiModelKeyUniquePerProvider(ctx, prov.Id, mk, ""); err != nil { + return input, err + } + + if strings.TrimSpace(input.Name) == "" { + input.Name = defaultAiModelName(prov.Name, mk) + } + + return input, nil +} + +func (m *SAiModel) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.AiModelUpdateInput, +) (*api.AiModelUpdateInput, error) { + var err error + input.EnabledStatusStandaloneResourceBaseUpdateInput, err = m.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData") + } + + providerId := m.AiProviderId + if pid := strings.TrimSpace(input.AiProviderId); pid != "" { + prov, err := fetchEnabledAiProvider(ctx, userCred, pid) + if err != nil { + return input, err + } + providerId = prov.Id + input.AiProviderId = prov.Id + } + + modelKey := m.ModelKey + if mk := strings.TrimSpace(input.ModelKey); mk != "" { + modelKey, err = validateAiModelKey(mk) + if err != nil { + return input, err + } + input.ModelKey = modelKey + } + + if modelKey != m.ModelKey || providerId != m.AiProviderId { + if err := ensureAiModelKeyUniquePerProvider(ctx, providerId, modelKey, m.Id); err != nil { + return input, err + } + } + + return input, nil +} diff --git a/pkg/aiproxy/models/ai_providers.go b/pkg/aiproxy/models/ai_providers.go new file mode 100644 index 0000000000..e09a0492cd --- /dev/null +++ b/pkg/aiproxy/models/ai_providers.go @@ -0,0 +1,155 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// SAiProvider stores an LLM provider catalog entry (routing key and OpenAI-compatible config). +type SAiProvider struct { + db.SEnabledStatusStandaloneResourceBase + + // ProviderKey selects the upstream adapter implementation (e.g. openai, vllm, aliyun). + // Multiple ai_provider rows may share the same provider_key with different config. + ProviderKey string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` + // Config is a JSON snapshot of provider connectivity (base_url, optional api_key). + Config *api.SAiProviderConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"` +} + +type SAiProviderManager struct { + db.SEnabledStatusStandaloneResourceBaseManager +} + +var AiProviderManager *SAiProviderManager + +func init() { + AiProviderManager = &SAiProviderManager{ + SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager( + SAiProvider{}, + "ai_providers_tbl", + "ai_provider", + "ai_providers", + ), + } + AiProviderManager.SetVirtualObject(AiProviderManager) +} + +func (manager *SAiProviderManager) InitializeData() error { + return SeedStandardCatalog(context.Background()) +} + +func (manager *SAiProviderManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiProviderListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter") + } + if key := strings.TrimSpace(query.ProviderKey); key != "" { + q = q.Equals("provider_key", key) + } + return q, nil +} + +func (manager *SAiProviderManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiProviderDetails { + rows := make([]api.AiProviderDetails, len(objs)) + baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + for i := range objs { + rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i] + } + return rows +} + +func (manager *SAiProviderManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiProviderCreateInput, +) (api.AiProviderCreateInput, error) { + var err error + input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData") + } + + pk, err := validateAiCatalogIdentifier("provider_key", input.ProviderKey, maxAiProviderKeyLen) + if err != nil { + return input, err + } + input.ProviderKey = pk + + input.Config = normalizeAiProviderConfig(input.Config) + if err := validateAiProviderConfig(input.Config); err != nil { + return input, err + } + + if strings.TrimSpace(input.Name) == "" { + input.Name = pk + } + + return input, nil +} + +func (p *SAiProvider) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.AiProviderUpdateInput, +) (*api.AiProviderUpdateInput, error) { + var err error + input.EnabledStatusStandaloneResourceBaseUpdateInput, err = p.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData") + } + + if pk := strings.TrimSpace(input.ProviderKey); pk != "" { + pk, err = validateAiCatalogIdentifier("provider_key", pk, maxAiProviderKeyLen) + if err != nil { + return input, err + } + input.ProviderKey = pk + } + + if input.Config != nil { + input.Config = normalizeAiProviderConfig(input.Config) + if err := validateAiProviderConfig(input.Config); err != nil { + return input, err + } + } + + return input, nil +} diff --git a/pkg/aiproxy/models/ai_proxy_nodes.go b/pkg/aiproxy/models/ai_proxy_nodes.go new file mode 100644 index 0000000000..ea6ca94ea9 --- /dev/null +++ b/pkg/aiproxy/models/ai_proxy_nodes.go @@ -0,0 +1,354 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "database/sql" + "fmt" + "net" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/rbacscope" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/aiproxy/options" + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +const ( + defaultAiProxyNodeHbTimeout = 120 + defaultPrimaryAiProxyNodeId = "primary" + maxAiProxyNodeDomainLen = 256 +) + +// SAiProxyNode records an aiproxy instance reachable address and optional domain name. +type SAiProxyNode struct { + db.SEnabledStatusStandaloneResourceBase + + Address string `width:"256" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user" index:"true"` + Domain string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` + LastSeen time.Time `nullable:"true" list:"user"` + HbTimeout int `nullable:"false" default:"120" list:"user" create:"optional" update:"user"` +} + +type SAiProxyNodeManager struct { + db.SEnabledStatusStandaloneResourceBaseManager +} + +var AiProxyNodeManager *SAiProxyNodeManager + +func init() { + AiProxyNodeManager = &SAiProxyNodeManager{ + SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager( + SAiProxyNode{}, + "ai_proxy_nodes_tbl", + "ai_proxy_node", + "ai_proxy_nodes", + ), + } + AiProxyNodeManager.SetVirtualObject(AiProxyNodeManager) +} + +func (manager *SAiProxyNodeManager) ResourceScope() rbacscope.TRbacScope { + return rbacscope.ScopeUser +} + +func (manager *SAiProxyNodeManager) InitializeData() error { + ctx := context.Background() + addr, err := AdvertiseAddressFromOptions(nil) + if err != nil { + return err + } + node := SAiProxyNode{} + node.SetModelManager(manager, &node) + node.Id = defaultPrimaryAiProxyNodeId + node.Name = defaultPrimaryAiProxyNodeId + node.Description = "Default primary aiproxy node" + node.Address = addr + domain := "" + if existing, err := manager.FetchById(defaultPrimaryAiProxyNodeId); err == nil { + domain = strings.TrimSpace(existing.(*SAiProxyNode).Domain) + } + if domain == "" { + d, err := DomainFromApiServer(nil) + if err != nil { + return err + } + domain = d + } + node.Domain = domain + node.HbTimeout = defaultAiProxyNodeHbTimeout + node.LastSeen = time.Now() + node.SetEnabled(true) + node.Status = apis.STATUS_AVAILABLE + node.Progress = 100 + if err := manager.TableSpec().InsertOrUpdate(ctx, &node); err != nil { + return errors.Wrap(err, "insert or update default primary ai_proxy_node") + } + return nil +} + +func aiProxyNodeId(address string) string { + return stringutils2.GenId("aiproxy.node", address) +} + +func normalizeAiProxyNodeAddress(raw string) (string, error) { + address := strings.TrimSpace(raw) + if address == "" { + return "", errors.Wrap(httperrors.ErrInputParameter, "address is required") + } + if strings.Contains(address, "://") { + u, err := url.Parse(address) + if err != nil { + return "", errors.Wrapf(httperrors.ErrInputParameter, "invalid address URL: %v", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", errors.Wrap(httperrors.ErrInputParameter, "address scheme must be http or https") + } + if strings.TrimSpace(u.Host) == "" { + return "", errors.Wrap(httperrors.ErrInputParameter, "address must include host") + } + return strings.TrimRight(address, "/"), nil + } + if _, _, err := net.SplitHostPort(address); err != nil { + return "", errors.Wrapf(httperrors.ErrInputParameter, "invalid address %q: %v", address, err) + } + return fmt.Sprintf("http://%s", address), nil +} + +func normalizeAiProxyNodeDomain(domain string) (string, error) { + domain = strings.TrimSpace(domain) + if domain == "" { + return "", nil + } + if len(domain) > maxAiProxyNodeDomainLen { + return "", errors.Wrapf(httperrors.ErrInputParameter, "domain too long (max %d)", maxAiProxyNodeDomainLen) + } + if strings.Contains(domain, "://") || strings.ContainsAny(domain, "/:") { + return "", errors.Wrap(httperrors.ErrInputParameter, "domain must be a hostname without scheme or port") + } + return domain, nil +} + +func aiProxyNodeDisplayName(address string) string { + u, err := url.Parse(address) + if err != nil || strings.TrimSpace(u.Host) == "" { + return address + } + return u.Host +} + +// AdvertiseAddressFromOptions returns the service URL advertised by this instance. +func AdvertiseAddressFromOptions(opts *options.SAiProxyOptions) (string, error) { + if opts == nil { + opts = &options.Options + } + if addr := strings.TrimRight(strings.TrimSpace(opts.AdvertiseAddress), "/"); addr != "" { + return normalizeAiProxyNodeAddress(addr) + } + scheme := "http" + if opts.EnableSsl { + scheme = "https" + } + host := strings.TrimSpace(opts.Address) + if host == "" || host == "0.0.0.0" { + host = "127.0.0.1" + } + return normalizeAiProxyNodeAddress(fmt.Sprintf("%s://%s:%d", scheme, host, opts.Port)) +} + +// DomainFromApiServer derives ai_proxy_node.domain from --api-server (hostname only). +func DomainFromApiServer(opts *options.SAiProxyOptions) (string, error) { + if opts == nil { + opts = &options.Options + } + raw := strings.TrimSpace(opts.ApiServer) + if raw == "" { + return "", nil + } + host := raw + if strings.Contains(raw, "://") { + u, err := url.Parse(raw) + if err != nil { + return "", errors.Wrapf(err, "parse api_server %q", raw) + } + host = strings.TrimSpace(u.Hostname()) + } else if strings.ContainsAny(raw, "/:") { + if h, _, err := net.SplitHostPort(raw); err == nil { + host = h + } + } + if host == "" { + return "", nil + } + return normalizeAiProxyNodeDomain(host) +} + +func (manager *SAiProxyNodeManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiProxyNodeListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter") + } + if addr := strings.TrimSpace(query.Address); addr != "" { + q = q.Equals("address", addr) + } + if domain := strings.TrimSpace(query.Domain); domain != "" { + q = q.Equals("domain", domain) + } + return q, nil +} + +func (manager *SAiProxyNodeManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiProxyNodeDetails { + rows := make([]api.AiProxyNodeDetails, len(objs)) + baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + for i := range objs { + rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i] + node := objs[i].(*SAiProxyNode) + rows[i].IsActive = node.IsActive() + } + return rows +} + +func (manager *SAiProxyNodeManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiProxyNodeCreateInput, +) (api.AiProxyNodeCreateInput, error) { + var err error + input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData") + } + input.Address, err = normalizeAiProxyNodeAddress(input.Address) + if err != nil { + return input, err + } + input.Domain, err = normalizeAiProxyNodeDomain(input.Domain) + if err != nil { + return input, err + } + if input.HbTimeout <= 0 { + input.HbTimeout = defaultAiProxyNodeHbTimeout + } + if strings.TrimSpace(input.Name) == "" { + input.Name = aiProxyNodeDisplayName(input.Address) + } + return input, nil +} + +func (manager *SAiProxyNodeManager) PerformRegister( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.AiProxyNodeRegisterInput, +) (jsonutils.JSONObject, error) { + addr, err := normalizeAiProxyNodeAddress(input.Address) + if err != nil { + return nil, err + } + hbTimeout := input.HbTimeout + if hbTimeout <= 0 { + hbTimeout = defaultAiProxyNodeHbTimeout + } + nodeId := aiProxyNodeId(addr) + domain := "" + if existing, err := manager.FetchById(nodeId); err == nil { + domain = existing.(*SAiProxyNode).Domain + } else if errors.Cause(err) != sql.ErrNoRows { + return nil, errors.Wrap(err, "fetch ai_proxy_node") + } + node := SAiProxyNode{} + node.SetModelManager(manager, &node) + node.Id = nodeId + node.Name = aiProxyNodeDisplayName(addr) + node.Address = addr + node.Domain = domain + node.HbTimeout = hbTimeout + node.LastSeen = time.Now() + node.SetEnabled(true) + node.Status = apis.STATUS_AVAILABLE + node.Progress = 100 + if err := manager.TableSpec().InsertOrUpdate(ctx, &node); err != nil { + return nil, errors.Wrap(err, "insert or update ai_proxy_node") + } + return jsonutils.Marshal(api.AiProxyNodeRegisterOutput{Id: node.Id}), nil +} + +func (node *SAiProxyNode) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.AiProxyNodeUpdateInput, +) (*api.AiProxyNodeUpdateInput, error) { + var err error + input.EnabledStatusStandaloneResourceBaseUpdateInput, err = node.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData") + } + if input.Address != "" { + input.Address, err = normalizeAiProxyNodeAddress(input.Address) + if err != nil { + return input, err + } + } + if query.Contains("domain") { + input.Domain, err = normalizeAiProxyNodeDomain(input.Domain) + if err != nil { + return input, err + } + } + if input.HbTimeout < 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "hb_timeout must be >= 0") + } + return input, nil +} + +func (node *SAiProxyNode) IsActive() bool { + if node.Id == defaultPrimaryAiProxyNodeId { + return node.GetEnabled() + } + if node.LastSeen.IsZero() { + return false + } + timeout := node.HbTimeout + if timeout <= 0 { + timeout = defaultAiProxyNodeHbTimeout + } + return int(time.Since(node.LastSeen).Seconds()) < timeout +} diff --git a/pkg/aiproxy/models/ai_routing_models.go b/pkg/aiproxy/models/ai_routing_models.go new file mode 100644 index 0000000000..a796e446f1 --- /dev/null +++ b/pkg/aiproxy/models/ai_routing_models.go @@ -0,0 +1,337 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "database/sql" + stderrors "errors" + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// SAiRoutingModel binds a catalog model (and provider) to an ai_routing with per-entry priority. +type SAiRoutingModel struct { + db.SStandaloneResourceBase + + AiRoutingId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user" index:"true"` + AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` + AiModelId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` + // Priority orders models within the same ai_routing (lower value = higher priority). + Priority int `default:"100" nullable:"false" list:"user" create:"optional" update:"user"` + // ModelPattern optionally matches the client request "model" (same rules as ai_routing.model_pattern). + ModelPattern string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` + Enabled tristate.TriState `default:"true" nullable:"false" list:"user" create:"optional" update:"user"` +} + +type SAiRoutingModelManager struct { + db.SStandaloneResourceBaseManager +} + +var AiRoutingModelManager *SAiRoutingModelManager + +func init() { + AiRoutingModelManager = &SAiRoutingModelManager{ + SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager( + SAiRoutingModel{}, + "ai_routing_models_tbl", + "ai_routing_model", + "ai_routing_models", + ), + } + AiRoutingModelManager.SetVirtualObject(AiRoutingModelManager) +} + +func (manager *SAiRoutingModelManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiRoutingModelListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter") + } + if id := strings.TrimSpace(query.AiRoutingId); id != "" { + q = q.Equals("ai_routing_id", id) + } + if id := strings.TrimSpace(query.AiProviderId); id != "" { + q = q.Equals("ai_provider_id", id) + } + if id := strings.TrimSpace(query.AiModelId); id != "" { + q = q.Equals("ai_model_id", id) + } + if query.Enabled != nil { + q = q.Equals("enabled", *query.Enabled) + } + return q, nil +} + +func (manager *SAiRoutingModelManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiRoutingModelDetails { + rows := make([]api.AiRoutingModelDetails, len(objs)) + baseRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + for i := range objs { + rows[i].StandaloneResourceDetails = baseRows[i] + } + return rows +} + +func (manager *SAiRoutingModelManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiRoutingModelCreateInput, +) (api.AiRoutingModelCreateInput, error) { + var err error + input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SStandaloneResourceBaseManager.ValidateCreateData") + } + + routingId := strings.TrimSpace(input.AiRoutingId) + if routingId == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "ai_routing_id is required") + } + rObj, err := AiRoutingManager.FetchById(routingId) + if err != nil { + return input, errors.Wrap(err, "fetch ai_routing") + } + routing := rObj.(*SAiRouting) + + providerId, modelId, err := resolveAiRoutingModelRefs(ctx, userCred, strings.TrimSpace(input.AiProviderId), strings.TrimSpace(input.AiModelId)) + if err != nil { + return input, err + } + input.AiProviderId = providerId + input.AiModelId = modelId + input.AiRoutingId = routing.Id + + if strings.TrimSpace(input.Name) == "" { + input.Name = fmt.Sprintf("%s-%s-%d", routing.Name, modelId, input.Priority) + } + + if input.Enabled == nil { + enabled := true + input.Enabled = &enabled + } + return input, nil +} + +func fetchAiModelRef(ctx context.Context, userCred mcclient.TokenCredential, providerIdOrName, modelIdOrName string) (*SAiModel, error) { + modelIdOrName = strings.TrimSpace(modelIdOrName) + if modelIdOrName == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "ai_model_id is required") + } + mObj, err := AiModelManager.FetchByIdOrName(ctx, userCred, modelIdOrName) + if err == nil { + return mObj.(*SAiModel), nil + } + if !stderrors.Is(err, sql.ErrNoRows) { + return nil, errors.Wrap(err, "fetch ai_model") + } + if pk := strings.TrimSpace(providerIdOrName); pk != "" { + if mObj2, err2 := AiModelManager.FetchByIdOrName(ctx, userCred, catalogModelId(pk, modelIdOrName)); err2 == nil { + return mObj2.(*SAiModel), nil + } + } + providerIdOrName = strings.TrimSpace(providerIdOrName) + if providerIdOrName == "" { + return nil, errors.Wrap(err, "fetch ai_model") + } + prov, err := fetchEnabledAiProvider(ctx, userCred, providerIdOrName) + if err != nil { + return nil, errors.Wrap(err, "fetch ai_model by model_key") + } + mdl := SAiModel{} + q := AiModelManager.Query().Equals("ai_provider_id", prov.Id).Equals("model_key", modelIdOrName) + if err := q.First(&mdl); err != nil { + return nil, errors.Wrap(err, "fetch ai_model") + } + return &mdl, nil +} + +func resolveAiRoutingModelRefs(ctx context.Context, userCred mcclient.TokenCredential, providerIdOrName, modelIdOrName string) (string, string, error) { + mdl, err := fetchAiModelRef(ctx, userCred, providerIdOrName, modelIdOrName) + if err != nil { + return "", "", err + } + if !mdl.GetEnabled() { + return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_model disabled") + } + + providerId := strings.TrimSpace(providerIdOrName) + if providerId == "" { + providerId = mdl.AiProviderId + } + pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerId) + if err != nil { + return "", "", errors.Wrap(err, "fetch ai_provider") + } + prov := pObj.(*SAiProvider) + if !prov.GetEnabled() { + return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider disabled") + } + if mdl.AiProviderId != prov.Id { + return "", "", errors.Wrap(httperrors.ErrInputParameter, "ai_model does not belong to ai_provider") + } + return prov.Id, mdl.Id, nil +} + +func routingModelItemPriority(priority, weight int) int { + if priority != 0 { + return priority + } + if weight != 0 { + return weight + } + return 100 +} + +func validateAiRoutingModelItems( + ctx context.Context, + userCred mcclient.TokenCredential, + items []api.AiRoutingModelItem, +) ([]api.AiRoutingModelItem, error) { + if len(items) == 0 { + return nil, nil + } + out := make([]api.AiRoutingModelItem, len(items)) + for i := range items { + item := items[i] + providerId, modelId, err := resolveAiRoutingModelRefs(ctx, userCred, strings.TrimSpace(item.AiProviderId), strings.TrimSpace(item.AiModelId)) + if err != nil { + return nil, errors.Wrapf(err, "models[%d]", i) + } + item.AiProviderId = providerId + item.AiModelId = modelId + item.Priority = routingModelItemPriority(item.Priority, item.Weight) + item.Weight = 0 + if item.Enabled == nil { + enabled := true + item.Enabled = &enabled + } + out[i] = item + } + return out, nil +} + +func deleteAiRoutingModels(ctx context.Context, routingId string) error { + routingId = strings.TrimSpace(routingId) + if routingId == "" { + return errors.Wrap(httperrors.ErrInputParameter, "ai_routing_id is required") + } + _, err := sqlchemy.GetDB().Exec( + fmt.Sprintf("delete from %s where ai_routing_id = ?", AiRoutingModelManager.TableSpec().Name()), + routingId, + ) + if err != nil { + return errors.Wrap(err, "delete ai_routing_models") + } + return nil +} + +func createAiRoutingModels( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + routing *SAiRouting, + items []api.AiRoutingModelItem, +) error { + if routing == nil || len(items) == 0 { + return nil + } + for i := range items { + item := items[i] + enabled := true + if item.Enabled != nil { + enabled = *item.Enabled + } + dataDict := jsonutils.NewDict() + dataDict.Set("ai_routing_id", jsonutils.NewString(routing.Id)) + dataDict.Set("ai_provider_id", jsonutils.NewString(item.AiProviderId)) + dataDict.Set("ai_model_id", jsonutils.NewString(item.AiModelId)) + dataDict.Set("priority", jsonutils.NewInt(int64(item.Priority))) + if mp := strings.TrimSpace(item.ModelPattern); mp != "" { + dataDict.Set("model_pattern", jsonutils.NewString(mp)) + } + dataDict.Set("enabled", jsonutils.JSONTrue) + if !enabled { + dataDict.Set("enabled", jsonutils.JSONFalse) + } + dataDict.Set("name", jsonutils.NewString(fmt.Sprintf("%s-%s-%d", routing.Name, item.AiModelId, item.Priority))) + if _, err := db.DoCreate(AiRoutingModelManager, ctx, userCred, nil, dataDict, ownerId); err != nil { + return errors.Wrapf(err, "create ai_routing_model[%d]", i) + } + } + return nil +} + +func fetchAiRoutingModels(routingId string, enabledOnly bool) ([]SAiRoutingModel, error) { + entries := make([]SAiRoutingModel, 0, 8) + q := AiRoutingModelManager.Query().Equals("ai_routing_id", routingId) + if enabledOnly { + q = q.Equals("enabled", true) + } + err := q.Asc("priority").Asc("id").All(&entries) + if err != nil { + return nil, errors.Wrap(err, "list ai_routing_models") + } + return entries, nil +} + +func fetchEnabledAiRoutingModels(routingId string) ([]SAiRoutingModel, error) { + return fetchAiRoutingModels(routingId, true) +} + +// pickAiRoutingModel selects provider/model from ai_routing_models by request model name. +func pickAiRoutingModel(ctx context.Context, userCred mcclient.TokenCredential, routing *SAiRouting, reqModel string) (providerId, modelId string, err error) { + if routing == nil { + return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing") + } + entries, err := fetchEnabledAiRoutingModels(routing.Id) + if err != nil { + return "", "", err + } + if len(entries) == 0 { + return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_routing has no ai_routing_models") + } + for i := range entries { + e := &entries[i] + if !modelPatternMatches(e.ModelPattern, reqModel) { + continue + } + return e.AiProviderId, e.AiModelId, nil + } + return "", "", errors.Wrapf(httperrors.ErrNotFound, "no ai_routing_model matched request model %q", reqModel) +} diff --git a/pkg/aiproxy/models/ai_routings.go b/pkg/aiproxy/models/ai_routings.go new file mode 100644 index 0000000000..7a8a54d0e2 --- /dev/null +++ b/pkg/aiproxy/models/ai_routings.go @@ -0,0 +1,236 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +// SAiRouting stores a project-scoped (and optionally shared) routing rule. +type SAiRouting struct { + db.SSharableVirtualResourceBase + db.SEnabledResourceBase + + Priority int `default:"100" nullable:"false" list:"user" create:"optional" update:"user"` + // ModelPattern optionally matches the requested model id (implementation-specific glob/prefix). + ModelPattern string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` + // AiProxyNodeId optionally binds the rule to one aiproxy instance (ai_proxy_node id). + AiProxyNodeId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"` +} + +type SAiRoutingManager struct { + db.SSharableVirtualResourceBaseManager + db.SEnabledResourceBaseManager +} + +var AiRoutingManager *SAiRoutingManager + +func init() { + AiRoutingManager = &SAiRoutingManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SAiRouting{}, + "ai_routings_tbl", + "ai_routing", + "ai_routings", + ), + } + AiRoutingManager.SetVirtualObject(AiRoutingManager) +} + +func (manager *SAiRoutingManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiRoutingListInput, +) (*sqlchemy.SQuery, error) { + q, err := manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter") + } + if v := strings.TrimSpace(query.ModelPattern); v != "" { + q = q.Equals("model_pattern", v) + } + if v := strings.TrimSpace(query.AiProxyNodeId); v != "" { + q = q.Equals("ai_proxy_node_id", v) + } + q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledResourceBaseListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter") + } + return q, nil +} + +func (manager *SAiRoutingManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiRoutingDetails { + rows := make([]api.AiRoutingDetails, len(objs)) + sharableRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + routingIds := make([]string, len(objs)) + for i := range objs { + rows[i].SharableVirtualResourceDetails = sharableRows[i] + routingIds[i] = objs[i].(*SAiRouting).Id + } + if fields == nil || fields.Contains("routing_models") { + for i, rid := range routingIds { + if rid == "" { + continue + } + entries, err := fetchAiRoutingModels(rid, false) + if err != nil { + continue + } + rows[i].RoutingModels = make([]api.AiRoutingModelDetails, len(entries)) + for j := range entries { + e := entries[j] + rows[i].RoutingModels[j] = api.AiRoutingModelDetails{ + AiRoutingId: e.AiRoutingId, + AiProviderId: e.AiProviderId, + AiModelId: e.AiModelId, + Priority: e.Priority, + ModelPattern: e.ModelPattern, + Enabled: e.Enabled.IsTrue(), + } + } + } + } + return rows +} + +func (routing *SAiRouting) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformEnableInput) (jsonutils.JSONObject, error) { + if err := db.EnabledPerformEnable(routing, ctx, userCred, true); err != nil { + return nil, errors.Wrap(err, "EnabledPerformEnable") + } + return nil, nil +} + +func (routing *SAiRouting) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformDisableInput) (jsonutils.JSONObject, error) { + if err := db.EnabledPerformEnable(routing, ctx, userCred, false); err != nil { + return nil, errors.Wrap(err, "EnabledPerformEnable") + } + return nil, nil +} + +func (routing *SAiRouting) ValidateUpdateData( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input *api.AiRoutingUpdateInput, +) (*api.AiRoutingUpdateInput, error) { + var err error + input.SharableVirtualResourceBaseUpdateInput, err = routing.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData") + } + if input.AiProxyNodeId != "" { + input.AiProxyNodeId, err = validateAiProxyNodeId(ctx, userCred, input.AiProxyNodeId) + if err != nil { + return input, err + } + } else if query.Contains("ai_proxy_node_id") { + input.AiProxyNodeId = "" + } + return input, nil +} + +func (manager *SAiRoutingManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiRoutingCreateInput, +) (api.AiRoutingCreateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData") + } + + validatedModels, err := validateAiRoutingModelItems(ctx, userCred, input.Models) + if err != nil { + return input, err + } + input.Models = validatedModels + + input.AiProxyNodeId, err = validateAiProxyNodeId(ctx, userCred, input.AiProxyNodeId) + if err != nil { + return input, err + } + + if input.Enabled == nil && input.Disabled == nil { + input.SetEnabled() + } + return input, nil +} + +func (routing *SAiRouting) PostCreate( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) { + routing.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + input := api.AiRoutingCreateInput{} + if err := data.Unmarshal(&input); err != nil { + log.Errorf("ai_routing PostCreate unmarshal models: %v", err) + return + } + if len(input.Models) == 0 { + return + } + if err := createAiRoutingModels(ctx, userCred, ownerId, routing, input.Models); err != nil { + log.Errorf("ai_routing %s create routing_models: %v", routing.Id, err) + } +} + +// PerformSetModels replaces all ai_routing_models bound to this routing. +func (routing *SAiRouting) PerformSetModels( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.AiRoutingSetModelsInput, +) (jsonutils.JSONObject, error) { + items, err := validateAiRoutingModelItems(ctx, userCred, input.Models) + if err != nil { + return nil, err + } + if err := deleteAiRoutingModels(ctx, routing.Id); err != nil { + return nil, err + } + if len(items) == 0 { + return nil, nil + } + if err := createAiRoutingModels(ctx, userCred, routing.GetOwnerId(), routing, items); err != nil { + return nil, err + } + return nil, nil +} diff --git a/pkg/aiproxy/models/ai_virtual_keys.go b/pkg/aiproxy/models/ai_virtual_keys.go new file mode 100644 index 0000000000..a0e30f000a --- /dev/null +++ b/pkg/aiproxy/models/ai_virtual_keys.go @@ -0,0 +1,314 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "strings" + + "github.com/google/uuid" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/rbacscope" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +const virtualKeyPrefix = "sk-" + +// SAiVirtualKey stores a client-facing virtual API key; upstream routing is resolved from project-scoped ai_routing rules. +type SAiVirtualKey struct { + db.SVirtualResourceBase + db.SEnabledResourceBase + + // OwnerId is the user that owns this virtual key within the project. + OwnerId string `width:"128" charset:"ascii" index:"true" list:"user" nullable:"false" create:"optional" update:"user"` + // VirtualKey is the opaque key id or prefix presented to clients (not the upstream provider secret). + VirtualKey string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + // Limits constrains allowed providers, per-request max_tokens, and request rate. + Limits *api.SAiVirtualKeyLimits `length:"medium" charset:"utf8" list:"user" create:"optional" update:"user"` +} + +type SAiVirtualKeyManager struct { + db.SVirtualResourceBaseManager + db.SEnabledResourceBaseManager +} + +var AiVirtualKeyManager *SAiVirtualKeyManager + +func init() { + AiVirtualKeyManager = &SAiVirtualKeyManager{ + SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager( + SAiVirtualKey{}, + "ai_virtual_keys_tbl", + "ai_virtual_key", + "ai_virtual_keys", + ), + } + AiVirtualKeyManager.SetVirtualObject(AiVirtualKeyManager) +} + +func (m *SAiVirtualKey) GetOwnerId() mcclient.IIdentityProvider { + owner := db.SOwnerId{ + UserId: m.OwnerId, + DomainId: m.DomainId, + ProjectId: m.ProjectId, + } + return &owner +} + +func (manager *SAiVirtualKeyManager) NamespaceScope() rbacscope.TRbacScope { + return rbacscope.ScopeUser +} + +func (m *SAiVirtualKey) IsOwner(userCred mcclient.TokenCredential) bool { + return userCred.GetUserId() == m.OwnerId +} + +func (manager *SAiVirtualKeyManager) ResourceScope() rbacscope.TRbacScope { + return rbacscope.ScopeUser +} + +func (manager *SAiVirtualKeyManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) { + return db.FetchUserInfo(ctx, data) +} + +func (manager *SAiVirtualKeyManager) FilterByOwner( + ctx context.Context, + q *sqlchemy.SQuery, + man db.FilterByOwnerProvider, + userCred mcclient.TokenCredential, + owner mcclient.IIdentityProvider, + scope rbacscope.TRbacScope, +) *sqlchemy.SQuery { + if owner != nil && scope == rbacscope.ScopeUser { + if uid := strings.TrimSpace(owner.GetUserId()); uid != "" { + q = q.Equals("owner_id", uid) + } + } + return q +} + +func (manager *SAiVirtualKeyManager) ListItemFilter( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiVirtualKeyListInput, +) (*sqlchemy.SQuery, error) { + var err error + q, err = manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter") + } + q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledResourceBaseListInput) + if err != nil { + return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter") + } + if v := strings.TrimSpace(query.VirtualKey); v != "" { + q = q.Equals("virtual_key", v) + } + userId := strings.TrimSpace(query.UserId) + if userId != "" { + if !db.IsAdminAllowList(userCred, manager).Result.IsAllow() { + return nil, httperrors.NewForbiddenError("only admin may filter by user_id") + } + uc, err := db.UserCacheManager.FetchUserByIdOrName(ctx, userId) + if err != nil { + return nil, errors.Wrap(err, "fetch user") + } + q = q.Equals("owner_id", uc.Id) + } else if !db.IsAdminAllowList(userCred, manager).Result.IsAllow() { + q = q.Equals("owner_id", userCred.GetUserId()) + } + return q, nil +} + +func (manager *SAiVirtualKeyManager) OrderByExtraFields( + ctx context.Context, + q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, + query api.AiVirtualKeyListInput, +) (*sqlchemy.SQuery, error) { + return manager.SVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.VirtualResourceListInput) +} + +func (manager *SAiVirtualKeyManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + q, err := manager.SVirtualResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + return q, httperrors.ErrNotFound +} + +func (manager *SAiVirtualKeyManager) FetchCustomizeColumns( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + objs []interface{}, + fields stringutils2.SSortedStrings, + isList bool, +) []api.AiVirtualKeyDetails { + rows := make([]api.AiVirtualKeyDetails, len(objs)) + virtRows := manager.SVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + userIds := make([]string, len(objs)) + for i := range objs { + rows[i].VirtualResourceDetails = virtRows[i] + vk := objs[i].(*SAiVirtualKey) + if strings.TrimSpace(vk.OwnerId) != "" { + userIds[i] = vk.OwnerId + } + } + userMaps, err := db.FetchIdNameMap2(db.UserCacheManager, userIds) + if err != nil { + log.Errorf("FetchIdNameMap2 fail: %v", err) + return rows + } + for i := range rows { + rows[i].OwnerName, _ = userMaps[userIds[i]] + } + return rows +} + +func (m *SAiVirtualKey) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformEnableInput) (jsonutils.JSONObject, error) { + if err := db.EnabledPerformEnable(m, ctx, userCred, true); err != nil { + return nil, errors.Wrap(err, "EnabledPerformEnable") + } + return nil, nil +} + +func (m *SAiVirtualKey) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformDisableInput) (jsonutils.JSONObject, error) { + if err := db.EnabledPerformEnable(m, ctx, userCred, false); err != nil { + return nil, errors.Wrap(err, "EnabledPerformEnable") + } + return nil, nil +} + +func (manager *SAiVirtualKeyManager) ValidateCreateData( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.AiVirtualKeyCreateInput, +) (api.AiVirtualKeyCreateInput, error) { + var err error + input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "SVirtualResourceBaseManager.ValidateCreateData") + } + + if strings.TrimSpace(input.OwnerId) == "" { + input.OwnerId = userCred.GetUserId() + } else if !db.IsAdminAllowList(userCred, manager).Result.IsAllow() && input.OwnerId != userCred.GetUserId() { + return input, httperrors.NewForbiddenError("cannot create virtual key for another user") + } + + if err := validateAiVirtualKeyLimits(ctx, userCred, input.Limits); err != nil { + return input, err + } + + vk := strings.TrimSpace(input.VirtualKey) + if vk != "" { + if !strings.HasPrefix(vk, virtualKeyPrefix) { + return input, errors.Wrapf(httperrors.ErrInputParameter, "virtual_key must start with %q", virtualKeyPrefix) + } + if len(vk) > 128 { + return input, errors.Wrap(httperrors.ErrInputParameter, "virtual_key too long") + } + exists, err := aiVirtualKeyExists(vk) + if err != nil { + return input, err + } + if exists { + return input, errors.Wrap(httperrors.ErrConflict, "virtual_key already exists") + } + input.VirtualKey = vk + } else { + input.VirtualKey, err = generateUniqueVirtualKey() + if err != nil { + return input, err + } + } + + if input.Enabled.IsNone() { + input.Enabled = tristate.True + } + + return input, nil +} + +func validateAiVirtualKeyLimits(ctx context.Context, userCred mcclient.TokenCredential, lim *api.SAiVirtualKeyLimits) error { + if lim == nil { + return nil + } + if lim.MaxTokensPerRequest < 0 { + return errors.Wrap(httperrors.ErrInputParameter, "limits.max_tokens_per_request must be >= 0") + } + if lim.RequestsPerMinute < 0 { + return errors.Wrap(httperrors.ErrInputParameter, "limits.requests_per_minute must be >= 0") + } + if len(lim.AllowedAiProviderIds) == 0 { + return nil + } + resolved := make([]string, 0, len(lim.AllowedAiProviderIds)) + for _, idOrName := range lim.AllowedAiProviderIds { + idOrName = strings.TrimSpace(idOrName) + if idOrName == "" { + continue + } + pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, idOrName) + if err != nil { + return errors.Wrapf(err, "limits.allowed_ai_provider_ids: fetch %q", idOrName) + } + prov := pObj.(*SAiProvider) + if !prov.GetEnabled() { + return errors.Wrapf(httperrors.ErrInvalidStatus, "limits.allowed_ai_provider_ids: ai_provider %q disabled", idOrName) + } + resolved = append(resolved, prov.Id) + } + lim.AllowedAiProviderIds = resolved + return nil +} + +func aiVirtualKeyExists(virtualKey string) (bool, error) { + cnt, err := AiVirtualKeyManager.Query().Equals("virtual_key", virtualKey).CountWithError() + if err != nil { + return false, errors.Wrap(err, "count ai_virtual_key") + } + return cnt > 0, nil +} + +func generateUniqueVirtualKey() (string, error) { + const maxAttempts = 8 + for i := 0; i < maxAttempts; i++ { + vk := virtualKeyPrefix + strings.ReplaceAll(uuid.New().String(), "-", "") + exists, err := aiVirtualKeyExists(vk) + if err != nil { + return "", err + } + if !exists { + return vk, nil + } + } + return "", errors.Wrap(httperrors.ErrConflict, "failed to generate unique virtual_key") +} diff --git a/pkg/aiproxy/models/aiproxy_catalog_validate.go b/pkg/aiproxy/models/aiproxy_catalog_validate.go new file mode 100644 index 0000000000..f1dc62a297 --- /dev/null +++ b/pkg/aiproxy/models/aiproxy_catalog_validate.go @@ -0,0 +1,181 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "net/url" + "regexp" + "strings" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +const ( + maxAiProviderKeyLen = 64 + maxAiModelKeyLen = 256 +) + +var aiCatalogIdentifierRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +func validateAiCatalogIdentifier(field, value string, maxLen int) (string, error) { + v := strings.TrimSpace(value) + if v == "" { + return "", errors.Wrapf(httperrors.ErrInputParameter, "%s is required", field) + } + if len(v) > maxLen { + return "", errors.Wrapf(httperrors.ErrInputParameter, "%s too long (max %d)", field, maxLen) + } + if !aiCatalogIdentifierRe.MatchString(v) { + return "", errors.Wrapf(httperrors.ErrInputParameter, "%s must match [a-z0-9][a-z0-9_-]*", field) + } + return v, nil +} + +func validateAiModelKey(modelKey string) (string, error) { + key := strings.TrimSpace(modelKey) + if key == "" { + return "", errors.Wrap(httperrors.ErrInputParameter, "model_key is required") + } + if len(key) > maxAiModelKeyLen { + return "", errors.Wrap(httperrors.ErrInputParameter, "model_key too long") + } + return key, nil +} + +// catalogModelId returns a stable ai_model row id for catalog seed (readable when model_key is simple). +// Format: {provider_key}-{slug(model_key)}; falls back to GenId for path-like or overlong keys. +func catalogModelId(providerKey, modelKey string) string { + pk := strings.ToLower(strings.TrimSpace(providerKey)) + mk := strings.TrimSpace(modelKey) + if pk == "" || mk == "" { + return stringutils2.GenId("aiproxy.ai_model", providerKey, modelKey) + } + slug := catalogModelKeySlug(mk) + id := pk + "-" + slug + const maxIdLen = 128 + if len(id) > maxIdLen { + trim := maxIdLen - len(pk) - 1 + if trim > 0 { + id = pk + "-" + slug[:trim] + } else { + id = pk[:maxIdLen] + } + } + if aiCatalogIdentifierRe.MatchString(id) { + return id + } + return stringutils2.GenId("aiproxy.ai_model", providerKey, modelKey) +} + +func catalogModelKeySlug(modelKey string) string { + var b strings.Builder + lastDash := false + for _, r := range strings.TrimSpace(modelKey) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + lastDash = false + case r >= 'A' && r <= 'Z': + b.WriteRune(r + ('a' - 'A')) + lastDash = false + case r == '-', r == '_', r == '.', r == '/': + if b.Len() > 0 && !lastDash { + b.WriteByte('-') + lastDash = true + } + } + } + s := strings.Trim(b.String(), "-") + if s == "" { + return "model" + } + return s +} + +func validateAiProviderConfig(cfg *api.SAiProviderConfig) error { + if cfg == nil || cfg.IsZero() { + return nil + } + baseURL := cfg.ResolvedBaseURL() + if baseURL == "" { + return nil + } + u, err := url.Parse(baseURL) + if err != nil { + return errors.Wrapf(httperrors.ErrInputParameter, "config.base_url: invalid URL: %v", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return errors.Wrap(httperrors.ErrInputParameter, "config.base_url must use http or https scheme") + } + if strings.TrimSpace(u.Host) == "" { + return errors.Wrap(httperrors.ErrInputParameter, "config.base_url must include a host") + } + return nil +} + +func normalizeAiProviderConfig(cfg *api.SAiProviderConfig) *api.SAiProviderConfig { + if cfg == nil || cfg.IsZero() { + return cfg + } + out := &api.SAiProviderConfig{} + if base := cfg.ResolvedBaseURL(); base != "" { + out.BaseURL = base + } + if key := cfg.ResolvedAPIKey(); key != "" { + out.APIKey = key + } + return out +} + +func ensureAiModelKeyUniquePerProvider(ctx context.Context, providerId, modelKey, excludeId string) error { + q := AiModelManager.Query().Equals("ai_provider_id", providerId).Equals("model_key", modelKey) + if excludeId != "" { + q = q.NotEquals("id", excludeId) + } + cnt, err := q.CountWithError() + if err != nil { + return errors.Wrap(err, "count ai_model by provider and model_key") + } + if cnt > 0 { + return errors.Wrapf(httperrors.ErrConflict, "model_key %q already exists for ai_provider", modelKey) + } + return nil +} + +func fetchEnabledAiProvider(ctx context.Context, userCred mcclient.TokenCredential, idOrName string) (*SAiProvider, error) { + idOrName = strings.TrimSpace(idOrName) + if idOrName == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "ai_provider_id is required") + } + pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, idOrName) + if err != nil { + return nil, errors.Wrap(err, "fetch ai_provider") + } + prov := pObj.(*SAiProvider) + if !prov.GetEnabled() { + return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "ai_provider %q is disabled", idOrName) + } + return prov, nil +} + +func defaultAiModelName(providerName, modelKey string) string { + return catalogModelId(providerName, modelKey) +} diff --git a/pkg/aiproxy/models/catalog_seed.go b/pkg/aiproxy/models/catalog_seed.go new file mode 100644 index 0000000000..f3d2fa939b --- /dev/null +++ b/pkg/aiproxy/models/catalog_seed.go @@ -0,0 +1,229 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "fmt" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) + +// standardCatalogProviderKeys lists built-in provider_key values seeded at InitDB. +var standardCatalogProviderKeys = []string{ + "anthropic", + "azure", + "bedrock", + "cerebras", + "cohere", + "gemini", + "groq", + "mistral", + "ollama", + "openai", + "parasail", + "perplexity", + "sgl", + "vertex", + "openrouter", + "elevenlabs", + "huggingface", + "nebius", + "xai", + "replicate", + "vllm", + "runway", + "fireworks", + "aliyun", + "baidu", + "xiaomi", +} + +// defaultPublicBaseURL returns a well-known public API base for OpenAI-compatible upstreams. +// Empty string means no default in catalog (user must set base_url in provider config). +func defaultPublicBaseURL(providerKey string) string { + switch strings.ToLower(strings.TrimSpace(providerKey)) { + case "openai": + return "https://api.openai.com" + case "anthropic": + return "https://api.anthropic.com" + case "azure", "bedrock", "sgl", "vertex": + return "" + case "cerebras": + return "https://api.cerebras.ai" + case "cohere": + return "https://api.cohere.ai" + case "gemini": + return "https://generativelanguage.googleapis.com/v1beta" + case "groq": + return "https://api.groq.com/openai" + case "mistral": + return "https://api.mistral.ai" + case "ollama": + return "http://127.0.0.1:11434" + case "vllm": + return "http://127.0.0.1:8000" + case "parasail": + return "https://api.parasail.io" + case "perplexity": + return "https://api.perplexity.ai" + case "openrouter": + return "https://openrouter.ai/api" + case "elevenlabs": + return "https://api.elevenlabs.io" + case "huggingface": + return "https://router.huggingface.co" + case "nebius": + return "https://api.tokenfactory.nebius.com" + case "xai": + return "https://api.x.ai" + case "replicate": + return "https://api.replicate.com" + case "runway": + return "https://api.dev.runwayml.com" + case "fireworks": + return "https://api.fireworks.ai/inference" + case "aliyun": + return "https://dashscope.aliyuncs.com/compatible-mode" + case "baidu": + return "https://qianfan.baidubce.com/v2" + case "xiaomi": + return "https://api.xiaomimimo.com" + default: + return "" + } +} + +func standardProviderConfig(providerKey string) *api.SAiProviderConfig { + if u := defaultPublicBaseURL(providerKey); u != "" { + return &api.SAiProviderConfig{BaseURL: u} + } + return nil +} + +const placeholderCatalogModelKey = "default" + +func catalogProviderId(providerKey string) string { + return providerKey +} + +func catalogProviderExists(providerId string) (bool, error) { + cnt, err := AiProviderManager.Query().Equals("id", providerId).CountWithError() + if err != nil { + return false, errors.Wrap(err, "count catalog ai_provider") + } + return cnt > 0, nil +} + +func catalogModelExists(modelId string) (bool, error) { + cnt, err := AiModelManager.Query().Equals("id", modelId).CountWithError() + if err != nil { + return false, errors.Wrap(err, "count catalog ai_model") + } + return cnt > 0, nil +} + +func insertCatalogProvider(ctx context.Context, providerKey, description string, cfg *api.SAiProviderConfig) error { + providerId := catalogProviderId(providerKey) + exists, err := catalogProviderExists(providerId) + if err != nil { + return err + } + if exists { + return nil + } + + prov := SAiProvider{} + prov.SetModelManager(AiProviderManager, &prov) + prov.Id = providerId + prov.Name = providerKey + prov.ProviderKey = providerKey + prov.Description = description + prov.Config = cfg + prov.SetEnabled(true) + prov.Status = apis.STATUS_AVAILABLE + prov.Progress = 100 + if err := AiProviderManager.TableSpec().Insert(ctx, &prov); err != nil { + return errors.Wrapf(err, "insert ai_provider %s", providerKey) + } + return nil +} + +func insertCatalogModel(ctx context.Context, providerId, providerKey, modelKey, description string) error { + modelId := catalogModelId(providerKey, modelKey) + exists, err := catalogModelExists(modelId) + if err != nil { + return err + } + if exists { + return nil + } + + m := SAiModel{} + m.SetModelManager(AiModelManager, &m) + m.Id = modelId + m.Name = modelId + m.AiProviderId = providerId + m.ModelKey = modelKey + m.Description = description + m.SetEnabled(true) + m.Status = apis.STATUS_AVAILABLE + m.Progress = 100 + if err := AiModelManager.TableSpec().Insert(ctx, &m); err != nil { + return errors.Wrapf(err, "insert ai_model %s/%s", providerKey, modelKey) + } + return nil +} + +func ensureSeedModelsEntries(ctx context.Context, providerId, providerKey string, entries []catalogSeedModel) error { + if len(entries) == 0 { + return insertCatalogModel(ctx, providerId, providerKey, placeholderCatalogModelKey, + "Catalog seed placeholder; replace with concrete model_key values or use a provider with a built-in catalog.") + } + for i := range entries { + if err := insertCatalogModel(ctx, providerId, providerKey, entries[i].ModelKey, entries[i].Description); err != nil { + return err + } + } + return nil +} + +func ensureSeedProvider(ctx context.Context, providerKey string) error { + providerKey = strings.TrimSpace(providerKey) + providerId := catalogProviderId(providerKey) + if err := insertCatalogProvider(ctx, providerKey, + fmt.Sprintf("Standard provider catalog entry: %s", providerKey), + standardProviderConfig(providerKey)); err != nil { + return err + } + return ensureSeedModelsEntries(ctx, providerId, providerKey, catalogSeedModelsForProvider(providerKey)) +} + +// SeedStandardCatalog inserts built-in ai_provider / ai_model catalog rows on first init only. +// Existing rows are left unchanged so user config survives service restarts. +func SeedStandardCatalog(ctx context.Context) error { + for _, pk := range standardCatalogProviderKeys { + if err := ensureSeedProvider(ctx, pk); err != nil { + return err + } + } + log.Infof("aiproxy: standard catalog seed completed (%d providers)", len(standardCatalogProviderKeys)) + return nil +} diff --git a/pkg/aiproxy/models/catalog_seed_models.go b/pkg/aiproxy/models/catalog_seed_models.go new file mode 100644 index 0000000000..d36f9ce0ba --- /dev/null +++ b/pkg/aiproxy/models/catalog_seed_models.go @@ -0,0 +1,286 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +// catalogSeedModel is one row to insert into ai_models when seeding a standard provider. +// ModelKey is the id sent to the upstream API (no "provider/" prefix). +type catalogSeedModel struct { + ModelKey string + Description string +} + +// catalogSeedModelsForProvider returns known public model ids for seeding. +// Curated from vendor/provider docs; extend as products ship. +// Providers without a list return nil and the seeder inserts model_key "default". +func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { + switch providerKey { + case "anthropic": + return []catalogSeedModel{ + {ModelKey: "claude-opus-4-20250514", Description: "Anthropic Claude Opus 4"}, + {ModelKey: "claude-sonnet-4-20250514", Description: "Anthropic Claude Sonnet 4"}, + {ModelKey: "claude-3-7-sonnet-20250219", Description: "Anthropic Claude 3.7 Sonnet"}, + {ModelKey: "claude-3-5-sonnet-20241022", Description: "Anthropic Claude 3.5 Sonnet"}, + {ModelKey: "claude-3-5-haiku-20241022", Description: "Anthropic Claude 3.5 Haiku"}, + {ModelKey: "claude-3-opus-20240229", Description: "Anthropic Claude 3 Opus"}, + {ModelKey: "claude-3-haiku-20240307", Description: "Anthropic Claude 3 Haiku"}, + } + case "azure": + // Azure OpenAI uses deployment names; these match common Azure OpenAI deployment ids. + return []catalogSeedModel{ + {ModelKey: "gpt-4o", Description: "Azure OpenAI GPT-4o deployment"}, + {ModelKey: "gpt-4o-mini", Description: "Azure OpenAI GPT-4o mini deployment"}, + {ModelKey: "gpt-4", Description: "Azure OpenAI GPT-4 deployment"}, + {ModelKey: "gpt-35-turbo", Description: "Azure OpenAI GPT-3.5 Turbo deployment"}, + {ModelKey: "o3-mini", Description: "Azure OpenAI o3-mini deployment"}, + } + case "bedrock": + return []catalogSeedModel{ + {ModelKey: "anthropic.claude-3-5-sonnet-20241022-v2:0", Description: "Bedrock Claude 3.5 Sonnet"}, + {ModelKey: "anthropic.claude-3-5-haiku-20241022-v1:0", Description: "Bedrock Claude 3.5 Haiku"}, + {ModelKey: "anthropic.claude-3-opus-20240229-v1:0", Description: "Bedrock Claude 3 Opus"}, + {ModelKey: "anthropic.claude-3-sonnet-20240229-v1:0", Description: "Bedrock Claude 3 Sonnet"}, + {ModelKey: "anthropic.claude-3-haiku-20240307-v1:0", Description: "Bedrock Claude 3 Haiku"}, + {ModelKey: "meta.llama3-70b-instruct-v1:0", Description: "Bedrock Llama 3 70B Instruct"}, + {ModelKey: "meta.llama3-8b-instruct-v1:0", Description: "Bedrock Llama 3 8B Instruct"}, + {ModelKey: "mistral.mistral-large-2402-v1:0", Description: "Bedrock Mistral Large"}, + {ModelKey: "amazon.titan-text-express-v1", Description: "Bedrock Amazon Titan Text Express"}, + } + case "cerebras": + return []catalogSeedModel{ + {ModelKey: "llama3.1-8b", Description: "Cerebras Llama 3.1 8B"}, + {ModelKey: "llama3.1-70b", Description: "Cerebras Llama 3.1 70B"}, + {ModelKey: "llama-3.3-70b", Description: "Cerebras Llama 3.3 70B"}, + } + case "cohere": + return []catalogSeedModel{ + {ModelKey: "command-r-plus", Description: "Cohere Command R+"}, + {ModelKey: "command-r", Description: "Cohere Command R"}, + {ModelKey: "command-a", Description: "Cohere Command A"}, + {ModelKey: "command", Description: "Cohere Command"}, + {ModelKey: "command-light", Description: "Cohere Command Light"}, + {ModelKey: "embed-english-v3.0", Description: "Cohere Embed English v3"}, + {ModelKey: "embed-multilingual-v3.0", Description: "Cohere Embed Multilingual v3"}, + } + case "elevenlabs": + return []catalogSeedModel{ + {ModelKey: "eleven_multilingual_v2", Description: "ElevenLabs multilingual v2"}, + {ModelKey: "eleven_turbo_v2_5", Description: "ElevenLabs Turbo v2.5"}, + {ModelKey: "eleven_flash_v2_5", Description: "ElevenLabs Flash v2.5"}, + {ModelKey: "eleven_multilingual_v1", Description: "ElevenLabs multilingual v1"}, + } + case "fireworks": + return []catalogSeedModel{ + {ModelKey: "accounts/fireworks/models/llama-v3p1-8b-instruct", Description: "Fireworks Llama 3.1 8B Instruct"}, + {ModelKey: "accounts/fireworks/models/llama-v3p1-70b-instruct", Description: "Fireworks Llama 3.1 70B Instruct"}, + {ModelKey: "accounts/fireworks/models/llama-v3p3-70b-instruct", Description: "Fireworks Llama 3.3 70B Instruct"}, + {ModelKey: "accounts/fireworks/models/mixtral-8x7b-instruct", Description: "Fireworks Mixtral 8x7B Instruct"}, + } + case "gemini": + return []catalogSeedModel{ + {ModelKey: "gemini-2.0-flash", Description: "Google Gemini 2.0 Flash"}, + {ModelKey: "gemini-2.0-flash-lite", Description: "Google Gemini 2.0 Flash-Lite"}, + {ModelKey: "gemini-1.5-pro", Description: "Google Gemini 1.5 Pro"}, + {ModelKey: "gemini-1.5-flash", Description: "Google Gemini 1.5 Flash"}, + {ModelKey: "gemini-1.5-flash-8b", Description: "Google Gemini 1.5 Flash 8B"}, + {ModelKey: "gemini-embedding-001", Description: "Google Gemini Embedding 001"}, + } + case "groq": + return []catalogSeedModel{ + {ModelKey: "llama-3.3-70b-versatile", Description: "Groq Llama 3.3 70B Versatile"}, + {ModelKey: "llama-3.1-8b-instant", Description: "Groq Llama 3.1 8B Instant"}, + {ModelKey: "llama-3.1-70b-versatile", Description: "Groq Llama 3.1 70B Versatile"}, + {ModelKey: "mixtral-8x7b-32768", Description: "Groq Mixtral 8x7B"}, + {ModelKey: "gemma2-9b-it", Description: "Groq Gemma2 9B IT"}, + } + case "huggingface": + return []catalogSeedModel{ + {ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "HF Llama 3.1 8B Instruct"}, + {ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "HF Llama 3.1 70B Instruct"}, + {ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "HF Mistral 7B Instruct"}, + {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "HF Qwen2.5 72B Instruct"}, + } + case "mistral": + return []catalogSeedModel{ + {ModelKey: "mistral-large-latest", Description: "Mistral Large (latest)"}, + {ModelKey: "mistral-small-latest", Description: "Mistral Small (latest)"}, + {ModelKey: "pixtral-12b-2409", Description: "Mistral Pixtral 12B"}, + {ModelKey: "codestral-latest", Description: "Mistral Codestral (latest)"}, + {ModelKey: "ministral-8b-latest", Description: "Mistral Ministral 8B"}, + {ModelKey: "open-mistral-nemo", Description: "Mistral Open Mistral Nemo"}, + {ModelKey: "mixtral-8x22b", Description: "Mistral Mixtral 8x22B"}, + {ModelKey: "mixtral-8x7b", Description: "Mistral Mixtral 8x7B"}, + } + case "nebius": + return []catalogSeedModel{ + {ModelKey: "deepseek-ai/DeepSeek-V3", Description: "Nebius DeepSeek V3"}, + {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "Nebius Qwen2.5 72B Instruct"}, + {ModelKey: "meta-llama/Llama-3.3-70B-Instruct", Description: "Nebius Llama 3.3 70B Instruct"}, + } + case "ollama": + return []catalogSeedModel{ + {ModelKey: "llama3.2", Description: "Ollama Llama 3.2"}, + {ModelKey: "llama3.1", Description: "Ollama Llama 3.1"}, + {ModelKey: "mistral", Description: "Ollama Mistral"}, + {ModelKey: "qwen2.5", Description: "Ollama Qwen 2.5"}, + {ModelKey: "codellama", Description: "Ollama Code Llama"}, + {ModelKey: "phi3", Description: "Ollama Phi 3"}, + } + case "vllm": + return []catalogSeedModel{ + {ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "vLLM Llama 3.1 8B Instruct"}, + {ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "vLLM Llama 3.1 70B Instruct"}, + {ModelKey: "Qwen/Qwen2.5-7B-Instruct", Description: "vLLM Qwen2.5 7B Instruct"}, + {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "vLLM Qwen2.5 72B Instruct"}, + {ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "vLLM Mistral 7B Instruct"}, + } + case "openai": + return []catalogSeedModel{ + {ModelKey: "gpt-5-nano", Description: "OpenAI GPT-5 nano"}, + {ModelKey: "gpt-5-mini", Description: "OpenAI GPT-5 mini"}, + {ModelKey: "gpt-5", Description: "OpenAI GPT-5"}, + {ModelKey: "gpt-5.1", Description: "OpenAI GPT-5.1"}, + {ModelKey: "gpt-5.1-mini", Description: "OpenAI GPT-5.1 mini"}, + {ModelKey: "gpt-5.1-codex", Description: "OpenAI GPT-5.1 Codex"}, + {ModelKey: "gpt-5.1-codex-max", Description: "OpenAI GPT-5.1 Codex Max"}, + {ModelKey: "gpt-5.2", Description: "OpenAI GPT-5.2"}, + {ModelKey: "gpt-5.2-pro", Description: "OpenAI GPT-5.2 pro"}, + {ModelKey: "gpt-5.2-codex", Description: "OpenAI GPT-5.2 Codex"}, + {ModelKey: "gpt-4.1", Description: "OpenAI GPT-4.1"}, + {ModelKey: "gpt-4.1-mini", Description: "OpenAI GPT-4.1 mini"}, + {ModelKey: "gpt-4.1-nano", Description: "OpenAI GPT-4.1 nano"}, + {ModelKey: "gpt-4o", Description: "OpenAI GPT-4o"}, + {ModelKey: "gpt-4o-mini", Description: "OpenAI GPT-4o mini"}, + {ModelKey: "chatgpt-4o-latest", Description: "OpenAI ChatGPT-4o latest"}, + {ModelKey: "gpt-4-turbo", Description: "OpenAI GPT-4 Turbo"}, + {ModelKey: "gpt-4", Description: "OpenAI GPT-4"}, + {ModelKey: "gpt-3.5-turbo", Description: "OpenAI GPT-3.5 Turbo"}, + {ModelKey: "o1", Description: "OpenAI o1"}, + {ModelKey: "o1-mini", Description: "OpenAI o1-mini"}, + {ModelKey: "o1-preview", Description: "OpenAI o1-preview"}, + {ModelKey: "o3", Description: "OpenAI o3"}, + {ModelKey: "o3-mini", Description: "OpenAI o3-mini"}, + {ModelKey: "o4-mini", Description: "OpenAI o4-mini"}, + {ModelKey: "text-embedding-3-small", Description: "OpenAI text-embedding-3-small"}, + {ModelKey: "text-embedding-3-large", Description: "OpenAI text-embedding-3-large"}, + {ModelKey: "text-embedding-ada-002", Description: "OpenAI text-embedding-ada-002"}, + } + case "openrouter": + return []catalogSeedModel{ + {ModelKey: "openai/gpt-4o", Description: "OpenRouter OpenAI GPT-4o"}, + {ModelKey: "openai/gpt-4o-mini", Description: "OpenRouter OpenAI GPT-4o mini"}, + {ModelKey: "anthropic/claude-3.5-sonnet", Description: "OpenRouter Claude 3.5 Sonnet"}, + {ModelKey: "anthropic/claude-3.5-haiku", Description: "OpenRouter Claude 3.5 Haiku"}, + {ModelKey: "google/gemini-2.0-flash-001", Description: "OpenRouter Gemini 2.0 Flash"}, + {ModelKey: "meta-llama/llama-3.3-70b-instruct", Description: "OpenRouter Llama 3.3 70B Instruct"}, + {ModelKey: "mistralai/mistral-large", Description: "OpenRouter Mistral Large"}, + } + case "perplexity": + return []catalogSeedModel{ + {ModelKey: "sonar", Description: "Perplexity Sonar"}, + {ModelKey: "sonar-pro", Description: "Perplexity Sonar Pro"}, + {ModelKey: "sonar-reasoning", Description: "Perplexity Sonar Reasoning"}, + {ModelKey: "llama-3.1-sonar-small-128k-online", Description: "Perplexity Llama 3.1 Sonar Small online"}, + {ModelKey: "llama-3.1-sonar-large-128k-online", Description: "Perplexity Llama 3.1 Sonar Large online"}, + } + case "replicate": + return []catalogSeedModel{ + {ModelKey: "meta/meta-llama-3-8b-instruct", Description: "Replicate Meta Llama 3 8B Instruct"}, + {ModelKey: "meta/meta-llama-3-70b-instruct", Description: "Replicate Meta Llama 3 70B Instruct"}, + {ModelKey: "mistralai/mixtral-8x7b-instruct-v0.1", Description: "Replicate Mixtral 8x7B Instruct"}, + } + case "runway": + return []catalogSeedModel{ + {ModelKey: "gen3a_turbo", Description: "Runway Gen-3 Alpha Turbo"}, + {ModelKey: "gen3a", Description: "Runway Gen-3 Alpha"}, + {ModelKey: "gen4_aleph", Description: "Runway Gen-4 Aleph"}, + } + case "vertex": + return []catalogSeedModel{ + {ModelKey: "gemini-2.0-flash", Description: "Vertex AI Gemini 2.0 Flash"}, + {ModelKey: "gemini-1.5-pro", Description: "Vertex AI Gemini 1.5 Pro"}, + {ModelKey: "gemini-1.5-flash", Description: "Vertex AI Gemini 1.5 Flash"}, + {ModelKey: "publishers/google/models/gemini-1.5-pro", Description: "Vertex publisher path Gemini 1.5 Pro"}, + } + case "xai": + return []catalogSeedModel{ + {ModelKey: "grok-3", Description: "xAI Grok 3"}, + {ModelKey: "grok-3-mini", Description: "xAI Grok 3 mini"}, + {ModelKey: "grok-2-latest", Description: "xAI Grok 2 latest"}, + {ModelKey: "grok-2-1212", Description: "xAI Grok 2 1212"}, + {ModelKey: "grok-beta", Description: "xAI Grok beta"}, + } + case "aliyun": + return aliyunQwenSeedModels() + case "baidu": + return baiduErnieSeedModels() + case "xiaomi": + return xiaomiMimoSeedModels() + default: + return nil + } +} + +func aliyunQwenSeedModels() []catalogSeedModel { + return []catalogSeedModel{ + {ModelKey: "qwen-turbo", Description: "Alibaba Qwen Turbo"}, + {ModelKey: "qwen-plus", Description: "Alibaba Qwen Plus"}, + {ModelKey: "qwen-max", Description: "Alibaba Qwen Max"}, + {ModelKey: "qwen-long", Description: "Alibaba Qwen Long context"}, + {ModelKey: "qwen-vl-max", Description: "Alibaba Qwen-VL Max"}, + {ModelKey: "qwen-vl-plus", Description: "Alibaba Qwen-VL Plus"}, + {ModelKey: "qwen-vl-ocr", Description: "Alibaba Qwen-VL OCR"}, + {ModelKey: "qwen2.5-0.5b-instruct", Description: "Alibaba Qwen2.5 0.5B Instruct"}, + {ModelKey: "qwen2.5-1.5b-instruct", Description: "Alibaba Qwen2.5 1.5B Instruct"}, + {ModelKey: "qwen2.5-3b-instruct", Description: "Alibaba Qwen2.5 3B Instruct"}, + {ModelKey: "qwen2.5-7b-instruct", Description: "Alibaba Qwen2.5 7B Instruct"}, + {ModelKey: "qwen2.5-14b-instruct", Description: "Alibaba Qwen2.5 14B Instruct"}, + {ModelKey: "qwen2.5-32b-instruct", Description: "Alibaba Qwen2.5 32B Instruct"}, + {ModelKey: "qwen2.5-72b-instruct", Description: "Alibaba Qwen2.5 72B Instruct"}, + {ModelKey: "qwen2.5-coder-7b-instruct", Description: "Alibaba Qwen2.5 Coder 7B Instruct"}, + {ModelKey: "qwen2.5-coder-32b-instruct", Description: "Alibaba Qwen2.5 Coder 32B Instruct"}, + {ModelKey: "qwen3-30b-a3b", Description: "Alibaba Qwen3 30B A3B MoE"}, + {ModelKey: "qwen3-32b", Description: "Alibaba Qwen3 32B"}, + {ModelKey: "qwen3-235b-a22b", Description: "Alibaba Qwen3 235B A22B MoE"}, + {ModelKey: "qwen-math-plus", Description: "Alibaba Qwen Math Plus"}, + {ModelKey: "qwen-coder-plus", Description: "Alibaba Qwen Coder Plus"}, + {ModelKey: "text-embedding-v3", Description: "Alibaba text-embedding-v3"}, + {ModelKey: "text-embedding-v4", Description: "Alibaba text-embedding-v4"}, + } +} + +func baiduErnieSeedModels() []catalogSeedModel { + return []catalogSeedModel{ + {ModelKey: "ernie-4.0-turbo-8k", Description: "Baidu ERNIE 4.0 Turbo 8K"}, + {ModelKey: "ernie-4.0-8k", Description: "Baidu ERNIE 4.0 8K"}, + {ModelKey: "ernie-4.0-turbo-128k", Description: "Baidu ERNIE 4.0 Turbo 128K"}, + {ModelKey: "ernie-3.5-8k", Description: "Baidu ERNIE 3.5 8K"}, + {ModelKey: "ernie-3.5-128k", Description: "Baidu ERNIE 3.5 128K"}, + {ModelKey: "ernie-speed-128k", Description: "Baidu ERNIE Speed 128K"}, + {ModelKey: "ernie-lite-8k", Description: "Baidu ERNIE Lite 8K"}, + {ModelKey: "ernie-char-8k", Description: "Baidu ERNIE Character 8K"}, + {ModelKey: "embedding-v1", Description: "Baidu Wenxin embedding-v1"}, + {ModelKey: "tao-8k", Description: "Baidu ERNIE Tao 8K"}, + } +} + +func xiaomiMimoSeedModels() []catalogSeedModel { + return []catalogSeedModel{ + {ModelKey: "mimo-v2.5-pro", Description: "Xiaomi MiMo 2.5 Pro (flagship text)"}, + {ModelKey: "mimo-v2-pro", Description: "Xiaomi MiMo 2 Pro"}, + {ModelKey: "mimo-v2.5", Description: "Xiaomi MiMo 2.5 (multimodal text)"}, + {ModelKey: "mimo-v2-omni", Description: "Xiaomi MiMo 2 Omni (multimodal)"}, + {ModelKey: "mimo-v2-flash", Description: "Xiaomi MiMo 2 Flash (fast)"}, + } +} diff --git a/pkg/aiproxy/models/chat_upstream.go b/pkg/aiproxy/models/chat_upstream.go new file mode 100644 index 0000000000..db80ccd006 --- /dev/null +++ b/pkg/aiproxy/models/chat_upstream.go @@ -0,0 +1,255 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "database/sql" + stderrors "errors" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/rbacscope" + + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +// ChatUpstream holds resolved upstream and the model id to send. +type ChatUpstream struct { + BaseURL string + APIKey string + UpstreamModel string + ProviderKey string + AiProviderId string + AiKeyId string + + // VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row. + VirtualKeyId string + MaxTokensPerRequest int + RequestsPerMinute int +} + +func modelPatternMatches(pattern, requestedModel string) bool { + pattern = strings.TrimSpace(pattern) + if pattern == "" { + return true + } + rm := strings.TrimSpace(requestedModel) + if strings.HasSuffix(pattern, "*") { + return strings.HasPrefix(rm, strings.TrimSuffix(pattern, "*")) + } + return strings.EqualFold(pattern, rm) +} + +func virtualKeyAllowsProvider(vk *SAiVirtualKey, prov *SAiProvider) bool { + if vk == nil || prov == nil { + return false + } + if vk.Limits == nil || len(vk.Limits.AllowedAiProviderIds) == 0 { + return true + } + for _, idOrName := range vk.Limits.AllowedAiProviderIds { + idOrName = strings.TrimSpace(idOrName) + if idOrName == "" { + continue + } + if idOrName == prov.Id || strings.EqualFold(idOrName, prov.Name) { + return true + } + } + return false +} + +func loadEnabledVirtualKey(virtualKey string) (*SAiVirtualKey, error) { + virtualKey = strings.TrimSpace(virtualKey) + if virtualKey == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "missing virtual key (Authorization: Bearer or X-Ai-Virtual-Key)") + } + vk := SAiVirtualKey{} + qvk := AiVirtualKeyManager.Query().Equals("virtual_key", virtualKey).Equals("enabled", true) + err := qvk.First(&vk) + if err != nil { + if stderrors.Is(err, sql.ErrNoRows) { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "virtual key not found or disabled") + } + return nil, errors.Wrap(err, "query ai_virtual_key") + } + if strings.TrimSpace(vk.ProjectId) == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "virtual key has no project") + } + return &vk, nil +} + +// listProjectRoutingsForVirtualKey returns enabled ai_routing rows owned by or shared with the virtual key's project. +func listProjectRoutingsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey) ([]SAiRouting, error) { + if vk == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "nil virtual key") + } + routings := make([]SAiRouting, 0, 16) + qr := AiRoutingManager.Query().Equals("enabled", true) + qr = AiRoutingManager.FilterByOwner(ctx, qr, AiRoutingManager, userCred, vk.GetOwnerId(), rbacscope.ScopeProject) + qr = qr.Asc("priority") + if err := qr.All(&routings); err != nil { + return nil, errors.Wrap(err, "list ai_routings for virtual key project") + } + return routings, nil +} + +// pickRoutingForRequest chooses the first matching ai_routing (lowest priority value wins) +// on the current aiproxy instance. When a matched rule is bound to another node, returns forbidden. +func pickRoutingForRequest(routings []SAiRouting, reqModel, currentNodeId string) (*SAiRouting, error) { + var boundElsewhere *SAiRouting + for i := range routings { + r := &routings[i] + if !modelPatternMatches(r.ModelPattern, reqModel) { + continue + } + if !proxyNodeScopeMatches(r.AiProxyNodeId, currentNodeId) { + if boundElsewhere == nil && strings.TrimSpace(r.AiProxyNodeId) != "" { + boundElsewhere = r + } + continue + } + return r, nil + } + if boundElsewhere != nil { + return nil, errors.Wrapf(httperrors.ErrForbidden, + "ai_routing %q is bound to ai_proxy_node %q; use that instance endpoint", + boundElsewhere.Name, boundElsewhere.AiProxyNodeId) + } + return nil, nil +} + +type resolvedCatalogModel struct { + provider *SAiProvider + model *SAiModel +} + +// resolveCatalogModelFromRouting picks ai_routing_models for the routing and loads catalog provider/model rows. +func resolveCatalogModelFromRouting( + ctx context.Context, + userCred mcclient.TokenCredential, + vk *SAiVirtualKey, + routing *SAiRouting, + reqModel string, +) (*resolvedCatalogModel, error) { + if routing == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing") + } + providerId, modelId, err := pickAiRoutingModel(ctx, userCred, routing, reqModel) + if err != nil { + return nil, err + } + + pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerId) + if err != nil { + return nil, errors.Wrap(err, "fetch ai_provider") + } + prov := pObj.(*SAiProvider) + if !prov.GetEnabled() { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider disabled") + } + if !virtualKeyAllowsProvider(vk, prov) { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider not allowed for this virtual key") + } + + mObj, err := AiModelManager.FetchByIdOrName(ctx, userCred, modelId) + if err != nil { + return nil, errors.Wrap(err, "fetch ai_model") + } + mdl := mObj.(*SAiModel) + if !mdl.GetEnabled() { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model disabled") + } + if strings.TrimSpace(mdl.AiProviderId) != "" && mdl.AiProviderId != prov.Id { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model does not belong to resolved ai_provider") + } + return &resolvedCatalogModel{provider: prov, model: mdl}, nil +} + +// ResolveChatUpstream resolves upstream URL, API key, and catalog model_key for a chat request: +// 1. ai_virtual_key (auth + project scope) +// 2. ai_routing in that project (model_pattern / optional proxy-node scope, priority) +// 3. ai_routing_model -> ai_provider + ai_model +// 4. ai_key rows for that provider matching the catalog model_key (weight), else provider.config api_key +func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, virtualKey string, body *jsonutils.JSONDict) (*ChatUpstream, error) { + vk, err := loadEnabledVirtualKey(virtualKey) + if err != nil { + return nil, err + } + + reqModel, _ := body.GetString("model") + if strings.TrimSpace(reqModel) == "" { + return nil, errors.Wrap(httperrors.ErrInputParameter, "missing model in JSON body") + } + + routings, err := listProjectRoutingsForVirtualKey(ctx, userCred, vk) + if err != nil { + return nil, err + } + routing, err := pickRoutingForRequest(routings, reqModel, CurrentProxyNodeId()) + if err != nil { + return nil, err + } + if routing == nil { + return nil, errors.Wrap(httperrors.ErrNotFound, "no ai_routing matched for virtual key project on this aiproxy node") + } + + resolved, err := resolveCatalogModelFromRouting(ctx, userCred, vk, routing, reqModel) + if err != nil { + return nil, err + } + prov := resolved.provider + mdl := resolved.model + + if prov.Config == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty") + } + baseURL := prov.Config.ResolvedBaseURL() + if baseURL == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config must include base_url") + } + + upstreamModel := strings.TrimSpace(mdl.ModelKey) + if upstreamModel == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model.model_key is empty") + } + + // Keys are scoped to ai_provider; routing on each ai_key matches the resolved catalog model_key. + keyRes, err := resolveUpstreamAPIKey(prov, upstreamModel) + if err != nil { + return nil, err + } + if keyRes == nil || keyRes.Secret == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no api_key for ai_provider and catalog model") + } + + up := &ChatUpstream{ + BaseURL: baseURL, + APIKey: keyRes.Secret, + UpstreamModel: upstreamModel, + ProviderKey: prov.ProviderKey, + AiProviderId: prov.Id, + AiKeyId: keyRes.AiKeyId, + VirtualKeyId: vk.Id, + } + if vk.Limits != nil { + up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest + up.RequestsPerMinute = vk.Limits.RequestsPerMinute + } + return up, nil +} diff --git a/pkg/aiproxy/models/doc.go b/pkg/aiproxy/models/doc.go new file mode 100644 index 0000000000..0d2593c257 --- /dev/null +++ b/pkg/aiproxy/models/doc.go @@ -0,0 +1 @@ +package models // import "yunion.io/x/onecloud/pkg/aiproxy/models" diff --git a/pkg/aiproxy/models/initdb.go b/pkg/aiproxy/models/initdb.go new file mode 100644 index 0000000000..0a73e66fca --- /dev/null +++ b/pkg/aiproxy/models/initdb.go @@ -0,0 +1,46 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" +) + +func InitDB() error { + for _, manager := range []db.IModelManager{ + /* + * Important!!! + * initialization order matters, do not change the order + */ + db.Metadata, + + AiProviderManager, + AiModelManager, + AiKeyManager, + AiVirtualKeyManager, + AiRoutingManager, + AiRoutingModelManager, + AiProxyNodeManager, + } { + err := manager.InitializeData() + if err != nil { + log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err) + return err + } + } + return nil +} diff --git a/pkg/aiproxy/models/list_models.go b/pkg/aiproxy/models/list_models.go new file mode 100644 index 0000000000..0e2c2459a5 --- /dev/null +++ b/pkg/aiproxy/models/list_models.go @@ -0,0 +1,190 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "sort" + "strings" + "time" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +// ModelsListEntry is one OpenAI-compatible model object in GET /openai/v1/models. +type ModelsListEntry struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + OwnedBy string `json:"owned_by"` +} + +// ListModelsForVirtualKey returns OpenAI-compatible model ids reachable by the virtual key +// on the current aiproxy node (project ai_routing -> ai_routing_model -> ai_model). +func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredential, virtualKey string) ([]ModelsListEntry, error) { + vk, err := loadEnabledVirtualKey(virtualKey) + if err != nil { + return nil, err + } + rpm := 0 + if vk.Limits != nil { + rpm = vk.Limits.RequestsPerMinute + } + if err := TakeVirtualKeyRequestsPerMinute(vk.Id, rpm); err != nil { + return nil, err + } + routings, err := listProjectRoutingsForVirtualKey(ctx, userCred, vk) + if err != nil { + return nil, err + } + currentNode := CurrentProxyNodeId() + routingIds := make([]string, 0, len(routings)) + for i := range routings { + if proxyNodeScopeMatches(routings[i].AiProxyNodeId, currentNode) { + routingIds = append(routingIds, routings[i].Id) + } + } + if len(routingIds) == 0 { + return nil, nil + } + + entries := make([]SAiRoutingModel, 0, 16) + q := AiRoutingModelManager.Query().In("ai_routing_id", routingIds).Equals("enabled", true) + if err := q.All(&entries); err != nil { + return nil, errors.Wrap(err, "list ai_routing_models") + } + if len(entries) == 0 { + return nil, nil + } + + providerIds := make([]string, 0, len(entries)) + modelIds := make([]string, 0, len(entries)) + for i := range entries { + providerIds = append(providerIds, entries[i].AiProviderId) + modelIds = append(modelIds, entries[i].AiModelId) + } + + providers, err := fetchEnabledAiProvidersByIds(providerIds) + if err != nil { + return nil, err + } + modelsById, err := fetchEnabledAiModelsByIds(modelIds) + if err != nil { + return nil, err + } + + seen := make(map[string]ModelsListEntry, len(entries)) + created := time.Now().Unix() + for i := range entries { + e := &entries[i] + prov := providers[e.AiProviderId] + mdl := modelsById[e.AiModelId] + if prov == nil || mdl == nil { + continue + } + if !virtualKeyAllowsProvider(vk, prov) { + continue + } + id := clientFacingModelID(e, mdl) + if id == "" { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = ModelsListEntry{ + ID: id, + Object: "model", + Created: created, + OwnedBy: strings.TrimSpace(prov.ProviderKey), + } + } + if len(seen) == 0 { + return nil, nil + } + out := make([]ModelsListEntry, 0, len(seen)) + for _, item := range seen { + out = append(out, item) + } + sort.Slice(out, func(i, j int) bool { + return out[i].ID < out[j].ID + }) + return out, nil +} + +func clientFacingModelID(entry *SAiRoutingModel, mdl *SAiModel) string { + if entry != nil { + if mp := strings.TrimSpace(entry.ModelPattern); mp != "" && !strings.Contains(mp, "*") { + return mp + } + } + if mdl != nil { + return strings.TrimSpace(mdl.ModelKey) + } + return "" +} + +func fetchEnabledAiProvidersByIds(ids []string) (map[string]*SAiProvider, error) { + ids = uniqueNonEmptyStrings(ids) + if len(ids) == 0 { + return map[string]*SAiProvider{}, nil + } + rows := make([]SAiProvider, 0, len(ids)) + q := AiProviderManager.Query().In("id", ids).Equals("enabled", true) + if err := q.All(&rows); err != nil { + return nil, errors.Wrap(err, "list ai_providers") + } + out := make(map[string]*SAiProvider, len(rows)) + for i := range rows { + out[rows[i].Id] = &rows[i] + } + return out, nil +} + +func fetchEnabledAiModelsByIds(ids []string) (map[string]*SAiModel, error) { + ids = uniqueNonEmptyStrings(ids) + if len(ids) == 0 { + return map[string]*SAiModel{}, nil + } + rows := make([]SAiModel, 0, len(ids)) + q := AiModelManager.Query().In("id", ids).Equals("enabled", true) + if err := q.All(&rows); err != nil { + return nil, errors.Wrap(err, "list ai_models") + } + out := make(map[string]*SAiModel, len(rows)) + for i := range rows { + out[rows[i].Id] = &rows[i] + } + return out, nil +} + +func uniqueNonEmptyStrings(in []string) []string { + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + s = strings.TrimSpace(s) + if s == "" { + continue + } + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} diff --git a/pkg/aiproxy/models/list_models_test.go b/pkg/aiproxy/models/list_models_test.go new file mode 100644 index 0000000000..5c0b74a897 --- /dev/null +++ b/pkg/aiproxy/models/list_models_test.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import "testing" + +func TestClientFacingModelID(t *testing.T) { + mdl := &SAiModel{ModelKey: "gpt-4o-mini"} + if got := clientFacingModelID(&SAiRoutingModel{ModelPattern: "fast"}, mdl); got != "fast" { + t.Fatalf("expected alias fast, got %q", got) + } + if got := clientFacingModelID(&SAiRoutingModel{ModelPattern: "gpt-*"}, mdl); got != "gpt-4o-mini" { + t.Fatalf("expected catalog model_key for wildcard pattern, got %q", got) + } + if got := clientFacingModelID(&SAiRoutingModel{}, mdl); got != "gpt-4o-mini" { + t.Fatalf("expected catalog model_key, got %q", got) + } +} + +func TestUniqueNonEmptyStrings(t *testing.T) { + out := uniqueNonEmptyStrings([]string{"a", "a", "", "b", "b"}) + if len(out) != 2 || out[0] != "a" || out[1] != "b" { + t.Fatalf("unexpected dedupe result: %#v", out) + } +} diff --git a/pkg/aiproxy/models/proxy_node_local.go b/pkg/aiproxy/models/proxy_node_local.go new file mode 100644 index 0000000000..f3bde16b2d --- /dev/null +++ b/pkg/aiproxy/models/proxy_node_local.go @@ -0,0 +1,71 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/aiproxy/options" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +var localProxyNodeId string + +// InitLocalProxyNodeId records the ai_proxy_node id for this running aiproxy process. +func InitLocalProxyNodeId(opts *options.SAiProxyOptions, isSlave bool) error { + if isSlave { + addr, err := AdvertiseAddressFromOptions(opts) + if err != nil { + return err + } + localProxyNodeId = aiProxyNodeId(addr) + return nil + } + localProxyNodeId = defaultPrimaryAiProxyNodeId + return nil +} + +// CurrentProxyNodeId returns the ai_proxy_node id of this process. +func CurrentProxyNodeId() string { + return localProxyNodeId +} + +func validateAiProxyNodeId(ctx context.Context, userCred mcclient.TokenCredential, idOrName string) (string, error) { + idOrName = strings.TrimSpace(idOrName) + if idOrName == "" { + return "", nil + } + obj, err := AiProxyNodeManager.FetchByIdOrName(ctx, userCred, idOrName) + if err != nil { + return "", errors.Wrap(err, "fetch ai_proxy_node") + } + node := obj.(*SAiProxyNode) + if !node.GetEnabled() { + return "", errors.Wrapf(httperrors.ErrInvalidStatus, "ai_proxy_node %q is disabled", idOrName) + } + return node.Id, nil +} + +func proxyNodeScopeMatches(routingNodeId, currentNodeId string) bool { + routingNodeId = strings.TrimSpace(routingNodeId) + if routingNodeId == "" { + return true + } + return routingNodeId == strings.TrimSpace(currentNodeId) +} diff --git a/pkg/aiproxy/models/virtual_key_guard.go b/pkg/aiproxy/models/virtual_key_guard.go new file mode 100644 index 0000000000..4f04ac4fc8 --- /dev/null +++ b/pkg/aiproxy/models/virtual_key_guard.go @@ -0,0 +1,63 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "strings" + "sync" + + "golang.org/x/time/rate" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" +) + +var vkRpmLimiters sync.Map // virtual key id -> *rate.Limiter + +// TakeVirtualKeyRequestsPerMinute enforces an approximate per-minute request budget per virtual key (in-process). +func TakeVirtualKeyRequestsPerMinute(vkId string, rpm int) error { + if rpm <= 0 || strings.TrimSpace(vkId) == "" { + return nil + } + limAny, _ := vkRpmLimiters.LoadOrStore(vkId, rate.NewLimiter(rate.Limit(float64(rpm))/60.0, rpm)) + lim := limAny.(*rate.Limiter) + if !lim.Allow() { + return errors.Wrap(httperrors.ErrTooManyRequests, "virtual key request rate exceeded") + } + return nil +} + +// EnforceVirtualKeyMaxTokens caps or injects max_tokens from virtual key limits. +func EnforceVirtualKeyMaxTokens(body *jsonutils.JSONDict, lim *api.SAiVirtualKeyLimits) error { + if lim == nil || lim.MaxTokensPerRequest <= 0 { + return nil + } + cap := int64(lim.MaxTokensPerRequest) + if body.Contains("max_tokens") { + mt, err := body.Int("max_tokens") + if err != nil { + return errors.Wrap(httperrors.ErrInputParameter, "invalid max_tokens") + } + if mt > cap { + return errors.Wrap(httperrors.ErrInputParameter, "max_tokens exceeds virtual key limit") + } + return nil + } + body.Set("max_tokens", jsonutils.NewInt(cap)) + return nil +} diff --git a/pkg/aiproxy/options/doc.go b/pkg/aiproxy/options/doc.go new file mode 100644 index 0000000000..a588194fba --- /dev/null +++ b/pkg/aiproxy/options/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options // import "yunion.io/x/onecloud/pkg/aiproxy/options" diff --git a/pkg/aiproxy/options/options.go b/pkg/aiproxy/options/options.go new file mode 100644 index 0000000000..0b09cbf97d --- /dev/null +++ b/pkg/aiproxy/options/options.go @@ -0,0 +1,45 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" +) + +type SAiProxyOptions struct { + common_options.CommonOptions + common_options.DBOptions + + AdvertiseAddress string `help:"Standby node address advertised to clients, e.g. http://10.0.0.2:30889; default derives from bind address and port" default:""` + NodeHeartbeatIntervalSeconds int `help:"Interval in seconds for standby node registration heartbeat" default:"60"` +} + +var ( + Options SAiProxyOptions +) + +func OnOptionsChange(oldO, newO interface{}) bool { + oldOpts := oldO.(*SAiProxyOptions) + newOpts := newO.(*SAiProxyOptions) + + changed := false + if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) { + changed = true + } + if common_options.OnDBOptionsChange(&oldOpts.DBOptions, &newOpts.DBOptions) { + changed = true + } + return changed +} diff --git a/pkg/aiproxy/policy/defaults.go b/pkg/aiproxy/policy/defaults.go new file mode 100644 index 0000000000..187369fa17 --- /dev/null +++ b/pkg/aiproxy/policy/defaults.go @@ -0,0 +1,58 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "yunion.io/x/pkg/util/rbacscope" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/consts" + common_policy "yunion.io/x/onecloud/pkg/cloudcommon/policy" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +const ( + PolicyActionGet = common_policy.PolicyActionGet + PolicyActionList = common_policy.PolicyActionList +) + +var ( + predefinedDefaultPolicies = []rbacutils.SRbacPolicy{ + { + Auth: true, + Scope: rbacscope.ScopeUser, + Rules: []rbacutils.SRbacRule{ + { + Service: api.SERVICE_TYPE, + Resource: "ai_proxy_nodes", + Action: PolicyActionList, + Result: rbacutils.Allow, + }, + { + Service: api.SERVICE_TYPE, + Resource: "ai_proxy_nodes", + Action: PolicyActionGet, + Result: rbacutils.Allow, + }, + }, + }, + } +) + +func Init() { + if consts.IsEnableDefaultPolicy() { + common_policy.AppendDefaultPolicies(predefinedDefaultPolicies) + } +} diff --git a/pkg/aiproxy/policy/doc.go b/pkg/aiproxy/policy/doc.go new file mode 100644 index 0000000000..b6e646c90b --- /dev/null +++ b/pkg/aiproxy/policy/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy // import "yunion.io/x/onecloud/pkg/aiproxy/policy" diff --git a/pkg/aiproxy/policy/resources.go b/pkg/aiproxy/policy/resources.go new file mode 100644 index 0000000000..b3751d7939 --- /dev/null +++ b/pkg/aiproxy/policy/resources.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + common_policy "yunion.io/x/onecloud/pkg/cloudcommon/policy" +) + +var ( + aiproxySystemResources = []string{} + aiproxyDomainResources = []string{} + aiproxyUserResources = []string{ + "ai_proxy_nodes", + } +) + +func init() { + common_policy.RegisterSystemResources(api.SERVICE_TYPE, aiproxySystemResources) + common_policy.RegisterDomainResources(api.SERVICE_TYPE, aiproxyDomainResources) + common_policy.RegisterUserResources(api.SERVICE_TYPE, aiproxyUserResources) +} diff --git a/pkg/aiproxy/providerapi/doc.go b/pkg/aiproxy/providerapi/doc.go new file mode 100644 index 0000000000..730d9b444c --- /dev/null +++ b/pkg/aiproxy/providerapi/doc.go @@ -0,0 +1 @@ +package providerapi // import "yunion.io/x/onecloud/pkg/aiproxy/providerapi" diff --git a/pkg/aiproxy/providerapi/stream.go b/pkg/aiproxy/providerapi/stream.go new file mode 100644 index 0000000000..d4fc7e65a1 --- /dev/null +++ b/pkg/aiproxy/providerapi/stream.go @@ -0,0 +1,20 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providerapi + +// ContextualStreamPassthrough allows providers to choose SSE passthrough per request. +type ContextualStreamPassthrough interface { + OpenAIStreamPassthroughForContext(ctx *ChatContext) bool +} diff --git a/pkg/aiproxy/providerapi/types.go b/pkg/aiproxy/providerapi/types.go new file mode 100644 index 0000000000..037c1e9cea --- /dev/null +++ b/pkg/aiproxy/providerapi/types.go @@ -0,0 +1,84 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package providerapi defines shared types and interfaces for aiproxy provider adapters. +package providerapi // import "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + +import ( + "yunion.io/x/jsonutils" +) + +// ChatContext holds resolved upstream connectivity for one proxied request. +type ChatContext struct { + ProviderKey string + BaseURL string + APIKey string + UpstreamModel string +} + +// HTTPRequest is the wire-format call sent to an upstream provider. +type HTTPRequest struct { + Method string + URL string + Headers map[string]string + Body []byte +} + +// StreamChunk is one normalized OpenAI chat.completion.chunk SSE payload (JSON only, no "data:" prefix). +type StreamChunk struct { + Data []byte + Done bool +} + +// StreamState carries per-stream conversion state for providers that emit non-OpenAI SSE. +type StreamState struct { + Model string + ResponseID string + ChunkIndex int + TextStarted bool + ToolIndex int + ToolID string + ToolName string + ToolArgsPending string + InToolBlock bool +} + +// Provider converts OpenAI chat/completions to a provider-native HTTP call and +// normalizes responses back to OpenAI format. +type Provider interface { + Key() string + BuildUpstreamRequest(ctx *ChatContext, body *jsonutils.JSONDict, stream bool) (*HTTPRequest, error) + NormalizeResponse(body []byte) ([]byte, error) + OpenAIStreamPassthrough() bool + ConvertStreamEvent(eventType string, payload []byte, state *StreamState) ([]StreamChunk, error) +} + +// EmbeddingsProvider converts OpenAI /v1/embeddings requests to provider-native APIs. +type EmbeddingsProvider interface { + BuildEmbeddingsRequest(ctx *ChatContext, body *jsonutils.JSONDict) (*HTTPRequest, error) + NormalizeEmbeddingsResponse(body []byte) ([]byte, error) +} + +// ImagesProvider converts OpenAI /v1/images/generations requests to provider-native APIs. +type ImagesProvider interface { + BuildImagesGenerationsRequest(ctx *ChatContext, body *jsonutils.JSONDict) (*HTTPRequest, error) + NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) +} + +// CompletionsProvider converts OpenAI /v1/completions requests to provider-native APIs. +type CompletionsProvider interface { + BuildCompletionsRequest(ctx *ChatContext, body *jsonutils.JSONDict, stream bool) (*HTTPRequest, error) + NormalizeCompletionsResponse(body []byte) ([]byte, error) + OpenAICompletionsStreamPassthrough() bool +} diff --git a/pkg/aiproxy/providers/aliyun/aliyun.go b/pkg/aiproxy/providers/aliyun/aliyun.go new file mode 100644 index 0000000000..01b845f583 --- /dev/null +++ b/pkg/aiproxy/providers/aliyun/aliyun.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aliyun + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +func patchEnableThinkingFalse(body *jsonutils.JSONDict, stream bool) { + if stream { + return + } + if _, err := body.Get("enable_thinking"); err == nil { + return + } + body.Set("enable_thinking", jsonutils.JSONFalse) +} + +// New returns the Aliyun (DashScope compatible-mode) provider adapter. +func New() providerapi.Provider { + return openai.NewCompat("aliyun", patchEnableThinkingFalse) +} diff --git a/pkg/aiproxy/providers/aliyun/doc.go b/pkg/aiproxy/providers/aliyun/doc.go new file mode 100644 index 0000000000..422b24dc6b --- /dev/null +++ b/pkg/aiproxy/providers/aliyun/doc.go @@ -0,0 +1 @@ +package aliyun // import "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" diff --git a/pkg/aiproxy/providers/anthropic/anthropic.go b/pkg/aiproxy/providers/anthropic/anthropic.go new file mode 100644 index 0000000000..78656fd406 --- /dev/null +++ b/pkg/aiproxy/providers/anthropic/anthropic.go @@ -0,0 +1,285 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package anthropic + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +const apiVersion = "2023-06-01" + +type provider struct{} + +// New returns the Anthropic Messages API provider adapter. +func New() providerapi.Provider { + return &provider{} +} + +func (p *provider) Key() string { + return "anthropic" +} + +func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + msgs, system, err := openai.ParseMessages(body) + if err != nil { + return nil, err + } + anthropicMsgs, err := openai.MessagesToAnthropic(msgs) + if err != nil { + return nil, err + } + maxTokens := 4096 + if v, ok := openai.IntParam(body, "max_tokens", "max_completion_tokens"); ok { + maxTokens = v + } + reqBody := map[string]interface{}{ + "model": ctx.UpstreamModel, + "max_tokens": maxTokens, + "messages": anthropicMsgs, + } + if system != "" { + reqBody["system"] = system + } + if stream { + reqBody["stream"] = true + } + if v, ok := openai.FloatParam(body, "temperature"); ok { + reqBody["temperature"] = v + } + if v, ok := openai.FloatParam(body, "top_p"); ok { + reqBody["top_p"] = v + } + if tools, toolChoice, err := openai.ExtractTools(body); err != nil { + return nil, err + } else if len(tools) > 0 { + reqBody["tools"] = openai.ToolsToAnthropic(tools) + if tc := openai.ToolChoiceToAnthropic(toolChoice); tc != nil { + reqBody["tool_choice"] = tc + } + } + raw, err := openai.MarshalJSON(reqBody) + if err != nil { + return nil, err + } + base := strings.TrimSpace(ctx.BaseURL) + if base == "" { + base = "https://api.anthropic.com" + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: openai.JoinURL(base, "/v1/messages"), + Headers: map[string]string{ + "x-api-key": strings.TrimSpace(ctx.APIKey), + "anthropic-version": apiVersion, + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func (p *provider) NormalizeResponse(body []byte) ([]byte, error) { + var resp struct { + ID string `json:"id"` + Model string `json:"model"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input map[string]interface{} `json:"input"` + } `json:"content"` + StopReason string `json:"stop_reason"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return body, nil + } + blocks := make([]openai.AnthropicBlock, len(resp.Content)) + for i, c := range resp.Content { + blocks[i] = openai.AnthropicBlock{ + Type: c.Type, + Text: c.Text, + ID: c.ID, + Name: c.Name, + Input: c.Input, + } + } + msg := openai.AnthropicBlocksToAssistant(blocks) + out, err := openai.MarshalJSON(openai.NewChatCompletionWithTools( + resp.Model, + resp.ID, + msg, + resp.StopReason, + resp.Usage.InputTokens, + resp.Usage.OutputTokens, + )) + if err != nil { + return nil, err + } + return out, nil +} + +func (p *provider) OpenAIStreamPassthrough() bool { + return false +} + +func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) { + if state == nil || len(payload) == 0 { + return nil, nil + } + var wrap struct { + Type string `json:"type"` + } + if err := json.Unmarshal(payload, &wrap); err != nil { + return nil, nil + } + switch wrap.Type { + case "message_start": + var start struct { + Message struct { + ID string `json:"id"` + Model string `json:"model"` + } `json:"message"` + } + if err := json.Unmarshal(payload, &start); err == nil { + if start.Message.ID != "" { + state.ResponseID = start.Message.ID + } + if start.Message.Model != "" { + state.Model = start.Message.Model + } + } + return nil, nil + case "content_block_start": + var start struct { + ContentBlock struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"content_block"` + } + if err := json.Unmarshal(payload, &start); err != nil { + return nil, nil + } + if start.ContentBlock.Type != "tool_use" { + return nil, nil + } + state.InToolBlock = true + state.ToolID = start.ContentBlock.ID + state.ToolName = start.ContentBlock.Name + state.ToolArgsPending = "" + chunk, err := openai.MarshalJSON(openai.NewStreamChunkToolDelta( + state.Model, state.ResponseID, state.ToolIndex, + openai.ToolCall{ + ID: state.ToolID, + Type: "function", + Function: openai.ToolFunction{ + Name: state.ToolName, + }, + }, "", + )) + if err != nil { + return nil, err + } + return []providerapi.StreamChunk{{Data: chunk}}, nil + case "content_block_delta": + var delta struct { + Delta struct { + Type string `json:"type"` + Text string `json:"text"` + PartialJSON string `json:"partial_json"` + } `json:"delta"` + } + if err := json.Unmarshal(payload, &delta); err != nil { + return nil, nil + } + if delta.Delta.Type == "input_json_delta" && delta.Delta.PartialJSON != "" { + state.ToolArgsPending += delta.Delta.PartialJSON + chunk, err := openai.MarshalJSON(openai.NewStreamChunkToolDelta( + state.Model, state.ResponseID, state.ToolIndex, + openai.ToolCall{ + Function: openai.ToolFunction{ + Arguments: delta.Delta.PartialJSON, + }, + }, "", + )) + if err != nil { + return nil, err + } + return []providerapi.StreamChunk{{Data: chunk}}, nil + } + if delta.Delta.Text == "" { + return nil, nil + } + chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, delta.Delta.Text, "")) + if err != nil { + return nil, err + } + return []providerapi.StreamChunk{{Data: chunk}}, nil + case "content_block_stop": + if state.InToolBlock { + state.InToolBlock = false + state.ToolIndex++ + state.ToolArgsPending = "" + } + return nil, nil + case "message_delta": + var end struct { + Delta struct { + StopReason string `json:"stop_reason"` + } `json:"delta"` + } + if err := json.Unmarshal(payload, &end); err != nil { + return nil, nil + } + chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, "", end.Delta.StopReason)) + if err != nil { + return nil, err + } + return []providerapi.StreamChunk{{Data: chunk}}, nil + default: + return nil, nil + } +} + +func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + return nil, fmt.Errorf("provider %q does not support embeddings", p.Key()) +} + +func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + return nil, fmt.Errorf("provider %q does not support embeddings", p.Key()) +} + +func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + return nil, fmt.Errorf("provider %q does not support images/generations", p.Key()) +} + +func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) { + return nil, fmt.Errorf("provider %q does not support images/generations", p.Key()) +} diff --git a/pkg/aiproxy/providers/anthropic/doc.go b/pkg/aiproxy/providers/anthropic/doc.go new file mode 100644 index 0000000000..e667137fc9 --- /dev/null +++ b/pkg/aiproxy/providers/anthropic/doc.go @@ -0,0 +1 @@ +package anthropic // import "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" diff --git a/pkg/aiproxy/providers/azure/azure.go b/pkg/aiproxy/providers/azure/azure.go new file mode 100644 index 0000000000..6f7ac0a21d --- /dev/null +++ b/pkg/aiproxy/providers/azure/azure.go @@ -0,0 +1,117 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "fmt" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type provider struct{} + +// New returns the Azure OpenAI provider adapter. +func New() providerapi.Provider { + return &provider{} +} + +func (p *provider) Key() string { + return "azure" +} + +func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + return p.buildRequest(ctx, body, "chat/completions") +} + +func (p *provider) NormalizeResponse(body []byte) ([]byte, error) { + return body, nil +} + +func (p *provider) OpenAIStreamPassthrough() bool { + return true +} + +func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) { + return nil, nil +} + +func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + return p.buildRequest(ctx, body, "embeddings") +} + +func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + return body, nil +} + +func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + return p.buildRequest(ctx, body, "images/generations") +} + +func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) { + return body, nil +} + +func (p *provider) buildRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, action string) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + dup := openai.CloneBodyWithModel(body, ctx.UpstreamModel) + url, err := deploymentURL(ctx.BaseURL, ctx.UpstreamModel, body, action) + if err != nil { + return nil, err + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: url, + Headers: map[string]string{ + "api-key": strings.TrimSpace(ctx.APIKey), + "Content-Type": "application/json", + }, + Body: []byte(dup.String()), + }, nil +} + +func deploymentURL(baseURL, deployment string, body *jsonutils.JSONDict, action string) (string, error) { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if base == "" { + return "", fmt.Errorf("azure provider requires base_url (resource endpoint)") + } + url := base + suffix := "/" + action + if strings.Contains(url, "/"+action) { + return url, nil + } + if strings.HasSuffix(url, "/v1") { + return url + suffix, nil + } + if strings.Contains(url, "/openai/deployments/") { + return url + suffix, nil + } + url = openai.JoinURL(url, fmt.Sprintf("openai/deployments/%s/%s", deployment, action)) + if body != nil { + if v, err := body.Get("api-version"); err == nil { + ver, _ := v.GetString() + if ver != "" { + url = url + "?api-version=" + ver + } + } + } + return url, nil +} diff --git a/pkg/aiproxy/providers/azure/doc.go b/pkg/aiproxy/providers/azure/doc.go new file mode 100644 index 0000000000..a7a88591b3 --- /dev/null +++ b/pkg/aiproxy/providers/azure/doc.go @@ -0,0 +1 @@ +package azure // import "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" diff --git a/pkg/aiproxy/providers/baidu/baidu.go b/pkg/aiproxy/providers/baidu/baidu.go new file mode 100644 index 0000000000..3c97f71447 --- /dev/null +++ b/pkg/aiproxy/providers/baidu/baidu.go @@ -0,0 +1,103 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baidu + +import ( + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type provider struct { + v2 *openai.Compat +} + +// New returns the Baidu Wenxin / Qianfan provider adapter. +func New() providerapi.Provider { + return &provider{v2: openai.NewCompat("baidu")} +} + +func (p *provider) Key() string { + return "baidu" +} + +func (p *provider) useV2(ctx *providerapi.ChatContext) bool { + if ctx == nil { + return true + } + return useQianfanV2(ctx.BaseURL) +} + +func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if p.useV2(ctx) { + return p.v2.BuildUpstreamRequest(ctx, body, stream) + } + return buildWenxinV1ChatRequest(ctx, body, stream) +} + +func (p *provider) NormalizeResponse(body []byte) ([]byte, error) { + if p.v2 != nil { + // Try wenxin v1 only when response looks non-OpenAI. + if out, err := normalizeWenxinV1ChatResponse(body); err != nil { + return nil, err + } else if string(out) != string(body) { + return out, nil + } + } + return body, nil +} + +func (p *provider) OpenAIStreamPassthrough() bool { + return false +} + +func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) { + return convertWenxinV1StreamEvent(payload, state) +} + +func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if p.useV2(ctx) { + return p.v2.BuildEmbeddingsRequest(ctx, body) + } + return buildWenxinV1EmbeddingsRequest(ctx, body) +} + +func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + if out, err := normalizeWenxinV1EmbeddingsResponse(body); err != nil { + return nil, err + } else if len(out) > 0 && string(out) != string(body) { + return out, nil + } + return body, nil +} + +func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if p.useV2(ctx) { + return p.v2.BuildImagesGenerationsRequest(ctx, body) + } + return nil, fmt.Errorf("provider %q wenxin v1 does not support images/generations; use qianfan v2 base_url", p.Key()) +} + +func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) { + return body, nil +} + +// OpenAIStreamPassthroughForContext reports whether upstream SSE is already OpenAI-compatible. +func (p *provider) OpenAIStreamPassthroughForContext(ctx *providerapi.ChatContext) bool { + return p.useV2(ctx) +} diff --git a/pkg/aiproxy/providers/baidu/baidu_test.go b/pkg/aiproxy/providers/baidu/baidu_test.go new file mode 100644 index 0000000000..1bb4469d1d --- /dev/null +++ b/pkg/aiproxy/providers/baidu/baidu_test.go @@ -0,0 +1,116 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baidu + +import ( + "encoding/json" + "strings" + "testing" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +func TestQianfanV2ChatBuild(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("ernie-3.5-8k"), "model") + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("你好")) + body.Add(jsonutils.NewArray(userMsg), "messages") + + p := New() + req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{ + BaseURL: "https://qianfan.baidubce.com/v2", + APIKey: "bce-v3/ALTAK-test", + UpstreamModel: "ernie-3.5-8k", + }, body, false) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://qianfan.baidubce.com/v2/chat/completions" { + t.Fatalf("unexpected url: %s", req.URL) + } + if req.Headers["Authorization"] != "Bearer bce-v3/ALTAK-test" { + t.Fatalf("missing bearer auth: %#v", req.Headers) + } +} + +func TestWenxinV1ChatBuild(t *testing.T) { + body := jsonutils.NewDict() + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("你好")) + body.Add(jsonutils.NewArray(userMsg), "messages") + + p := New() + req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{ + BaseURL: "https://aip.baidubce.com", + APIKey: "test-access-token", + UpstreamModel: "eb-instant", + }, body, false) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(req.URL, "wenxinworkshop/chat/eb-instant") { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestWenxinV1NormalizeResponse(t *testing.T) { + p := New() + out, err := p.NormalizeResponse([]byte(`{ + "id":"as-1", + "result":"你好", + "usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3} + }`)) + if err != nil { + t.Fatal(err) + } + var resp map[string]interface{} + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if resp["object"] != "chat.completion" { + t.Fatalf("unexpected object: %#v", resp["object"]) + } +} + +func TestStreamPassthroughV2(t *testing.T) { + p := New().(providerapi.ContextualStreamPassthrough) + if !p.OpenAIStreamPassthroughForContext(&providerapi.ChatContext{ + BaseURL: "https://qianfan.baidubce.com/v2", + }) { + t.Fatal("qianfan v2 should passthrough SSE") + } + if p.OpenAIStreamPassthroughForContext(&providerapi.ChatContext{ + BaseURL: "https://aip.baidubce.com", + }) { + t.Fatal("wenxin v1 should not passthrough SSE") + } +} + +func TestUseQianfanV2(t *testing.T) { + if !useQianfanV2("") { + t.Fatal("empty base should default to v2") + } + if !useQianfanV2("https://qianfan.baidubce.com/v2") { + t.Fatal("qianfan host should be v2") + } + if useQianfanV2("https://aip.baidubce.com") { + t.Fatal("aip host should be v1") + } +} diff --git a/pkg/aiproxy/providers/baidu/doc.go b/pkg/aiproxy/providers/baidu/doc.go new file mode 100644 index 0000000000..84bda56cd6 --- /dev/null +++ b/pkg/aiproxy/providers/baidu/doc.go @@ -0,0 +1 @@ +package baidu // import "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" diff --git a/pkg/aiproxy/providers/baidu/token.go b/pkg/aiproxy/providers/baidu/token.go new file mode 100644 index 0000000000..54e00ae282 --- /dev/null +++ b/pkg/aiproxy/providers/baidu/token.go @@ -0,0 +1,146 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baidu + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +const ( + defaultTokenURL = "https://aip.baidubce.com/oauth/2.0/token" + tokenSkew = 5 * time.Minute +) + +type cachedToken struct { + value string + expiry time.Time +} + +var tokenCache sync.Map + +// ResolveAccessToken returns a Wenxin access_token. +// apiKey may be a raw access_token, or "APIKey:SecretKey" for OAuth exchange. +func ResolveAccessToken(apiKey string) (string, error) { + apiKey = strings.TrimSpace(apiKey) + if apiKey == "" { + return "", fmt.Errorf("empty baidu api key") + } + if !strings.Contains(apiKey, ":") { + return apiKey, nil + } + parts := strings.SplitN(apiKey, ":", 2) + clientID := strings.TrimSpace(parts[0]) + clientSecret := strings.TrimSpace(parts[1]) + if clientID == "" || clientSecret == "" { + return "", fmt.Errorf("invalid baidu api key format, want access_token or APIKey:SecretKey") + } + cacheKey := clientID + ":" + clientSecret + if v, ok := tokenCache.Load(cacheKey); ok { + entry := v.(cachedToken) + if time.Now().Before(entry.expiry.Add(-tokenSkew)) { + return entry.value, nil + } + } + token, expiresIn, err := fetchAccessToken(clientID, clientSecret) + if err != nil { + return "", err + } + if expiresIn <= 0 { + expiresIn = 30 * 24 * time.Hour + } + tokenCache.Store(cacheKey, cachedToken{ + value: token, + expiry: time.Now().Add(expiresIn), + }) + return token, nil +} + +func fetchAccessToken(clientID, clientSecret string) (string, time.Duration, error) { + q := url.Values{} + q.Set("grant_type", "client_credentials") + q.Set("client_id", clientID) + q.Set("client_secret", clientSecret) + req, err := http.NewRequest(http.MethodPost, defaultTokenURL+"?"+q.Encode(), nil) + if err != nil { + return "", 0, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", 0, err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", 0, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", 0, fmt.Errorf("baidu oauth HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + var wrap struct { + AccessToken string `json:"access_token"` + ExpiresIn int64 `json:"expires_in"` + Error string `json:"error"` + ErrorDesc string `json:"error_description"` + } + if err := json.Unmarshal(body, &wrap); err != nil { + return "", 0, err + } + if wrap.AccessToken == "" { + msg := wrap.ErrorDesc + if msg == "" { + msg = wrap.Error + } + if msg == "" { + msg = string(body) + } + return "", 0, fmt.Errorf("baidu oauth failed: %s", msg) + } + return wrap.AccessToken, time.Duration(wrap.ExpiresIn) * time.Second, nil +} + +func wenxinBaseURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if base == "" { + return "https://aip.baidubce.com" + } + return base +} + +func wenxinChatURL(baseURL, model, accessToken string) string { + path := fmt.Sprintf("/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/%s", strings.TrimSpace(model)) + u := wenxinBaseURL(baseURL) + path + return u + "?access_token=" + url.QueryEscape(accessToken) +} + +func wenxinEmbeddingsURL(baseURL, model, accessToken string) string { + path := fmt.Sprintf("/rpc/2.0/ai_custom/v1/wenxinworkshop/embeddings/%s", strings.TrimSpace(model)) + u := wenxinBaseURL(baseURL) + path + return u + "?access_token=" + url.QueryEscape(accessToken) +} + +func useQianfanV2(baseURL string) bool { + base := strings.ToLower(strings.TrimSpace(baseURL)) + if base == "" { + return true + } + return strings.Contains(base, "qianfan.baidubce.com") +} diff --git a/pkg/aiproxy/providers/baidu/wenxin_v1.go b/pkg/aiproxy/providers/baidu/wenxin_v1.go new file mode 100644 index 0000000000..c2ffda9af3 --- /dev/null +++ b/pkg/aiproxy/providers/baidu/wenxin_v1.go @@ -0,0 +1,235 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baidu + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +func buildWenxinV1ChatRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + token, err := ResolveAccessToken(ctx.APIKey) + if err != nil { + return nil, err + } + msgs, _, err := openai.ParseMessages(body) + if err != nil { + return nil, err + } + wenxinMsgs := make([]map[string]interface{}, 0, len(msgs)) + for _, m := range msgs { + role := strings.ToLower(strings.TrimSpace(m.Role)) + if role == "tool" { + role = "user" + } + text := openai.MessageTextContent(m.Content) + if text == "" { + continue + } + wenxinMsgs = append(wenxinMsgs, map[string]interface{}{ + "role": role, + "content": text, + }) + } + if len(wenxinMsgs) == 0 { + return nil, fmt.Errorf("no convertible messages for wenxin") + } + reqBody := map[string]interface{}{ + "messages": wenxinMsgs, + "stream": stream, + } + if v, ok := openai.IntParam(body, "max_tokens", "max_completion_tokens"); ok { + reqBody["max_output_tokens"] = v + } + if v, ok := openai.FloatParam(body, "temperature"); ok { + reqBody["temperature"] = v + } + if v, ok := openai.FloatParam(body, "top_p"); ok { + reqBody["top_p"] = v + } + raw, err := openai.MarshalJSON(reqBody) + if err != nil { + return nil, err + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: wenxinChatURL(ctx.BaseURL, ctx.UpstreamModel, token), + Headers: map[string]string{ + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func normalizeWenxinV1ChatResponse(body []byte) ([]byte, error) { + var probe struct { + Choices json.RawMessage `json:"choices"` + Result *string `json:"result"` + } + if err := json.Unmarshal(body, &probe); err != nil { + return body, nil + } + if len(probe.Choices) > 0 { + return body, nil + } + if probe.Result == nil { + return body, nil + } + var resp struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Result string `json:"result"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return body, nil + } + if resp.ErrorCode != 0 || resp.ErrorMsg != "" { + return nil, fmt.Errorf("wenxin error %d: %s", resp.ErrorCode, resp.ErrorMsg) + } + if resp.Result == "" && resp.ID == "" { + return body, nil + } + out, err := openai.MarshalJSON(openai.NewChatCompletion( + "", + resp.ID, + resp.Result, + "stop", + resp.Usage.PromptTokens, + resp.Usage.CompletionTokens, + )) + if err != nil { + return nil, err + } + return out, nil +} + +func convertWenxinV1StreamEvent(payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) { + if state == nil || len(payload) == 0 { + return nil, nil + } + var chunk struct { + ID string `json:"id"` + Result string `json:"result"` + IsEnd bool `json:"is_end"` + IsTrunc bool `json:"is_truncated"` + } + if err := json.Unmarshal(payload, &chunk); err != nil { + return nil, nil + } + if chunk.ID != "" { + state.ResponseID = chunk.ID + } + finish := "" + if chunk.IsEnd { + finish = "stop" + } + if chunk.Result == "" && finish == "" { + return nil, nil + } + data, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, chunk.Result, finish)) + if err != nil { + return nil, err + } + return []providerapi.StreamChunk{{Data: data}}, nil +} + +func buildWenxinV1EmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + token, err := ResolveAccessToken(ctx.APIKey) + if err != nil { + return nil, err + } + texts, rawInput, isText, err := openai.ParseEmbeddingInput(body) + if err != nil { + return nil, err + } + var reqBody map[string]interface{} + if isText { + reqBody = map[string]interface{}{"input": texts} + } else { + if err := json.Unmarshal(rawInput, &reqBody); err != nil { + return nil, fmt.Errorf("invalid embeddings input") + } + } + raw, err := openai.MarshalJSON(reqBody) + if err != nil { + return nil, err + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: wenxinEmbeddingsURL(ctx.BaseURL, ctx.UpstreamModel, token), + Headers: map[string]string{ + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func normalizeWenxinV1EmbeddingsResponse(body []byte) ([]byte, error) { + var probe struct { + Object string `json:"object"` + Choices json.RawMessage `json:"choices"` + } + if err := json.Unmarshal(body, &probe); err == nil && (probe.Object == "list" || len(probe.Choices) > 0) { + return body, nil + } + var resp struct { + Data []struct { + Embedding []float64 `json:"embedding"` + Index int `json:"index"` + } `json:"data"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return body, nil + } + if resp.ErrorCode != 0 || resp.ErrorMsg != "" { + return nil, fmt.Errorf("wenxin embeddings error %d: %s", resp.ErrorCode, resp.ErrorMsg) + } + vectors := make([][]float64, len(resp.Data)) + for i := range resp.Data { + vectors[i] = resp.Data[i].Embedding + } + promptTokens := resp.Usage.PromptTokens + if promptTokens == 0 { + promptTokens = resp.Usage.TotalTokens + } + return openai.NewEmbeddingsResponse("", vectors, promptTokens) +} diff --git a/pkg/aiproxy/providers/bridge.go b/pkg/aiproxy/providers/bridge.go new file mode 100644 index 0000000000..c25c63ce34 --- /dev/null +++ b/pkg/aiproxy/providers/bridge.go @@ -0,0 +1,33 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/upstream" +) + +// ToUpstreamRequest maps a provider HTTPRequest to the shared upstream client shape. +func ToUpstreamRequest(r *HTTPRequest, fallbackAPIKey string) *upstream.Request { + if r == nil { + return nil + } + return &upstream.Request{ + BaseURL: "", + URL: r.URL, + APIKey: fallbackAPIKey, + Headers: r.Headers, + Body: r.Body, + } +} diff --git a/pkg/aiproxy/providers/cohere/cohere.go b/pkg/aiproxy/providers/cohere/cohere.go new file mode 100644 index 0000000000..e62518ea29 --- /dev/null +++ b/pkg/aiproxy/providers/cohere/cohere.go @@ -0,0 +1,94 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cohere + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type provider struct { + *openai.Compat +} + +// New returns the Cohere provider adapter (OpenAI-compatible chat, native embeddings). +func New() providerapi.Provider { + return &provider{Compat: openai.NewCompat("cohere")} +} + +func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + texts, _, isText, err := openai.ParseEmbeddingInput(body) + if err != nil { + return nil, err + } + if !isText { + return nil, fmt.Errorf("cohere embeddings requires string or string[] input") + } + reqBody := map[string]interface{}{ + "model": ctx.UpstreamModel, + "texts": texts, + "input_type": embeddingInputType(body), + "embedding_types": []string{"float"}, + } + raw, err := openai.MarshalJSON(reqBody) + if err != nil { + return nil, err + } + base := strings.TrimSpace(ctx.BaseURL) + if base == "" { + base = "https://api.cohere.ai" + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: openai.JoinURL(base, "/v2/embed"), + Headers: map[string]string{ + "Authorization": "Bearer " + strings.TrimSpace(ctx.APIKey), + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func embeddingInputType(body *jsonutils.JSONDict) string { + if body == nil { + return "search_document" + } + if v, err := body.GetString("input_type"); err == nil && strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + return "search_document" +} + +func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + var resp struct { + Embeddings struct { + Float [][]float64 `json:"float"` + } `json:"embeddings"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return body, nil + } + return openai.NewEmbeddingsResponse("", resp.Embeddings.Float, 0) +} diff --git a/pkg/aiproxy/providers/cohere/doc.go b/pkg/aiproxy/providers/cohere/doc.go new file mode 100644 index 0000000000..a9412ebb8e --- /dev/null +++ b/pkg/aiproxy/providers/cohere/doc.go @@ -0,0 +1 @@ +package cohere // import "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" diff --git a/pkg/aiproxy/providers/completions.go b/pkg/aiproxy/providers/completions.go new file mode 100644 index 0000000000..cbec953bc0 --- /dev/null +++ b/pkg/aiproxy/providers/completions.go @@ -0,0 +1,28 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "fmt" +) + +// GetCompletions returns the legacy completions adapter for providerKey. +func GetCompletions(providerKey string) (CompletionsProvider, error) { + p := Get(providerKey) + if cp, ok := p.(CompletionsProvider); ok { + return cp, nil + } + return nil, fmt.Errorf("provider %q does not support /v1/completions", providerKey) +} diff --git a/pkg/aiproxy/providers/doc.go b/pkg/aiproxy/providers/doc.go new file mode 100644 index 0000000000..450383b6b3 --- /dev/null +++ b/pkg/aiproxy/providers/doc.go @@ -0,0 +1,5 @@ +// Package providers converts OpenAI-compatible API requests to upstream AI provider APIs. +// +// Vendor-specific adapters live in subdirectories (openai/, anthropic/, gemini/, ...). +// Shared types are defined in providerapi; this package exposes registry lookup helpers. +package providers // import "yunion.io/x/onecloud/pkg/aiproxy/providers" diff --git a/pkg/aiproxy/providers/embeddings.go b/pkg/aiproxy/providers/embeddings.go new file mode 100644 index 0000000000..93c0bd2727 --- /dev/null +++ b/pkg/aiproxy/providers/embeddings.go @@ -0,0 +1,30 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +var defaultEmbeddings EmbeddingsProvider = openai.DefaultEmbeddingCompat() + +// GetEmbeddings returns the embeddings adapter for providerKey, or OpenAI-compatible passthrough. +func GetEmbeddings(providerKey string) EmbeddingsProvider { + p := Get(providerKey) + if ep, ok := p.(EmbeddingsProvider); ok { + return ep + } + return defaultEmbeddings +} diff --git a/pkg/aiproxy/providers/embeddings_test.go b/pkg/aiproxy/providers/embeddings_test.go new file mode 100644 index 0000000000..adacb9ba6d --- /dev/null +++ b/pkg/aiproxy/providers/embeddings_test.go @@ -0,0 +1,121 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "encoding/json" + "testing" + + "yunion.io/x/jsonutils" +) + +func TestOpenAIEmbeddingsCompatBuild(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("text-embedding-3-small"), "model") + body.Add(jsonutils.NewString("hello"), "input") + + p := GetEmbeddings("openai") + req, err := p.BuildEmbeddingsRequest(&ChatContext{ + BaseURL: "https://api.openai.com", + APIKey: "sk-test", + UpstreamModel: "text-embedding-3-small", + }, body) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.openai.com/v1/embeddings" { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestGeminiEmbeddingsBuildSingle(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("text-embedding-004"), "model") + body.Add(jsonutils.NewString("hello world"), "input") + + p := GetEmbeddings("gemini") + req, err := p.BuildEmbeddingsRequest(&ChatContext{ + BaseURL: "https://generativelanguage.googleapis.com/v1beta", + APIKey: "key", + UpstreamModel: "text-embedding-004", + }, body) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent" { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestGeminiEmbeddingsNormalize(t *testing.T) { + p := GetEmbeddings("gemini") + out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embedding":{"values":[0.1,0.2]}}`)) + if err != nil { + t.Fatal(err) + } + var resp map[string]interface{} + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if resp["object"] != "list" { + t.Fatalf("unexpected object: %#v", resp["object"]) + } +} + +func TestCohereEmbeddingsBuild(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("embed-english-v3.0"), "model") + body.Add(jsonutils.NewArray(jsonutils.NewString("a"), jsonutils.NewString("b")), "input") + + p := GetEmbeddings("cohere") + req, err := p.BuildEmbeddingsRequest(&ChatContext{ + BaseURL: "https://api.cohere.ai", + APIKey: "key", + UpstreamModel: "embed-english-v3.0", + }, body) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.cohere.ai/v2/embed" { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestCohereEmbeddingsNormalize(t *testing.T) { + p := GetEmbeddings("cohere") + out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embeddings":{"float":[[0.1],[0.2]]}}`)) + if err != nil { + t.Fatal(err) + } + var resp struct { + Data []struct { + Index int `json:"index"` + } `json:"data"` + } + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data) != 2 { + t.Fatalf("expected 2 embeddings, got %d", len(resp.Data)) + } +} + +func TestAnthropicEmbeddingsUnsupported(t *testing.T) { + p := GetEmbeddings("anthropic") + _, err := p.BuildEmbeddingsRequest(&ChatContext{ProviderKey: "anthropic"}, jsonutils.NewDict()) + if err == nil { + t.Fatal("expected error for anthropic embeddings") + } +} diff --git a/pkg/aiproxy/providers/gemini/doc.go b/pkg/aiproxy/providers/gemini/doc.go new file mode 100644 index 0000000000..04ceb08b4a --- /dev/null +++ b/pkg/aiproxy/providers/gemini/doc.go @@ -0,0 +1 @@ +package gemini // import "yunion.io/x/onecloud/pkg/aiproxy/providers/gemini" diff --git a/pkg/aiproxy/providers/gemini/gemini.go b/pkg/aiproxy/providers/gemini/gemini.go new file mode 100644 index 0000000000..06acdc6041 --- /dev/null +++ b/pkg/aiproxy/providers/gemini/gemini.go @@ -0,0 +1,367 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gemini + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/google/uuid" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type provider struct{} + +// New returns the Google Gemini provider adapter. +func New() providerapi.Provider { + return &provider{} +} + +func (p *provider) Key() string { + return "gemini" +} + +func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + msgs, system, err := openai.ParseMessages(body) + if err != nil { + return nil, err + } + contents, err := openai.MessagesToGemini(msgs) + if err != nil { + return nil, err + } + genConfig := map[string]interface{}{} + if v, ok := openai.IntParam(body, "max_tokens", "max_completion_tokens"); ok { + genConfig["maxOutputTokens"] = v + } + if v, ok := openai.FloatParam(body, "temperature"); ok { + genConfig["temperature"] = v + } + if v, ok := openai.FloatParam(body, "top_p"); ok { + genConfig["topP"] = v + } + reqBody := map[string]interface{}{ + "contents": contents, + } + if len(genConfig) > 0 { + reqBody["generationConfig"] = genConfig + } + if system != "" { + reqBody["systemInstruction"] = map[string]interface{}{ + "parts": []map[string]interface{}{ + {"text": system}, + }, + } + } + if tools, _, err := openai.ExtractTools(body); err != nil { + return nil, err + } else if gemTools := openai.ToolsToGemini(tools); len(gemTools) > 0 { + reqBody["tools"] = gemTools + } + raw, err := openai.MarshalJSON(reqBody) + if err != nil { + return nil, err + } + base := strings.TrimRight(strings.TrimSpace(ctx.BaseURL), "/") + if base == "" { + base = "https://generativelanguage.googleapis.com/v1beta" + } + action := "generateContent" + if stream { + action = "streamGenerateContent" + } + modelPath := fmt.Sprintf("/models/%s:%s", ctx.UpstreamModel, action) + url := openai.JoinURL(base, modelPath) + if stream { + url += "?alt=sse" + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: url, + Headers: map[string]string{ + "x-goog-api-key": strings.TrimSpace(ctx.APIKey), + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func (p *provider) NormalizeResponse(body []byte) ([]byte, error) { + var resp struct { + Candidates []struct { + Content struct { + Parts []openai.GeminiPart `json:"parts"` + } `json:"content"` + FinishReason string `json:"finishReason"` + } `json:"candidates"` + UsageMetadata struct { + PromptTokenCount int `json:"promptTokenCount"` + CandidatesTokenCount int `json:"candidatesTokenCount"` + } `json:"usageMetadata"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return body, nil + } + finish := "stop" + msg := openai.AssistantMessage{} + if len(resp.Candidates) > 0 { + msg = openai.GeminiPartsToAssistant(resp.Candidates[0].Content.Parts) + if resp.Candidates[0].FinishReason != "" { + finish = resp.Candidates[0].FinishReason + } + } + out, err := openai.MarshalJSON(openai.NewChatCompletionWithTools( + "", + "gemini-"+uuid.New().String(), + msg, + finish, + resp.UsageMetadata.PromptTokenCount, + resp.UsageMetadata.CandidatesTokenCount, + )) + if err != nil { + return nil, err + } + return out, nil +} + +func (p *provider) OpenAIStreamPassthrough() bool { + return false +} + +func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) { + if state == nil || len(payload) == 0 { + return nil, nil + } + var resp struct { + Candidates []struct { + Content struct { + Parts []openai.GeminiPart `json:"parts"` + } `json:"content"` + FinishReason string `json:"finishReason"` + } `json:"candidates"` + } + if err := json.Unmarshal(payload, &resp); err != nil { + return nil, nil + } + if len(resp.Candidates) == 0 { + return nil, nil + } + c := resp.Candidates[0] + if state.ResponseID == "" { + state.ResponseID = "gemini-" + uuid.New().String() + } + var chunks []providerapi.StreamChunk + msg := openai.GeminiPartsToAssistant(c.Content.Parts) + if msg.Content != "" { + chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, msg.Content, "")) + if err != nil { + return nil, err + } + chunks = append(chunks, providerapi.StreamChunk{Data: chunk}) + } + for _, tc := range msg.ToolCalls { + chunk, err := openai.MarshalJSON(openai.NewStreamChunkToolDelta( + state.Model, state.ResponseID, state.ToolIndex, tc, "", + )) + if err != nil { + return nil, err + } + chunks = append(chunks, providerapi.StreamChunk{Data: chunk}) + state.ToolIndex++ + } + if c.FinishReason != "" { + chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, "", c.FinishReason)) + if err != nil { + return nil, err + } + chunks = append(chunks, providerapi.StreamChunk{Data: chunk}) + } + if len(chunks) == 0 { + return nil, nil + } + return chunks, nil +} + +func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + texts, _, isText, err := openai.ParseEmbeddingInput(body) + if err != nil { + return nil, err + } + if !isText { + return openai.DefaultEmbeddingCompat().BuildEmbeddingsRequest(ctx, body) + } + base := strings.TrimRight(strings.TrimSpace(ctx.BaseURL), "/") + if base == "" { + base = "https://generativelanguage.googleapis.com/v1beta" + } + modelRef := modelRef(ctx.UpstreamModel) + var raw []byte + var action string + if len(texts) == 1 { + action = "embedContent" + reqBody := map[string]interface{}{ + "content": map[string]interface{}{ + "parts": []map[string]interface{}{{"text": texts[0]}}, + }, + } + raw, err = openai.MarshalJSON(reqBody) + } else { + action = "batchEmbedContents" + requests := make([]map[string]interface{}, len(texts)) + for i, text := range texts { + requests[i] = map[string]interface{}{ + "model": modelRef, + "content": map[string]interface{}{ + "parts": []map[string]interface{}{{"text": text}}, + }, + } + } + raw, err = openai.MarshalJSON(map[string]interface{}{"requests": requests}) + } + if err != nil { + return nil, err + } + url := openai.JoinURL(base, fmt.Sprintf("/models/%s:%s", ctx.UpstreamModel, action)) + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: url, + Headers: map[string]string{ + "x-goog-api-key": strings.TrimSpace(ctx.APIKey), + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + var single struct { + Embedding struct { + Values []float64 `json:"values"` + } `json:"embedding"` + } + if err := json.Unmarshal(body, &single); err == nil && len(single.Embedding.Values) > 0 { + return openai.NewEmbeddingsResponse("", [][]float64{single.Embedding.Values}, 0) + } + var batch struct { + Embeddings []struct { + Values []float64 `json:"values"` + } `json:"embeddings"` + } + if err := json.Unmarshal(body, &batch); err != nil { + return body, nil + } + vectors := make([][]float64, len(batch.Embeddings)) + for i := range batch.Embeddings { + vectors[i] = batch.Embeddings[i].Values + } + return openai.NewEmbeddingsResponse("", vectors, 0) +} + +func modelRef(model string) string { + model = strings.TrimSpace(model) + if strings.HasPrefix(model, "models/") { + return model + } + return "models/" + model +} + +func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + prompt, err := openai.ParseImagePrompt(body) + if err != nil { + return nil, err + } + base := strings.TrimRight(strings.TrimSpace(ctx.BaseURL), "/") + if base == "" { + base = "https://generativelanguage.googleapis.com/v1beta" + } + reqBody := map[string]interface{}{ + "instances": []map[string]interface{}{ + {"prompt": prompt}, + }, + "parameters": map[string]interface{}{ + "sampleCount": openai.ImageCount(body), + "aspectRatio": openai.SizeToAspectRatio(openai.ImageSize(body)), + }, + } + raw, err := openai.MarshalJSON(reqBody) + if err != nil { + return nil, err + } + url := openai.JoinURL(base, fmt.Sprintf("/models/%s:predict", ctx.UpstreamModel)) + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: url, + Headers: map[string]string{ + "x-goog-api-key": strings.TrimSpace(ctx.APIKey), + "Content-Type": "application/json", + }, + Body: raw, + }, nil +} + +func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) { + var predict struct { + Predictions []struct { + BytesBase64Encoded string `json:"bytesBase64Encoded"` + } `json:"predictions"` + } + if err := json.Unmarshal(body, &predict); err == nil && len(predict.Predictions) > 0 { + items := make([]openai.ImageItem, 0, len(predict.Predictions)) + for _, pred := range predict.Predictions { + if pred.BytesBase64Encoded == "" { + continue + } + items = append(items, openai.ImageItem{B64: pred.BytesBase64Encoded}) + } + if len(items) > 0 { + return openai.NewImagesGenerationsResponse(items) + } + } + var generated struct { + GeneratedImages []struct { + Image struct { + ImageBytes string `json:"imageBytes"` + } `json:"image"` + } `json:"generatedImages"` + } + if err := json.Unmarshal(body, &generated); err == nil && len(generated.GeneratedImages) > 0 { + items := make([]openai.ImageItem, 0, len(generated.GeneratedImages)) + for _, img := range generated.GeneratedImages { + if img.Image.ImageBytes == "" { + continue + } + items = append(items, openai.ImageItem{B64: img.Image.ImageBytes}) + } + if len(items) > 0 { + return openai.NewImagesGenerationsResponse(items) + } + } + return body, nil +} diff --git a/pkg/aiproxy/providers/images.go b/pkg/aiproxy/providers/images.go new file mode 100644 index 0000000000..90deea2fa7 --- /dev/null +++ b/pkg/aiproxy/providers/images.go @@ -0,0 +1,30 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +var defaultImages ImagesProvider = openai.DefaultImagesCompat() + +// GetImages returns the images adapter for providerKey, or OpenAI-compatible passthrough. +func GetImages(providerKey string) ImagesProvider { + p := Get(providerKey) + if ip, ok := p.(ImagesProvider); ok { + return ip + } + return defaultImages +} diff --git a/pkg/aiproxy/providers/images_test.go b/pkg/aiproxy/providers/images_test.go new file mode 100644 index 0000000000..df69c6f613 --- /dev/null +++ b/pkg/aiproxy/providers/images_test.go @@ -0,0 +1,121 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "encoding/json" + "strings" + "testing" + + "yunion.io/x/jsonutils" +) + +func TestOpenAIImagesCompatBuild(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("dall-e-3"), "model") + body.Add(jsonutils.NewString("a red cat"), "prompt") + + p := GetImages("openai") + req, err := p.BuildImagesGenerationsRequest(&ChatContext{ + BaseURL: "https://api.openai.com", + APIKey: "sk-test", + UpstreamModel: "dall-e-3", + }, body) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.openai.com/v1/images/generations" { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestGeminiImagesBuild(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("imagen-3.0-generate-002"), "model") + body.Add(jsonutils.NewString("sunset over mountains"), "prompt") + body.Add(jsonutils.NewString("1792x1024"), "size") + body.Add(jsonutils.NewInt(2), "n") + + p := GetImages("gemini") + req, err := p.BuildImagesGenerationsRequest(&ChatContext{ + BaseURL: "https://generativelanguage.googleapis.com/v1beta", + APIKey: "key", + UpstreamModel: "imagen-3.0-generate-002", + }, body) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://generativelanguage.googleapis.com/v1beta/models/imagen-3.0-generate-002:predict" { + t.Fatalf("unexpected url: %s", req.URL) + } + var wire struct { + Parameters struct { + SampleCount int `json:"sampleCount"` + AspectRatio string `json:"aspectRatio"` + } `json:"parameters"` + } + if err := json.Unmarshal(req.Body, &wire); err != nil { + t.Fatal(err) + } + if wire.Parameters.SampleCount != 2 || wire.Parameters.AspectRatio != "16:9" { + t.Fatalf("unexpected parameters: %+v", wire.Parameters) + } +} + +func TestGeminiImagesNormalize(t *testing.T) { + p := GetImages("gemini") + out, err := p.NormalizeImagesGenerationsResponse([]byte(`{"predictions":[{"bytesBase64Encoded":"abc123"}]}`)) + if err != nil { + t.Fatal(err) + } + var resp struct { + Data []struct { + B64 string `json:"b64_json"` + } `json:"data"` + } + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data) != 1 || resp.Data[0].B64 != "abc123" { + t.Fatalf("unexpected response: %+v", resp) + } +} + +func TestAzureImagesBuild(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("dall-e-3"), "model") + body.Add(jsonutils.NewString("test"), "prompt") + + p := GetImages("azure") + req, err := p.BuildImagesGenerationsRequest(&ChatContext{ + BaseURL: "https://example.openai.azure.com", + APIKey: "key", + UpstreamModel: "dall-e-3", + }, body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(req.URL, "images/generations") { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestAnthropicImagesUnsupported(t *testing.T) { + p := GetImages("anthropic") + _, err := p.BuildImagesGenerationsRequest(&ChatContext{ProviderKey: "anthropic"}, jsonutils.NewDict()) + if err == nil { + t.Fatal("expected error for anthropic images") + } +} diff --git a/pkg/aiproxy/providers/openai/compat.go b/pkg/aiproxy/providers/openai/compat.go new file mode 100644 index 0000000000..5d97c92069 --- /dev/null +++ b/pkg/aiproxy/providers/openai/compat.go @@ -0,0 +1,102 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +var ( + defaultEmbeddings = NewEmbeddingCompat() + defaultImages = NewImagesCompat() +) + +// Compat forwards OpenAI-shaped JSON to OpenAI-compatible upstreams. +type Compat struct { + ProviderKey string + Patches []PatchFunc +} + +// NewCompat returns an OpenAI-compatible provider for the given catalog provider_key. +func NewCompat(providerKey string, patches ...PatchFunc) *Compat { + return &Compat{ProviderKey: providerKey, Patches: patches} +} + +func (p *Compat) Key() string { + return p.ProviderKey +} + +func (p *Compat) buildBody(body *jsonutils.JSONDict, upstreamModel string, stream bool) *jsonutils.JSONDict { + if len(p.Patches) == 0 { + return CloneBodyWithModel(body, upstreamModel) + } + return PatchBody(CloneBodyWithModel(body, upstreamModel), stream, p.Patches...) +} + +func (p *Compat) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + dup := p.buildBody(body, ctx.UpstreamModel, stream) + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: ChatCompletionsURL(ctx.BaseURL), + Headers: BearerAuthHeaders(ctx.APIKey), + Body: []byte(dup.String()), + }, nil +} + +func (p *Compat) NormalizeResponse(body []byte) ([]byte, error) { + return body, nil +} + +func (p *Compat) OpenAIStreamPassthrough() bool { + return true +} + +func (p *Compat) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) { + return nil, nil +} + +func (p *Compat) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + return defaultEmbeddings.BuildEmbeddingsRequest(ctx, body) +} + +func (p *Compat) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + return defaultEmbeddings.NormalizeEmbeddingsResponse(body) +} + +func (p *Compat) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + return defaultImages.BuildImagesGenerationsRequest(ctx, body) +} + +func (p *Compat) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) { + return defaultImages.NormalizeImagesGenerationsResponse(body) +} + +// DefaultEmbeddingCompat returns the shared OpenAI-compatible embeddings adapter. +func DefaultEmbeddingCompat() providerapi.EmbeddingsProvider { + return defaultEmbeddings +} + +// DefaultImagesCompat returns the shared OpenAI-compatible images adapter. +func DefaultImagesCompat() providerapi.ImagesProvider { + return defaultImages +} diff --git a/pkg/aiproxy/providers/openai/completions.go b/pkg/aiproxy/providers/openai/completions.go new file mode 100644 index 0000000000..0cd5806a34 --- /dev/null +++ b/pkg/aiproxy/providers/openai/completions.go @@ -0,0 +1,63 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +// CompletionsCompat forwards OpenAI legacy completions requests unchanged. +type CompletionsCompat struct { + Patches []PatchFunc +} + +// NewCompletionsCompat returns a new OpenAI-compatible completions adapter. +func NewCompletionsCompat(patches ...PatchFunc) *CompletionsCompat { + return &CompletionsCompat{Patches: patches} +} + +func (p *CompletionsCompat) buildBody(body *jsonutils.JSONDict, upstreamModel string, stream bool) *jsonutils.JSONDict { + dup := CloneBodyWithModel(body, upstreamModel) + if len(p.Patches) == 0 { + return dup + } + return PatchBody(dup, stream, p.Patches...) +} + +func (p *CompletionsCompat) BuildCompletionsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + dup := p.buildBody(body, ctx.UpstreamModel, stream) + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: CompletionsURL(ctx.BaseURL), + Headers: BearerAuthHeaders(ctx.APIKey), + Body: []byte(dup.String()), + }, nil +} + +func (p *CompletionsCompat) NormalizeCompletionsResponse(body []byte) ([]byte, error) { + return body, nil +} + +func (p *CompletionsCompat) OpenAICompletionsStreamPassthrough() bool { + return true +} diff --git a/pkg/aiproxy/providers/openai/doc.go b/pkg/aiproxy/providers/openai/doc.go new file mode 100644 index 0000000000..64a2134fd4 --- /dev/null +++ b/pkg/aiproxy/providers/openai/doc.go @@ -0,0 +1 @@ +package openai // import "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" diff --git a/pkg/aiproxy/providers/openai/embeddings.go b/pkg/aiproxy/providers/openai/embeddings.go new file mode 100644 index 0000000000..d7293bf337 --- /dev/null +++ b/pkg/aiproxy/providers/openai/embeddings.go @@ -0,0 +1,49 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +// EmbeddingCompat forwards OpenAI embeddings requests unchanged. +type EmbeddingCompat struct{} + +// NewEmbeddingCompat returns a new OpenAI-compatible embeddings adapter. +func NewEmbeddingCompat() *EmbeddingCompat { + return &EmbeddingCompat{} +} + +func (p *EmbeddingCompat) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + dup := CloneBodyWithModel(body, ctx.UpstreamModel) + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: EmbeddingsURL(ctx.BaseURL), + Headers: BearerAuthHeaders(ctx.APIKey), + Body: []byte(dup.String()), + }, nil +} + +func (p *EmbeddingCompat) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) { + return body, nil +} diff --git a/pkg/aiproxy/providers/openai/images.go b/pkg/aiproxy/providers/openai/images.go new file mode 100644 index 0000000000..a2b228b3da --- /dev/null +++ b/pkg/aiproxy/providers/openai/images.go @@ -0,0 +1,49 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +// ImagesCompat forwards OpenAI images/generations requests unchanged. +type ImagesCompat struct{} + +// NewImagesCompat returns a new OpenAI-compatible images adapter. +func NewImagesCompat() *ImagesCompat { + return &ImagesCompat{} +} + +func (p *ImagesCompat) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + dup := CloneBodyWithModel(body, ctx.UpstreamModel) + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: ImagesGenerationsURL(ctx.BaseURL), + Headers: BearerAuthHeaders(ctx.APIKey), + Body: []byte(dup.String()), + }, nil +} + +func (p *ImagesCompat) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) { + return body, nil +} diff --git a/pkg/aiproxy/providers/openai/schema.go b/pkg/aiproxy/providers/openai/schema.go new file mode 100644 index 0000000000..b122fc540c --- /dev/null +++ b/pkg/aiproxy/providers/openai/schema.go @@ -0,0 +1,414 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" +) + +// Message is one OpenAI chat message entry. +type Message struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` +} + +// ImageItem is one generated image in an OpenAI images response. +type ImageItem struct { + URL string + B64 string + RevisedPrompt string +} + +// MessageTextContent extracts plain text from an OpenAI message content field. +func MessageTextContent(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal(raw, &parts); err == nil { + var b strings.Builder + for _, p := range parts { + if p.Type == "text" && p.Text != "" { + if b.Len() > 0 { + b.WriteString("\n") + } + b.WriteString(p.Text) + } + } + return b.String() + } + return string(raw) +} + +// ParseMessages splits OpenAI chat messages and hoists system prompts. +func ParseMessages(body *jsonutils.JSONDict) ([]Message, string, error) { + if body == nil { + return nil, "", fmt.Errorf("nil request body") + } + arr, err := body.Get("messages") + if err != nil { + return nil, "", fmt.Errorf("missing messages") + } + raw := []byte(arr.String()) + var msgs []Message + if err := json.Unmarshal(raw, &msgs); err != nil { + return nil, "", fmt.Errorf("invalid messages: %w", err) + } + var system strings.Builder + out := make([]Message, 0, len(msgs)) + for i := range msgs { + role := strings.ToLower(strings.TrimSpace(msgs[i].Role)) + switch role { + case "system": + text := MessageTextContent(msgs[i].Content) + if text != "" { + if system.Len() > 0 { + system.WriteString("\n\n") + } + system.WriteString(text) + } + case "user", "assistant", "tool": + out = append(out, msgs[i]) + default: + out = append(out, msgs[i]) + } + } + return out, system.String(), nil +} + +// IntParam reads the first positive int param from an OpenAI JSON body. +func IntParam(body *jsonutils.JSONDict, keys ...string) (int, bool) { + for _, k := range keys { + if v, err := body.Int(k); err == nil && v > 0 { + return int(v), true + } + } + return 0, false +} + +// FloatParam reads a float param from an OpenAI JSON body. +func FloatParam(body *jsonutils.JSONDict, key string) (float64, bool) { + if v, err := body.Float(key); err == nil { + return v, true + } + return 0, false +} + +// CloneBodyWithModel clones the request body and sets the upstream model id. +func CloneBodyWithModel(body *jsonutils.JSONDict, upstreamModel string) *jsonutils.JSONDict { + dup := jsonutils.NewDict() + if body != nil { + dup = body.Copy() + } + dup.Set("model", jsonutils.NewString(upstreamModel)) + return dup +} + +// ChatCompletionsURL builds an OpenAI-compatible chat completions endpoint. +func ChatCompletionsURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(base, "/chat/completions") { + return base + } + if strings.HasSuffix(base, "/v2") { + return base + "/chat/completions" + } + if strings.HasSuffix(base, "/v1") { + return base + "/chat/completions" + } + return base + "/v1/chat/completions" +} + +// CompletionsURL builds an OpenAI-compatible legacy completions endpoint. +func CompletionsURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(base, "/completions") { + return base + } + if strings.HasSuffix(base, "/v2") { + return base + "/completions" + } + if strings.HasSuffix(base, "/v1") { + return base + "/completions" + } + return base + "/v1/completions" +} + +// EmbeddingsURL builds an OpenAI-compatible embeddings endpoint. +func EmbeddingsURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(base, "/embeddings") { + return base + } + if strings.HasSuffix(base, "/v2") { + return base + "/embeddings" + } + if strings.HasSuffix(base, "/v1") { + return base + "/embeddings" + } + return base + "/v1/embeddings" +} + +// ImagesGenerationsURL builds an OpenAI-compatible images/generations endpoint. +func ImagesGenerationsURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(base, "/images/generations") { + return base + } + if strings.HasSuffix(base, "/v2") { + return base + "/images/generations" + } + if strings.HasSuffix(base, "/v1") { + return base + "/images/generations" + } + return base + "/v1/images/generations" +} + +// BearerAuthHeaders returns standard OpenAI bearer auth headers. +func BearerAuthHeaders(apiKey string) map[string]string { + return map[string]string{ + "Authorization": "Bearer " + strings.TrimSpace(apiKey), + "Content-Type": "application/json", + } +} + +// JoinURL joins a base URL and path segment. +func JoinURL(base, path string) string { + base = strings.TrimRight(strings.TrimSpace(base), "/") + path = strings.TrimLeft(strings.TrimSpace(path), "/") + if base == "" { + return "/" + path + } + return base + "/" + path +} + +// FinishReasonFromStop maps provider-specific stop reasons to OpenAI finish_reason values. +func FinishReasonFromStop(stop string) string { + switch strings.TrimSpace(stop) { + case "end_turn", "stop_sequence", "stop", "STOP": + return "stop" + case "max_tokens", "length": + return "length" + case "tool_use": + return "tool_calls" + default: + if stop == "" { + return "stop" + } + return stop + } +} + +// NewChatCompletion builds an OpenAI chat.completion response object. +func NewChatCompletion(model, id, content, finishReason string, promptTokens, completionTokens int) map[string]interface{} { + total := promptTokens + completionTokens + return map[string]interface{}{ + "id": id, + "object": "chat.completion", + "created": time.Now().Unix(), + "model": model, + "choices": []map[string]interface{}{ + { + "index": 0, + "message": map[string]interface{}{ + "role": "assistant", + "content": content, + }, + "finish_reason": FinishReasonFromStop(finishReason), + }, + }, + "usage": map[string]interface{}{ + "prompt_tokens": promptTokens, + "completion_tokens": completionTokens, + "total_tokens": total, + }, + } +} + +// NewStreamChunk builds an OpenAI chat.completion.chunk object. +func NewStreamChunk(model, id string, index int, content string, finishReason string) map[string]interface{} { + delta := map[string]interface{}{ + "role": "assistant", + } + if content != "" { + delta["content"] = content + } + choice := map[string]interface{}{ + "index": index, + "delta": delta, + } + if finishReason != "" { + choice["finish_reason"] = FinishReasonFromStop(finishReason) + } + return map[string]interface{}{ + "id": id, + "object": "chat.completion.chunk", + "created": time.Now().Unix(), + "model": model, + "choices": []map[string]interface{}{choice}, + } +} + +// MarshalJSON marshals v to JSON bytes. +func MarshalJSON(v interface{}) ([]byte, error) { + return json.Marshal(v) +} + +// ParseEmbeddingInput extracts string inputs from an OpenAI embeddings body. +func ParseEmbeddingInput(body *jsonutils.JSONDict) (texts []string, rawInput json.RawMessage, isText bool, err error) { + if body == nil { + return nil, nil, false, fmt.Errorf("nil request body") + } + inp, err := body.Get("input") + if err != nil { + return nil, nil, false, fmt.Errorf("missing input") + } + rawInput = []byte(inp.String()) + var s string + if err := json.Unmarshal(rawInput, &s); err == nil { + return []string{s}, rawInput, true, nil + } + var arr []string + if err := json.Unmarshal(rawInput, &arr); err == nil { + return arr, rawInput, true, nil + } + return nil, rawInput, false, nil +} + +// NewEmbeddingsResponse builds an OpenAI embeddings list response. +func NewEmbeddingsResponse(model string, vectors [][]float64, promptTokens int) ([]byte, error) { + data := make([]map[string]interface{}, len(vectors)) + for i, v := range vectors { + data[i] = map[string]interface{}{ + "object": "embedding", + "index": i, + "embedding": v, + } + } + if promptTokens <= 0 { + promptTokens = 0 + } + return MarshalJSON(map[string]interface{}{ + "object": "list", + "data": data, + "model": model, + "usage": map[string]interface{}{ + "prompt_tokens": promptTokens, + "total_tokens": promptTokens, + }, + }) +} + +// ParseImagePrompt reads the prompt from an OpenAI images/generations body. +func ParseImagePrompt(body *jsonutils.JSONDict) (string, error) { + if body == nil { + return "", fmt.Errorf("nil request body") + } + prompt, err := body.GetString("prompt") + if err != nil || strings.TrimSpace(prompt) == "" { + return "", fmt.Errorf("missing prompt") + } + return strings.TrimSpace(prompt), nil +} + +// ImageCount reads n from an OpenAI images/generations body. +func ImageCount(body *jsonutils.JSONDict) int { + if body == nil { + return 1 + } + if n, err := body.Int("n"); err == nil && n > 0 { + return int(n) + } + return 1 +} + +// ImageSize reads size from an OpenAI images/generations body. +func ImageSize(body *jsonutils.JSONDict) string { + if body == nil { + return "1024x1024" + } + if s, err := body.GetString("size"); err == nil && strings.TrimSpace(s) != "" { + return strings.TrimSpace(s) + } + return "1024x1024" +} + +// SizeToAspectRatio maps OpenAI image size strings to provider aspect ratios. +func SizeToAspectRatio(size string) string { + switch strings.TrimSpace(size) { + case "1024x1792", "768x1344", "720x1280": + return "9:16" + case "1792x1024", "1344x768", "1280x720": + return "16:9" + case "256x256", "512x512", "1024x1024": + return "1:1" + default: + return "1:1" + } +} + +// NewImagesGenerationsResponse builds an OpenAI images/generations response. +func NewImagesGenerationsResponse(items []ImageItem) ([]byte, error) { + data := make([]map[string]interface{}, 0, len(items)) + for _, item := range items { + row := map[string]interface{}{} + if item.URL != "" { + row["url"] = item.URL + } + if item.B64 != "" { + row["b64_json"] = item.B64 + } + if item.RevisedPrompt != "" { + row["revised_prompt"] = item.RevisedPrompt + } + data = append(data, row) + } + return MarshalJSON(map[string]interface{}{ + "created": time.Now().Unix(), + "data": data, + }) +} + +// PatchFunc mutates an OpenAI request body before forwarding upstream. +type PatchFunc func(body *jsonutils.JSONDict, stream bool) + +// PatchBody clones body and applies optional patches. +func PatchBody(body *jsonutils.JSONDict, stream bool, patches ...PatchFunc) *jsonutils.JSONDict { + dup := jsonutils.NewDict() + if body != nil { + dup = body.Copy() + } + for _, patch := range patches { + if patch != nil { + patch(dup, stream) + } + } + return dup +} diff --git a/pkg/aiproxy/providers/openai/tools.go b/pkg/aiproxy/providers/openai/tools.go new file mode 100644 index 0000000000..d8bde19f01 --- /dev/null +++ b/pkg/aiproxy/providers/openai/tools.go @@ -0,0 +1,528 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" +) + +// ToolCall is one OpenAI assistant tool invocation. +type ToolCall struct { + Index int `json:"index,omitempty"` + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Function ToolFunction `json:"function"` +} + +// ToolFunction is the function payload inside a tool call. +type ToolFunction struct { + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +// ToolDefinition describes one OpenAI function tool. +type ToolDefinition struct { + Type string `json:"type"` + Function ToolFunctionDef `json:"function"` +} + +// ToolFunctionDef is the function schema in an OpenAI tools array. +type ToolFunctionDef struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters json.RawMessage `json:"parameters,omitempty"` +} + +// AssistantMessage is a normalized assistant response for OpenAI chat.completion. +type AssistantMessage struct { + Content string + ToolCalls []ToolCall +} + +// ExtractTools reads tools and tool_choice from an OpenAI chat body. +func ExtractTools(body *jsonutils.JSONDict) ([]ToolDefinition, json.RawMessage, error) { + if body == nil { + return nil, nil, nil + } + toolsRaw, err := body.Get("tools") + if err != nil { + return nil, nil, nil + } + var tools []ToolDefinition + if err := json.Unmarshal([]byte(toolsRaw.String()), &tools); err != nil { + return nil, nil, fmt.Errorf("invalid tools: %w", err) + } + var toolChoice json.RawMessage + if tc, err := body.Get("tool_choice"); err == nil { + toolChoice = []byte(tc.String()) + } + return tools, toolChoice, nil +} + +// ToolsToAnthropic converts OpenAI tools to Anthropic tools. +func ToolsToAnthropic(tools []ToolDefinition) []map[string]interface{} { + out := make([]map[string]interface{}, 0, len(tools)) + for _, t := range tools { + if strings.TrimSpace(t.Type) != "" && t.Type != "function" { + continue + } + name := strings.TrimSpace(t.Function.Name) + if name == "" { + continue + } + item := map[string]interface{}{ + "name": name, + } + if desc := strings.TrimSpace(t.Function.Description); desc != "" { + item["description"] = desc + } + if len(t.Function.Parameters) > 0 && string(t.Function.Parameters) != "null" { + var schema interface{} + if json.Unmarshal(t.Function.Parameters, &schema) == nil { + item["input_schema"] = schema + } + } + out = append(out, item) + } + return out +} + +// ToolChoiceToAnthropic converts OpenAI tool_choice to Anthropic tool_choice. +func ToolChoiceToAnthropic(raw json.RawMessage) interface{} { + if len(raw) == 0 { + return nil + } + var s string + if json.Unmarshal(raw, &s) == nil { + switch strings.ToLower(strings.TrimSpace(s)) { + case "", "auto": + return map[string]interface{}{"type": "auto"} + case "none": + return map[string]interface{}{"type": "none"} + case "required": + return map[string]interface{}{"type": "any"} + } + } + var obj struct { + Type string `json:"type"` + Function struct { + Name string `json:"name"` + } `json:"function"` + } + if json.Unmarshal(raw, &obj) == nil { + if strings.EqualFold(obj.Type, "function") && strings.TrimSpace(obj.Function.Name) != "" { + return map[string]interface{}{ + "type": "tool", + "name": strings.TrimSpace(obj.Function.Name), + } + } + } + return nil +} + +// MessagesToAnthropic converts OpenAI messages to Anthropic message objects. +func MessagesToAnthropic(msgs []Message) ([]map[string]interface{}, error) { + out := make([]map[string]interface{}, 0, len(msgs)) + for _, m := range msgs { + role := strings.ToLower(strings.TrimSpace(m.Role)) + switch role { + case "assistant": + blocks := assistantContentToAnthropic(m) + if len(blocks) == 0 { + continue + } + out = append(out, map[string]interface{}{ + "role": "assistant", + "content": blocks, + }) + case "tool": + text := MessageTextContent(m.Content) + if m.ToolCallID == "" && text == "" { + continue + } + out = append(out, map[string]interface{}{ + "role": "user", + "content": []map[string]interface{}{ + { + "type": "tool_result", + "tool_use_id": m.ToolCallID, + "content": text, + }, + }, + }) + case "user": + text := MessageTextContent(m.Content) + if text == "" { + continue + } + out = append(out, map[string]interface{}{ + "role": "user", + "content": []map[string]interface{}{ + {"type": "text", "text": text}, + }, + }) + default: + text := MessageTextContent(m.Content) + if text == "" { + continue + } + out = append(out, map[string]interface{}{ + "role": role, + "content": []map[string]interface{}{ + {"type": "text", "text": text}, + }, + }) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no convertible messages") + } + return out, nil +} + +func assistantContentToAnthropic(m Message) []map[string]interface{} { + blocks := make([]map[string]interface{}, 0, 1+len(m.ToolCalls)) + if text := MessageTextContent(m.Content); text != "" { + blocks = append(blocks, map[string]interface{}{ + "type": "text", + "text": text, + }) + } + for _, tc := range m.ToolCalls { + if strings.TrimSpace(tc.Function.Name) == "" { + continue + } + input := map[string]interface{}{} + args := strings.TrimSpace(tc.Function.Arguments) + if args != "" { + _ = json.Unmarshal([]byte(args), &input) + } + id := strings.TrimSpace(tc.ID) + if id == "" { + id = "toolu_" + strings.TrimSpace(tc.Function.Name) + } + blocks = append(blocks, map[string]interface{}{ + "type": "tool_use", + "id": id, + "name": strings.TrimSpace(tc.Function.Name), + "input": input, + }) + } + return blocks +} + +// AnthropicBlock is one Anthropic message content block. +type AnthropicBlock struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input map[string]interface{} `json:"input"` +} + +// AnthropicBlocksToAssistant converts Anthropic content blocks to OpenAI assistant message fields. +func AnthropicBlocksToAssistant(blocks []AnthropicBlock) AssistantMessage { + var out AssistantMessage + for _, b := range blocks { + switch b.Type { + case "text": + out.Content += b.Text + case "tool_use": + args, _ := json.Marshal(b.Input) + id := strings.TrimSpace(b.ID) + if id == "" { + id = "call_" + strings.TrimSpace(b.Name) + } + out.ToolCalls = append(out.ToolCalls, ToolCall{ + ID: id, + Type: "function", + Function: ToolFunction{ + Name: strings.TrimSpace(b.Name), + Arguments: string(args), + }, + }) + } + } + return out +} + +// ToolsToGemini converts OpenAI tools to Gemini functionDeclarations. +func ToolsToGemini(tools []ToolDefinition) []map[string]interface{} { + decls := make([]map[string]interface{}, 0, len(tools)) + for _, t := range tools { + if t.Type != "" && t.Type != "function" { + continue + } + name := strings.TrimSpace(t.Function.Name) + if name == "" { + continue + } + decl := map[string]interface{}{ + "name": name, + } + if desc := strings.TrimSpace(t.Function.Description); desc != "" { + decl["description"] = desc + } + if len(t.Function.Parameters) > 0 && string(t.Function.Parameters) != "null" { + var params interface{} + if json.Unmarshal(t.Function.Parameters, ¶ms) == nil { + decl["parameters"] = params + } + } + decls = append(decls, decl) + } + if len(decls) == 0 { + return nil + } + return []map[string]interface{}{{"functionDeclarations": decls}} +} + +// MessagesToGemini converts OpenAI messages to Gemini contents entries. +func MessagesToGemini(msgs []Message) ([]map[string]interface{}, error) { + out := make([]map[string]interface{}, 0, len(msgs)) + for _, m := range msgs { + role := strings.ToLower(strings.TrimSpace(m.Role)) + switch role { + case "assistant": + parts := assistantPartsToGemini(m) + if len(parts) == 0 { + continue + } + out = append(out, map[string]interface{}{ + "role": "model", + "parts": parts, + }) + case "tool": + name := strings.TrimSpace(m.Name) + if name == "" { + name = "tool" + } + resp := toolResultToGeminiResponse(MessageTextContent(m.Content)) + out = append(out, map[string]interface{}{ + "role": "user", + "parts": []map[string]interface{}{ + { + "functionResponse": map[string]interface{}{ + "name": name, + "response": resp, + }, + }, + }, + }) + case "user": + text := MessageTextContent(m.Content) + if text == "" { + continue + } + out = append(out, map[string]interface{}{ + "role": "user", + "parts": []map[string]interface{}{ + {"text": text}, + }, + }) + default: + text := MessageTextContent(m.Content) + if text == "" { + continue + } + out = append(out, map[string]interface{}{ + "role": "user", + "parts": []map[string]interface{}{ + {"text": text}, + }, + }) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no convertible messages") + } + return out, nil +} + +func assistantPartsToGemini(m Message) []map[string]interface{} { + parts := make([]map[string]interface{}, 0, 1+len(m.ToolCalls)) + if text := MessageTextContent(m.Content); text != "" { + parts = append(parts, map[string]interface{}{"text": text}) + } + for _, tc := range m.ToolCalls { + name := strings.TrimSpace(tc.Function.Name) + if name == "" { + continue + } + args := map[string]interface{}{} + if raw := strings.TrimSpace(tc.Function.Arguments); raw != "" { + _ = json.Unmarshal([]byte(raw), &args) + } + parts = append(parts, map[string]interface{}{ + "functionCall": map[string]interface{}{ + "name": name, + "args": args, + }, + }) + } + return parts +} + +func toolResultToGeminiResponse(content string) map[string]interface{} { + content = strings.TrimSpace(content) + if content == "" { + return map[string]interface{}{} + } + var obj map[string]interface{} + if json.Unmarshal([]byte(content), &obj) == nil { + return obj + } + return map[string]interface{}{"output": content} +} + +type geminiPart struct { + Text string `json:"text"` + FunctionCall *struct { + Name string `json:"name"` + Args map[string]interface{} `json:"args"` + } `json:"functionCall"` +} + +// GeminiPart is one Gemini content part in a candidate response. +type GeminiPart = geminiPart + +// GeminiPartsToAssistant converts Gemini candidate parts to OpenAI assistant fields. +func GeminiPartsToAssistant(parts []geminiPart) AssistantMessage { + var out AssistantMessage + for _, p := range parts { + if p.Text != "" { + out.Content += p.Text + } + if p.FunctionCall != nil && strings.TrimSpace(p.FunctionCall.Name) != "" { + args, _ := json.Marshal(p.FunctionCall.Args) + out.ToolCalls = append(out.ToolCalls, ToolCall{ + ID: "call_" + strings.TrimSpace(p.FunctionCall.Name), + Type: "function", + Function: ToolFunction{ + Name: strings.TrimSpace(p.FunctionCall.Name), + Arguments: string(args), + }, + }) + } + } + return out +} + +// NewChatCompletionWithTools builds an OpenAI chat.completion including tool_calls. +func NewChatCompletionWithTools(model, id string, msg AssistantMessage, finishReason string, promptTokens, completionTokens int) map[string]interface{} { + message := map[string]interface{}{ + "role": "assistant", + } + if msg.Content != "" { + message["content"] = msg.Content + } else if len(msg.ToolCalls) > 0 { + message["content"] = nil + } else { + message["content"] = "" + } + if len(msg.ToolCalls) > 0 { + calls := make([]map[string]interface{}, len(msg.ToolCalls)) + for i, tc := range msg.ToolCalls { + typ := tc.Type + if typ == "" { + typ = "function" + } + calls[i] = map[string]interface{}{ + "id": tc.ID, + "type": typ, + "function": map[string]interface{}{ + "name": tc.Function.Name, + "arguments": tc.Function.Arguments, + }, + } + } + message["tool_calls"] = calls + } + total := promptTokens + completionTokens + reason := FinishReasonFromStop(finishReason) + if len(msg.ToolCalls) > 0 && reason == "stop" { + reason = "tool_calls" + } + return map[string]interface{}{ + "id": id, + "object": "chat.completion", + "created": jsonNowUnix(), + "model": model, + "choices": []map[string]interface{}{ + { + "index": 0, + "message": message, + "finish_reason": reason, + }, + }, + "usage": map[string]interface{}{ + "prompt_tokens": promptTokens, + "completion_tokens": completionTokens, + "total_tokens": total, + }, + } +} + +// NewStreamChunkToolDelta builds an OpenAI stream chunk with tool_calls delta. +func NewStreamChunkToolDelta(model, id string, index int, tc ToolCall, finishReason string) map[string]interface{} { + delta := map[string]interface{}{ + "role": "assistant", + } + call := map[string]interface{}{ + "index": index, + } + if tc.ID != "" { + call["id"] = tc.ID + } + typ := tc.Type + if typ == "" { + typ = "function" + } + call["type"] = typ + fn := map[string]interface{}{} + if tc.Function.Name != "" { + fn["name"] = tc.Function.Name + } + if tc.Function.Arguments != "" { + fn["arguments"] = tc.Function.Arguments + } + if len(fn) > 0 { + call["function"] = fn + } + delta["tool_calls"] = []map[string]interface{}{call} + choice := map[string]interface{}{ + "index": 0, + "delta": delta, + } + if finishReason != "" { + choice["finish_reason"] = FinishReasonFromStop(finishReason) + } + return map[string]interface{}{ + "id": id, + "object": "chat.completion.chunk", + "created": jsonNowUnix(), + "model": model, + "choices": []map[string]interface{}{choice}, + } +} + +func jsonNowUnix() int64 { + return time.Now().Unix() +} diff --git a/pkg/aiproxy/providers/openai/tools_test.go b/pkg/aiproxy/providers/openai/tools_test.go new file mode 100644 index 0000000000..de2b4b9160 --- /dev/null +++ b/pkg/aiproxy/providers/openai/tools_test.go @@ -0,0 +1,136 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "encoding/json" + "testing" + + "yunion.io/x/jsonutils" +) + +func TestToolsToAnthropic(t *testing.T) { + tools := []ToolDefinition{{ + Type: "function", + Function: ToolFunctionDef{ + Name: "get_weather", + Description: "Get weather", + Parameters: json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}}}`), + }, + }} + out := ToolsToAnthropic(tools) + if len(out) != 1 { + t.Fatalf("expected 1 tool, got %d", len(out)) + } + if out[0]["name"] != "get_weather" { + t.Fatalf("unexpected name: %v", out[0]["name"]) + } +} + +func TestMessagesToAnthropicToolRoundTrip(t *testing.T) { + msgs := []Message{ + {Role: "user", Content: json.RawMessage(`"hello"`)}, + { + Role: "assistant", + ToolCalls: []ToolCall{{ + ID: "call_1", + Type: "function", + Function: ToolFunction{ + Name: "get_weather", + Arguments: `{"city":"Boston"}`, + }, + }}, + }, + {Role: "tool", ToolCallID: "call_1", Content: json.RawMessage(`"72F"`)}, + } + anthropicMsgs, err := MessagesToAnthropic(msgs) + if err != nil { + t.Fatal(err) + } + if len(anthropicMsgs) != 3 { + t.Fatalf("expected 3 messages, got %d", len(anthropicMsgs)) + } + blocks, ok := anthropicMsgs[1]["content"].([]map[string]interface{}) + if !ok || len(blocks) != 1 || blocks[0]["type"] != "tool_use" { + t.Fatalf("expected assistant tool_use block, got %#v", anthropicMsgs[1]) + } +} + +func TestAnthropicBlocksToAssistant(t *testing.T) { + msg := AnthropicBlocksToAssistant([]AnthropicBlock{ + {Type: "text", Text: "Checking"}, + {Type: "tool_use", ID: "toolu_1", Name: "get_weather", Input: map[string]interface{}{"city": "Boston"}}, + }) + if msg.Content != "Checking" { + t.Fatalf("unexpected content: %q", msg.Content) + } + if len(msg.ToolCalls) != 1 || msg.ToolCalls[0].Function.Name != "get_weather" { + t.Fatalf("unexpected tool calls: %#v", msg.ToolCalls) + } +} + +func TestMessagesToGemini(t *testing.T) { + msgs := []Message{ + {Role: "user", Content: json.RawMessage(`"hi"`)}, + { + Role: "assistant", + ToolCalls: []ToolCall{{ + Function: ToolFunction{Name: "fn", Arguments: `{"a":1}`}, + }}, + }, + {Role: "tool", Name: "fn", Content: json.RawMessage(`{"result":"ok"}`)}, + } + contents, err := MessagesToGemini(msgs) + if err != nil { + t.Fatal(err) + } + if len(contents) != 3 { + t.Fatalf("expected 3 contents, got %d", len(contents)) + } +} + +func TestExtractTools(t *testing.T) { + body, _ := jsonutils.Parse([]byte(`{ + "tools":[{"type":"function","function":{"name":"fn","parameters":{"type":"object"}}}], + "tool_choice":"auto" + }`)) + tools, choice, err := ExtractTools(body.(*jsonutils.JSONDict)) + if err != nil { + t.Fatal(err) + } + if len(tools) != 1 || tools[0].Function.Name != "fn" { + t.Fatalf("unexpected tools: %#v", tools) + } + if string(choice) != `"auto"` { + t.Fatalf("unexpected tool_choice: %s", choice) + } +} + +func TestNewChatCompletionWithTools(t *testing.T) { + out := NewChatCompletionWithTools("m", "id", AssistantMessage{ + ToolCalls: []ToolCall{{ + ID: "call_1", Type: "function", + Function: ToolFunction{Name: "fn", Arguments: `{}`}, + }}, + }, "tool_use", 1, 2) + choices := out["choices"].([]map[string]interface{}) + msg := choices[0]["message"].(map[string]interface{}) + if msg["tool_calls"] == nil { + t.Fatal("expected tool_calls in message") + } + if choices[0]["finish_reason"] != "tool_calls" { + t.Fatalf("expected finish_reason tool_calls, got %v", choices[0]["finish_reason"]) + } +} diff --git a/pkg/aiproxy/providers/providers_test.go b/pkg/aiproxy/providers/providers_test.go new file mode 100644 index 0000000000..6b6bfb5ae3 --- /dev/null +++ b/pkg/aiproxy/providers/providers_test.go @@ -0,0 +1,196 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "encoding/json" + "testing" + + "yunion.io/x/jsonutils" +) + +func TestAliyunProviderEnableThinkingPatch(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("qwen-turbo"), "model") + body.Add(jsonutils.NewArray(jsonutils.NewDict()), "messages") + + p := Get("aliyun") + req, err := p.BuildUpstreamRequest(&ChatContext{ + ProviderKey: "aliyun", + BaseURL: "https://dashscope.aliyuncs.com/compatible-mode", + APIKey: "sk-test", + UpstreamModel: "qwen-turbo", + }, body, false) + if err != nil { + t.Fatal(err) + } + var wire map[string]interface{} + if err := json.Unmarshal(req.Body, &wire); err != nil { + t.Fatal(err) + } + if v, ok := wire["enable_thinking"].(bool); !ok || v { + t.Fatalf("expected enable_thinking=false, got %#v", wire["enable_thinking"]) + } +} + +func TestAnthropicProviderBuildRequest(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("claude-3-5-sonnet"), "model") + body.Add(jsonutils.NewInt(1024), "max_tokens") + sysMsg := jsonutils.NewDict() + sysMsg.Set("role", jsonutils.NewString("system")) + sysMsg.Set("content", jsonutils.NewString("You are helpful.")) + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("Hi")) + msgs := jsonutils.NewArray(sysMsg, userMsg) + body.Add(msgs, "messages") + + p := Get("anthropic") + req, err := p.BuildUpstreamRequest(&ChatContext{ + ProviderKey: "anthropic", + BaseURL: "https://api.anthropic.com", + APIKey: "sk-ant", + UpstreamModel: "claude-3-5-sonnet-20241022", + }, body, false) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.anthropic.com/v1/messages" { + t.Fatalf("unexpected url: %s", req.URL) + } + if req.Headers["x-api-key"] != "sk-ant" { + t.Fatalf("missing x-api-key header") + } + var wire map[string]interface{} + if err := json.Unmarshal(req.Body, &wire); err != nil { + t.Fatal(err) + } + if wire["system"] != "You are helpful." { + t.Fatalf("expected system prompt, got %#v", wire["system"]) + } +} + +func TestAnthropicNormalizeResponse(t *testing.T) { + p := Get("anthropic") + raw := []byte(`{ + "id":"msg_1", + "model":"claude-3-5-sonnet-20241022", + "content":[{"type":"text","text":"Hello"}], + "stop_reason":"end_turn", + "usage":{"input_tokens":3,"output_tokens":1} + }`) + out, err := p.NormalizeResponse(raw) + if err != nil { + t.Fatal(err) + } + var resp map[string]interface{} + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if resp["object"] != "chat.completion" { + t.Fatalf("unexpected object: %#v", resp["object"]) + } + choices := resp["choices"].([]interface{}) + msg := choices[0].(map[string]interface{})["message"].(map[string]interface{}) + if msg["content"] != "Hello" { + t.Fatalf("unexpected content: %#v", msg["content"]) + } +} + +func TestAnthropicToolCalling(t *testing.T) { + body := jsonutils.NewDict() + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("Weather in Boston?")) + msgs := jsonutils.NewArray(userMsg) + body.Add(msgs, "messages") + tool := jsonutils.NewDict() + tool.Set("type", jsonutils.NewString("function")) + fn := jsonutils.NewDict() + fn.Set("name", jsonutils.NewString("get_weather")) + fn.Set("parameters", jsonutils.NewDict()) + tool.Set("function", fn) + body.Add(jsonutils.NewArray(tool), "tools") + + p := Get("anthropic") + req, err := p.BuildUpstreamRequest(&ChatContext{ + BaseURL: "https://api.anthropic.com", + APIKey: "sk-ant", + UpstreamModel: "claude-3-5-sonnet-20241022", + }, body, false) + if err != nil { + t.Fatal(err) + } + var wire map[string]interface{} + if err := json.Unmarshal(req.Body, &wire); err != nil { + t.Fatal(err) + } + tools, ok := wire["tools"].([]interface{}) + if !ok || len(tools) != 1 { + t.Fatalf("expected tools in request, got %#v", wire["tools"]) + } + + raw := []byte(`{ + "id":"msg_2", + "model":"claude-3-5-sonnet-20241022", + "content":[{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"Boston"}}], + "stop_reason":"tool_use", + "usage":{"input_tokens":10,"output_tokens":5} + }`) + out, err := p.NormalizeResponse(raw) + if err != nil { + t.Fatal(err) + } + var resp map[string]interface{} + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + choices := resp["choices"].([]interface{}) + choice := choices[0].(map[string]interface{}) + if choice["finish_reason"] != "tool_calls" { + t.Fatalf("expected finish_reason tool_calls, got %#v", choice["finish_reason"]) + } + msg := choice["message"].(map[string]interface{}) + if msg["tool_calls"] == nil { + t.Fatal("expected tool_calls in normalized response") + } +} + +func TestOpenAICompatPassthrough(t *testing.T) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("gpt-4o"), "model") + body.Add(jsonutils.NewArray(jsonutils.NewDict()), "messages") + + p := Get("openai") + req, err := p.BuildUpstreamRequest(&ChatContext{ + BaseURL: "https://api.openai.com", + APIKey: "sk-test", + UpstreamModel: "gpt-4o-mini", + }, body, true) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.openai.com/v1/chat/completions" { + t.Fatalf("unexpected url: %s", req.URL) + } +} + +func TestRegistryFallback(t *testing.T) { + p := Get("unknown-provider-key") + if !p.OpenAIStreamPassthrough() { + t.Fatal("unknown provider should use openai-compatible passthrough") + } +} diff --git a/pkg/aiproxy/providers/registry.go b/pkg/aiproxy/providers/registry.go new file mode 100644 index 0000000000..8bfd001d40 --- /dev/null +++ b/pkg/aiproxy/providers/registry.go @@ -0,0 +1,105 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "strings" + "sync" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" + "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" + "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" + "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" + "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" + "yunion.io/x/onecloud/pkg/aiproxy/providers/gemini" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + "yunion.io/x/onecloud/pkg/aiproxy/providers/vllm" +) + +var ( + registryMu sync.RWMutex + registry = map[string]providerapi.Provider{} + defaultP providerapi.Provider +) + +var openAICompatKeys = []string{ + "openai", + "groq", + "mistral", + "cerebras", + "perplexity", + "openrouter", + "fireworks", + "nebius", + "xai", + "parasail", + "sgl", + "huggingface", + "ollama", + "xiaomi", +} + +func init() { + defaultP = openai.NewCompat("") + register(defaultP) + for _, key := range openAICompatKeys { + register(openai.NewCompat(key)) + } + register(cohere.New()) + register(aliyun.New()) + register(baidu.New()) + register(anthropic.New()) + register(gemini.New()) + register(azure.New()) + register(vllm.New()) +} + +// Register adds or replaces a provider implementation for its Key(). +func Register(p Provider) { + if p == nil { + return + } + register(p) +} + +func register(p providerapi.Provider) { + registryMu.Lock() + defer registryMu.Unlock() + k := normalizeKey(p.Key()) + if k == "" { + registry[""] = p + return + } + registry[k] = p +} + +// Get returns the provider for providerKey, or the default OpenAI-compatible passthrough. +func Get(providerKey string) Provider { + registryMu.RLock() + defer registryMu.RUnlock() + k := normalizeKey(providerKey) + if p, ok := registry[k]; ok { + return p + } + if defaultP != nil { + return defaultP + } + return openai.NewCompat("") +} + +func normalizeKey(k string) string { + return strings.ToLower(strings.TrimSpace(k)) +} diff --git a/pkg/aiproxy/providers/stream.go b/pkg/aiproxy/providers/stream.go new file mode 100644 index 0000000000..a343e737e0 --- /dev/null +++ b/pkg/aiproxy/providers/stream.go @@ -0,0 +1,25 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import api "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + +// OpenAIStreamPassthrough reports whether upstream SSE chunks are already OpenAI-compatible. +func OpenAIStreamPassthrough(prov Provider, ctx *ChatContext) bool { + if p, ok := prov.(api.ContextualStreamPassthrough); ok { + return p.OpenAIStreamPassthroughForContext(ctx) + } + return prov.OpenAIStreamPassthrough() +} diff --git a/pkg/aiproxy/providers/types.go b/pkg/aiproxy/providers/types.go new file mode 100644 index 0000000000..588ccd4b5e --- /dev/null +++ b/pkg/aiproxy/providers/types.go @@ -0,0 +1,32 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + api "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +type ( + ChatContext = api.ChatContext + HTTPRequest = api.HTTPRequest + StreamChunk = api.StreamChunk + StreamState = api.StreamState + Provider = api.Provider + EmbeddingsProvider = api.EmbeddingsProvider + ImagesProvider = api.ImagesProvider + CompletionsProvider = api.CompletionsProvider +) + +type ContextualStreamPassthrough = api.ContextualStreamPassthrough diff --git a/pkg/aiproxy/providers/vllm/doc.go b/pkg/aiproxy/providers/vllm/doc.go new file mode 100644 index 0000000000..ba9ec2dd03 --- /dev/null +++ b/pkg/aiproxy/providers/vllm/doc.go @@ -0,0 +1,2 @@ +// Package vllm adapts OpenAI-compatible requests to vLLM OpenAI API servers. +package vllm // import "yunion.io/x/onecloud/pkg/aiproxy/providers/vllm" diff --git a/pkg/aiproxy/providers/vllm/vllm.go b/pkg/aiproxy/providers/vllm/vllm.go new file mode 100644 index 0000000000..94c40bcaf2 --- /dev/null +++ b/pkg/aiproxy/providers/vllm/vllm.go @@ -0,0 +1,57 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vllm + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type provider struct { + *openai.Compat + completions *openai.CompletionsCompat +} + +func patchVLLMRequest(body *jsonutils.JSONDict, stream bool) { + if body == nil { + return + } + if !stream { + body.Remove("stream_options") + } +} + +// New returns the vLLM OpenAI-compatible provider adapter. +func New() providerapi.Provider { + patches := []openai.PatchFunc{patchVLLMRequest} + return &provider{ + Compat: openai.NewCompat("vllm", patches...), + completions: openai.NewCompletionsCompat(patches...), + } +} + +func (p *provider) BuildCompletionsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + return p.completions.BuildCompletionsRequest(ctx, body, stream) +} + +func (p *provider) NormalizeCompletionsResponse(body []byte) ([]byte, error) { + return p.completions.NormalizeCompletionsResponse(body) +} + +func (p *provider) OpenAICompletionsStreamPassthrough() bool { + return p.completions.OpenAICompletionsStreamPassthrough() +} diff --git a/pkg/aiproxy/providers/vllm/vllm_test.go b/pkg/aiproxy/providers/vllm/vllm_test.go new file mode 100644 index 0000000000..eea710def7 --- /dev/null +++ b/pkg/aiproxy/providers/vllm/vllm_test.go @@ -0,0 +1,77 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vllm + +import ( + "strings" + "testing" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +func TestVLLMProviderBuildCompletionsRequest(t *testing.T) { + p := New() + cp, ok := p.(providerapi.CompletionsProvider) + if !ok { + t.Fatal("vllm provider should implement CompletionsProvider") + } + body := jsonutils.NewDict() + body.Add(jsonutils.NewString("hello"), "prompt") + streamOpts := jsonutils.NewDict() + streamOpts.Add(jsonutils.JSONTrue, "include_usage") + body.Add(streamOpts, "stream_options") + + req, err := cp.BuildCompletionsRequest(&providerapi.ChatContext{ + BaseURL: "http://127.0.0.1:8000", + UpstreamModel: "Qwen/Qwen2.5-7B-Instruct", + }, body, false) + if err != nil { + t.Fatal(err) + } + if req.URL != "http://127.0.0.1:8000/v1/completions" { + t.Fatalf("unexpected url: %s", req.URL) + } + if strings.Contains(string(req.Body), "stream_options") { + t.Fatalf("stream_options should be stripped for non-stream requests: %s", req.Body) + } +} + +func TestVLLMProviderBuildChatCompletionsRequest(t *testing.T) { + p := New() + body := jsonutils.NewDict() + msg := jsonutils.NewDict() + msg.Add(jsonutils.NewString("user"), "role") + msg.Add(jsonutils.NewString("hi"), "content") + body.Add(jsonutils.NewArray(msg), "messages") + streamOpts := jsonutils.NewDict() + streamOpts.Add(jsonutils.JSONTrue, "include_usage") + body.Add(streamOpts, "stream_options") + + req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{ + BaseURL: "http://127.0.0.1:8000", + UpstreamModel: "Qwen/Qwen2.5-7B-Instruct", + }, body, false) + if err != nil { + t.Fatal(err) + } + if req.URL != "http://127.0.0.1:8000/v1/chat/completions" { + t.Fatalf("unexpected url: %s", req.URL) + } + if strings.Contains(string(req.Body), "stream_options") { + t.Fatalf("stream_options should be stripped for non-stream requests: %s", req.Body) + } +} diff --git a/pkg/aiproxy/service/doc.go b/pkg/aiproxy/service/doc.go new file mode 100644 index 0000000000..2c22bf6889 --- /dev/null +++ b/pkg/aiproxy/service/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service // import "yunion.io/x/onecloud/pkg/aiproxy/service" diff --git a/pkg/aiproxy/service/service.go b/pkg/aiproxy/service/service.go new file mode 100644 index 0000000000..bf03ada705 --- /dev/null +++ b/pkg/aiproxy/service/service.go @@ -0,0 +1,92 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "os" + "time" + + "yunion.io/x/log" + _ "yunion.io/x/sqlchemy/backends" + + "yunion.io/x/onecloud/pkg/aiproxy/handlers" + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/options" + apolicy "yunion.io/x/onecloud/pkg/aiproxy/policy" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon" + app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" + "yunion.io/x/onecloud/pkg/cloudcommon/cronman" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/cachesync" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" +) + +func StartService() { + opts := &options.Options + commonOpts := &opts.CommonOptions + dbOpts := &opts.DBOptions + baseOpts := &opts.BaseOptions + common_options.ParseOptions(opts, os.Args, "aiproxy.conf", api.SERVICE_TYPE) + apolicy.Init() + if err := models.InitLocalProxyNodeId(opts, opts.IsSlaveNode); err != nil { + log.Fatalf("init local proxy node id: %v", err) + } + + app_common.InitAuth(commonOpts, func() { + log.Infof("Auth complete!!") + }) + common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange) + + app := app_common.InitApp(&opts.BaseOptions, false) + + cloudcommon.InitDB(dbOpts) + handlers.InitHandlers(app, opts.IsSlaveNode) + + if opts.IsSlaveNode { + if !db.CheckSync(false, dbOpts.EnableDBChecksumTables, dbOpts.DBChecksumSkipInit) { + log.Fatalf("database schema not in sync!") + } + if dbOpts.ExitAfterDBInit { + log.Infof("Exiting after db initialization ...") + os.Exit(0) + } + db.AppDBInit(app) + startSlaveNodeRegisterLoop(opts) + } else { + db.EnsureAppSyncDB(app, dbOpts, models.InitDB) + } + defer cloudcommon.CloseDB() + + if !opts.IsSlaveNode { + err := taskman.TaskManager.InitializeData() + if err != nil { + log.Fatalf("TaskManager.InitializeData fail %s", err) + } + + cachesync.StartTenantCacheSync(opts.TenantCacheExpireSeconds) + + cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount, opts.TimeZone) + cron.AddJobAtIntervalsWithStartRun("TaskCleanupJob", time.Duration(options.Options.TaskArchiveIntervalMinutes)*time.Minute, taskman.TaskManager.TaskCleanupJob, true) + + cron.Start() + defer cron.Stop() + } + + app_common.ServeForeverWithCleanup(app, baseOpts, func() { + cloudcommon.CloseDB() + }) +} diff --git a/pkg/aiproxy/service/slave_register.go b/pkg/aiproxy/service/slave_register.go new file mode 100644 index 0000000000..1f2a7a565f --- /dev/null +++ b/pkg/aiproxy/service/slave_register.go @@ -0,0 +1,67 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/httputils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/options" + "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/mcclient/auth" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" +) + +func startSlaveNodeRegisterLoop(opts *options.SAiProxyOptions) { + interval := time.Duration(opts.NodeHeartbeatIntervalSeconds) * time.Second + if interval <= 0 { + interval = 60 * time.Second + } + address, err := models.AdvertiseAddressFromOptions(opts) + if err != nil { + log.Fatalf("invalid standby advertise address: %v", err) + } + + go func() { + ctx := context.Background() + register := func() { + session := auth.GetAdminSessionWithPublic(ctx, opts.Region) + master, err := session.GetServiceURL(apmodules.AiProxyNodes.ServiceType(), identity.EndpointInterfacePublic, httputils.POST) + if err != nil { + log.Errorf("aiproxy standby resolve primary public endpoint failed: %v", err) + return + } + params := jsonutils.Marshal(map[string]interface{}{ + "address": address, + }) + if _, err := apmodules.AiProxyNodes.PerformClassAction(session, "register", params); err != nil { + log.Errorf("aiproxy standby register with primary %s failed: %v", master, err) + return + } + log.Debugf("aiproxy standby registered with primary %s as %s", master, address) + } + register() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + register() + } + }() +} diff --git a/pkg/aiproxy/upstream/doc.go b/pkg/aiproxy/upstream/doc.go new file mode 100644 index 0000000000..49b4f2b91a --- /dev/null +++ b/pkg/aiproxy/upstream/doc.go @@ -0,0 +1 @@ +package upstream // import "yunion.io/x/onecloud/pkg/aiproxy/upstream" diff --git a/pkg/aiproxy/upstream/openai_compat.go b/pkg/aiproxy/upstream/openai_compat.go new file mode 100644 index 0000000000..8d32739a41 --- /dev/null +++ b/pkg/aiproxy/upstream/openai_compat.go @@ -0,0 +1,281 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upstream + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" +) + +// Request is one upstream chat call (OpenAI-compatible by default, or provider-native when URL/headers are set). +type Request struct { + BaseURL string + URL string + APIKey string + Headers map[string]string + Body []byte +} + +// Response is a non-streaming upstream response body. +type Response struct { + StatusCode int + Body []byte +} + +// StreamChunk is one SSE event payload (bytes after "data: "). +type StreamChunk struct { + Data []byte + Done bool +} + +// RawSSEEvent is one parsed server-sent event line group from an upstream. +type RawSSEEvent struct { + Event string + Data []byte +} + +// Error carries upstream HTTP status and optional JSON error body. +type Error struct { + StatusCode int + Message string + Body []byte +} + +func (e *Error) Error() string { + if e == nil { + return "" + } + if e.Message != "" { + return e.Message + } + if len(e.Body) > 0 { + return string(e.Body) + } + return fmt.Sprintf("upstream HTTP %d", e.StatusCode) +} + +// ChatCompletionsURL builds the chat completions endpoint from a provider base URL. +// BaseURL is the origin + optional path prefix (e.g. https://dashscope.aliyuncs.com/compatible-mode). +func ChatCompletionsURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(base, "/v1") { + return base + "/chat/completions" + } + return base + "/v1/chat/completions" +} + +var ( + httpClient *http.Client + httpClientOnce sync.Once +) + +func sharedHTTPClient() *http.Client { + httpClientOnce.Do(func() { + httpClient = &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 256, + MaxIdleConnsPerHost: 64, + IdleConnTimeout: 90 * time.Second, + }, + } + }) + return httpClient +} + +func requestURL(req *Request) string { + if req == nil { + return "" + } + if u := strings.TrimSpace(req.URL); u != "" { + return u + } + return ChatCompletionsURL(req.BaseURL) +} + +func newUpstreamRequest(ctx context.Context, req *Request) (*http.Request, error) { + if req == nil { + return nil, fmt.Errorf("nil upstream request") + } + url := requestURL(req) + apiKey := strings.TrimSpace(req.APIKey) + if url == "" { + return nil, fmt.Errorf("empty upstream URL") + } + if apiKey == "" && len(req.Headers) == 0 { + return nil, fmt.Errorf("empty API key") + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(req.Body)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + for k, v := range req.Headers { + httpReq.Header.Set(k, v) + } + if apiKey != "" && httpReq.Header.Get("Authorization") == "" && httpReq.Header.Get("x-api-key") == "" && httpReq.Header.Get("api-key") == "" && httpReq.Header.Get("x-goog-api-key") == "" { + httpReq.Header.Set("Authorization", "Bearer "+apiKey) + } + return httpReq, nil +} + +func readResponseBody(resp *http.Response, maxBytes int64) ([]byte, error) { + defer resp.Body.Close() + if maxBytes <= 0 { + maxBytes = 32 << 20 + } + return io.ReadAll(io.LimitReader(resp.Body, maxBytes)) +} + +func errorFromResponse(resp *http.Response, body []byte) *Error { + status := resp.StatusCode + msg := strings.TrimSpace(resp.Status) + if len(body) > 0 { + var wrap struct { + Error *struct { + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(body, &wrap); err == nil && wrap.Error != nil && wrap.Error.Message != "" { + msg = wrap.Error.Message + } + } + return &Error{StatusCode: status, Message: msg, Body: body} +} + +// ChatCompletion performs a non-streaming chat completions request. +func ChatCompletion(ctx context.Context, req *Request) (*Response, *Error) { + httpReq, err := newUpstreamRequest(ctx, req) + if err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + resp, err := sharedHTTPClient().Do(httpReq) + if err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + body, err := readResponseBody(resp, 32<<20) + if err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, errorFromResponse(resp, body) + } + return &Response{StatusCode: resp.StatusCode, Body: body}, nil +} + +// ChatCompletionStream opens a streaming chat completions request and returns SSE data chunks. +func ChatCompletionStream(ctx context.Context, req *Request) (<-chan StreamChunk, *Error) { + _, resp, uerr := openChatCompletionStream(ctx, req) + if uerr != nil { + return nil, uerr + } + out := make(chan StreamChunk, 16) + go func() { + defer close(out) + defer resp.Body.Close() + for evt := range readSSE(resp.Body) { + if evt.Done { + out <- StreamChunk{Done: true} + return + } + out <- StreamChunk{Data: evt.Data} + } + if ctx.Err() != nil { + return + } + }() + return out, nil +} + +// ChatCompletionStreamRaw opens a streaming request and returns raw SSE events (event + data lines). +func ChatCompletionStreamRaw(ctx context.Context, req *Request) (<-chan RawSSEEvent, *Error) { + _, resp, uerr := openChatCompletionStream(ctx, req) + if uerr != nil { + return nil, uerr + } + out := make(chan RawSSEEvent, 16) + go func() { + defer close(out) + defer resp.Body.Close() + for evt := range readSSE(resp.Body) { + if evt.Done { + return + } + out <- RawSSEEvent{Event: evt.Event, Data: evt.Data} + } + }() + return out, nil +} + +type sseFrame struct { + Event string + Data []byte + Done bool +} + +func openChatCompletionStream(ctx context.Context, req *Request) (*http.Request, *http.Response, *Error) { + httpReq, err := newUpstreamRequest(ctx, req) + if err != nil { + return nil, nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + httpReq.Header.Set("Accept", "text/event-stream") + resp, err := sharedHTTPClient().Do(httpReq) + if err != nil { + return nil, nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := readResponseBody(resp, 1<<20) + return nil, nil, errorFromResponse(resp, body) + } + return httpReq, resp, nil +} + +func readSSE(r io.Reader) <-chan sseFrame { + out := make(chan sseFrame, 16) + go func() { + defer close(out) + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + var pendingEvent string + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + if strings.HasPrefix(line, "event:") { + pendingEvent = strings.TrimSpace(strings.TrimPrefix(line, "event:")) + continue + } + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if payload == "[DONE]" { + out <- sseFrame{Done: true} + return + } + out <- sseFrame{Event: pendingEvent, Data: []byte(payload)} + pendingEvent = "" + } + }() + return out +} diff --git a/pkg/apis/aiproxy/ai_key.go b/pkg/apis/aiproxy/ai_key.go new file mode 100644 index 0000000000..5155977fd0 --- /dev/null +++ b/pkg/apis/aiproxy/ai_key.go @@ -0,0 +1,91 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "encoding/json" + + "yunion.io/x/onecloud/pkg/apis" +) + +// SAiKeyRouting constrains which client request models may use an ai_key, and relative priority. +// Matching uses the JSON body "model" string (same as ai_routing.model_pattern: exact match, case-insensitive, +// or prefix* glob when the pattern ends with "*"). +// +// AllowedModelKeys: when non-empty, the requested model must match at least one entry. +// When empty, any model is allowed unless blocked by BlockedModelKeys. +// +// BlockedModelKeys: requested model must not match any entry (evaluated after allow-list pass). +// +// Weight in routing is legacy; prefer the ai_key.weight column. When ai_key.weight is unset (0), routing.weight is used. +type SAiKeyRouting struct { + AllowedModelKeys []string `json:"allowed_model_keys,omitempty"` + BlockedModelKeys []string `json:"blocked_model_keys,omitempty"` + Weight int `json:"weight,omitempty"` +} + +// String implements gotypes.ISerializable for sqlchemy JSON/compound columns. +func (r *SAiKeyRouting) String() string { + if r == nil { + return "{}" + } + b, err := json.Marshal(r) + if err != nil { + return "{}" + } + return string(b) +} + +// IsZero implements gotypes.ISerializable. +func (r *SAiKeyRouting) IsZero() bool { + if r == nil { + return true + } + return len(r.AllowedModelKeys) == 0 && len(r.BlockedModelKeys) == 0 && r.Weight == 0 +} + +type AiKeyListInput struct { + apis.EnabledStatusStandaloneResourceListInput + + AiProviderId string `json:"ai_provider_id"` +} + +type AiKeyCreateInput struct { + apis.EnabledStatusStandaloneResourceCreateInput + + AiProviderId string `json:"ai_provider_id"` + Secret string `json:"secret"` + Weight int `json:"weight"` + Routing *SAiKeyRouting `json:"routing"` +} + +type AiKeyUpdateInput struct { + apis.EnabledStatusStandaloneResourceBaseUpdateInput + + AiProviderId string `json:"ai_provider_id"` + Secret string `json:"secret"` + Weight int `json:"weight,omitzero"` + Routing *SAiKeyRouting `json:"routing"` + Enabled *bool `json:"enabled"` +} + +type AiKeyDetails struct { + apis.EnabledStatusStandaloneResourceDetails + + AiProviderId string `json:"ai_provider_id"` + AiProviderName string `json:"ai_provider_name"` + Weight int `json:"weight"` + Routing *SAiKeyRouting `json:"routing"` +} diff --git a/pkg/apis/aiproxy/ai_model.go b/pkg/apis/aiproxy/ai_model.go new file mode 100644 index 0000000000..10834303f6 --- /dev/null +++ b/pkg/apis/aiproxy/ai_model.go @@ -0,0 +1,49 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type AiModelListInput struct { + apis.EnabledStatusStandaloneResourceListInput + + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` +} + +type AiModelCreateInput struct { + apis.EnabledStatusStandaloneResourceCreateInput + + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` +} + +type AiModelUpdateInput struct { + apis.EnabledStatusStandaloneResourceBaseUpdateInput + + AiProviderId string `json:"ai_provider_id"` + ModelKey string `json:"model_key"` + Enabled *bool `json:"enabled"` +} + +type AiModelDetails struct { + apis.EnabledStatusStandaloneResourceDetails + + AiProviderId string `json:"ai_provider_id"` + AiProviderName string `json:"ai_provider_name"` + ModelKey string `json:"model_key"` +} diff --git a/pkg/apis/aiproxy/ai_provider.go b/pkg/apis/aiproxy/ai_provider.go new file mode 100644 index 0000000000..9a2bd0b0e2 --- /dev/null +++ b/pkg/apis/aiproxy/ai_provider.go @@ -0,0 +1,92 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "encoding/json" + "strings" + + "yunion.io/x/onecloud/pkg/apis" +) + +// SAiProviderConfig holds JSON-serialized provider connectivity settings for an ai_provider row. +type SAiProviderConfig struct { + BaseURL string `json:"base_url,omitempty"` + APIKey string `json:"api_key,omitempty"` +} + +// ResolvedBaseURL returns config.base_url. +func (c *SAiProviderConfig) ResolvedBaseURL() string { + if c == nil { + return "" + } + return strings.TrimSpace(c.BaseURL) +} + +// ResolvedAPIKey returns config.api_key. +func (c *SAiProviderConfig) ResolvedAPIKey() string { + if c == nil { + return "" + } + return strings.TrimSpace(c.APIKey) +} + +// String implements gotypes.ISerializable for sqlchemy JSON/compound columns. +func (c *SAiProviderConfig) String() string { + if c == nil { + return "{}" + } + b, err := json.Marshal(c) + if err != nil { + return "{}" + } + return string(b) +} + +// IsZero implements gotypes.ISerializable. +func (c *SAiProviderConfig) IsZero() bool { + if c == nil { + return true + } + return c.ResolvedBaseURL() == "" && c.ResolvedAPIKey() == "" +} + +type AiProviderListInput struct { + apis.EnabledStatusStandaloneResourceListInput + + ProviderKey string `json:"provider_key"` +} + +type AiProviderCreateInput struct { + apis.EnabledStatusStandaloneResourceCreateInput + + ProviderKey string `json:"provider_key"` + Config *SAiProviderConfig `json:"config"` +} + +type AiProviderUpdateInput struct { + apis.EnabledStatusStandaloneResourceBaseUpdateInput + + ProviderKey string `json:"provider_key"` + Config *SAiProviderConfig `json:"config"` + Enabled *bool `json:"enabled"` +} + +type AiProviderDetails struct { + apis.EnabledStatusStandaloneResourceDetails + + ProviderKey string `json:"provider_key"` + Config *SAiProviderConfig `json:"config"` +} diff --git a/pkg/apis/aiproxy/ai_proxy_node.go b/pkg/apis/aiproxy/ai_proxy_node.go new file mode 100644 index 0000000000..974b0efed5 --- /dev/null +++ b/pkg/apis/aiproxy/ai_proxy_node.go @@ -0,0 +1,65 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "time" + + "yunion.io/x/onecloud/pkg/apis" +) + +type AiProxyNodeListInput struct { + apis.EnabledStatusStandaloneResourceListInput + + Address string `json:"address"` + Domain string `json:"domain"` +} + +type AiProxyNodeCreateInput struct { + apis.EnabledStatusStandaloneResourceCreateInput + + Address string `json:"address"` + Domain string `json:"domain"` + HbTimeout int `json:"hb_timeout"` +} + +type AiProxyNodeUpdateInput struct { + apis.EnabledStatusStandaloneResourceBaseUpdateInput + + Address string `json:"address"` + Domain string `json:"domain"` + HbTimeout int `json:"hb_timeout"` + Enabled *bool `json:"enabled"` +} + +type AiProxyNodeDetails struct { + apis.EnabledStatusStandaloneResourceDetails + + Address string `json:"address"` + Domain string `json:"domain"` + LastSeen time.Time `json:"last_seen"` + HbTimeout int `json:"hb_timeout"` + IsActive bool `json:"is_active"` +} + +// AiProxyNodeRegisterInput is sent by standby instances to the primary on startup and heartbeat. +type AiProxyNodeRegisterInput struct { + Address string `json:"address"` + HbTimeout int `json:"hb_timeout"` +} + +type AiProxyNodeRegisterOutput struct { + Id string `json:"id"` +} diff --git a/pkg/apis/aiproxy/ai_routing.go b/pkg/apis/aiproxy/ai_routing.go new file mode 100644 index 0000000000..bb08d4f6dc --- /dev/null +++ b/pkg/apis/aiproxy/ai_routing.go @@ -0,0 +1,72 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type AiRoutingListInput struct { + apis.SharableVirtualResourceListInput + apis.EnabledResourceBaseListInput + + ModelPattern string `json:"model_pattern"` + AiProxyNodeId string `json:"ai_proxy_node_id"` +} + +// AiRoutingModelItem is one catalog model binding when creating ai_routing. +// Priority orders models within the routing (lower = higher priority). Weight is an alias for Priority. +type AiRoutingModelItem struct { + AiProviderId string `json:"ai_provider_id"` + AiModelId string `json:"ai_model_id"` + Priority int `json:"priority,omitempty"` + Weight int `json:"weight,omitempty"` + ModelPattern string `json:"model_pattern,omitempty"` + Enabled *bool `json:"enabled,omitempty"` +} + +type AiRoutingCreateInput struct { + apis.SharableVirtualResourceCreateInput + apis.EnabledBaseResourceCreateInput + + Priority int `json:"priority"` + ModelPattern string `json:"model_pattern"` + AiProxyNodeId string `json:"ai_proxy_node_id"` + Models []AiRoutingModelItem `json:"models"` +} + +type AiRoutingUpdateInput struct { + apis.SharableVirtualResourceBaseUpdateInput + + Priority int `json:"priority"` + ModelPattern string `json:"model_pattern"` + AiProxyNodeId string `json:"ai_proxy_node_id"` + Enabled *bool `json:"enabled"` +} + +type AiRoutingDetails struct { + apis.SharableVirtualResourceDetails + + Priority int `json:"priority"` + ModelPattern string `json:"model_pattern"` + AiProxyNodeId string `json:"ai_proxy_node_id"` + Enabled bool `json:"enabled"` + RoutingModels []AiRoutingModelDetails `json:"routing_models,omitempty"` +} + +// AiRoutingSetModelsInput replaces all ai_routing_models for an ai_routing. +type AiRoutingSetModelsInput struct { + Models []AiRoutingModelItem `json:"models"` +} diff --git a/pkg/apis/aiproxy/ai_routing_model.go b/pkg/apis/aiproxy/ai_routing_model.go new file mode 100644 index 0000000000..752b4068cc --- /dev/null +++ b/pkg/apis/aiproxy/ai_routing_model.go @@ -0,0 +1,61 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +type AiRoutingModelListInput struct { + apis.StandaloneResourceListInput + + AiRoutingId string `json:"ai_routing_id"` + AiProviderId string `json:"ai_provider_id"` + AiModelId string `json:"ai_model_id"` + Enabled *bool `json:"enabled"` +} + +type AiRoutingModelCreateInput struct { + apis.StandaloneResourceCreateInput + + AiRoutingId string `json:"ai_routing_id"` + AiProviderId string `json:"ai_provider_id"` + AiModelId string `json:"ai_model_id"` + Priority int `json:"priority"` + ModelPattern string `json:"model_pattern"` + Enabled *bool `json:"enabled"` +} + +type AiRoutingModelUpdateInput struct { + apis.StandaloneResourceBaseUpdateInput + + AiRoutingId string `json:"ai_routing_id,omitempty"` + AiProviderId string `json:"ai_provider_id,omitempty"` + AiModelId string `json:"ai_model_id,omitempty"` + Priority int `json:"priority,omitzero"` + ModelPattern string `json:"model_pattern,omitempty"` + Enabled *bool `json:"enabled,omitempty"` +} + +type AiRoutingModelDetails struct { + apis.StandaloneResourceDetails + + AiRoutingId string `json:"ai_routing_id"` + AiProviderId string `json:"ai_provider_id"` + AiModelId string `json:"ai_model_id"` + Priority int `json:"priority"` + ModelPattern string `json:"model_pattern"` + Enabled bool `json:"enabled"` +} diff --git a/pkg/apis/aiproxy/ai_virtual_key.go b/pkg/apis/aiproxy/ai_virtual_key.go new file mode 100644 index 0000000000..ea99d15e46 --- /dev/null +++ b/pkg/apis/aiproxy/ai_virtual_key.go @@ -0,0 +1,96 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "encoding/json" + + "yunion.io/x/pkg/tristate" + + "yunion.io/x/onecloud/pkg/apis" +) + +// SAiVirtualKeyLimits constrains which catalog providers a virtual key may route to, +// caps client max_tokens, and configures request rate (approximate per-minute token bucket). +// +// AllowedAiProviderIds: when non-empty, the resolved ai_provider must match one entry by id or name (case-insensitive for name). +// When empty, any provider allowed by routing applies. +// +// MaxTokensPerRequest: when > 0, caps the JSON body max_tokens (missing max_tokens is set to this value). +// +// RequestsPerMinute: when > 0, enforces an approximate per-minute limit per virtual key row (in-process; multi-replica deployments need external limiting). +type SAiVirtualKeyLimits struct { + AllowedAiProviderIds []string `json:"allowed_ai_provider_ids,omitempty"` + MaxTokensPerRequest int `json:"max_tokens_per_request,omitempty"` + RequestsPerMinute int `json:"requests_per_minute,omitempty"` +} + +// String implements gotypes.ISerializable for sqlchemy JSON columns. +func (l *SAiVirtualKeyLimits) String() string { + if l == nil { + return "{}" + } + b, err := json.Marshal(l) + if err != nil { + return "{}" + } + return string(b) +} + +// IsZero implements gotypes.ISerializable. +func (l *SAiVirtualKeyLimits) IsZero() bool { + if l == nil { + return true + } + return len(l.AllowedAiProviderIds) == 0 && l.MaxTokensPerRequest == 0 && l.RequestsPerMinute == 0 +} + +type AiVirtualKeyListInput struct { + apis.VirtualResourceListInput + apis.EnabledResourceBaseListInput + + VirtualKey string `json:"virtual_key"` + UserId string `json:"user_id"` +} + +type AiVirtualKeyCreateInput struct { + apis.VirtualResourceCreateInput + + // OwnerId is the owning user; defaults to the creating user when empty. + OwnerId string `json:"owner_id"` + // VirtualKey is optional; when empty a unique sk- prefixed key is generated. + VirtualKey string `json:"virtual_key"` + Limits *SAiVirtualKeyLimits `json:"limits"` + Enabled tristate.TriState `json:"enabled"` +} + +type AiVirtualKeyUpdateInput struct { + apis.VirtualResourceBaseUpdateInput + + OwnerId string `json:"owner_id"` + VirtualKey string `json:"virtual_key"` + Limits *SAiVirtualKeyLimits `json:"limits"` + Enabled tristate.TriState `json:"enabled"` +} + +type AiVirtualKeyDetails struct { + apis.VirtualResourceDetails + + OwnerId string `json:"owner_id"` + OwnerName string `json:"owner_name"` + VirtualKey string `json:"virtual_key"` + Limits *SAiVirtualKeyLimits `json:"limits"` + Enabled bool `json:"enabled"` +} diff --git a/pkg/apis/aiproxy/consts.go b/pkg/apis/aiproxy/consts.go new file mode 100644 index 0000000000..e9d0e04585 --- /dev/null +++ b/pkg/apis/aiproxy/consts.go @@ -0,0 +1,24 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +const ( + SERVICE_TYPE = apis.SERVICE_TYPE_AIPROXY + SERVICE_VERSION = "" +) diff --git a/pkg/apis/aiproxy/doc.go b/pkg/apis/aiproxy/doc.go new file mode 100644 index 0000000000..5dd08b93f3 --- /dev/null +++ b/pkg/apis/aiproxy/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy // import "yunion.io/x/onecloud/pkg/apis/aiproxy" diff --git a/pkg/apis/aiproxy/serialize_register.go b/pkg/apis/aiproxy/serialize_register.go new file mode 100644 index 0000000000..730639eea7 --- /dev/null +++ b/pkg/apis/aiproxy/serialize_register.go @@ -0,0 +1,33 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "reflect" + + "yunion.io/x/pkg/gotypes" +) + +func init() { + gotypes.RegisterSerializable(reflect.TypeOf((*SAiProviderConfig)(nil)), func() gotypes.ISerializable { + return &SAiProviderConfig{} + }) + gotypes.RegisterSerializable(reflect.TypeOf((*SAiKeyRouting)(nil)), func() gotypes.ISerializable { + return &SAiKeyRouting{} + }) + gotypes.RegisterSerializable(reflect.TypeOf((*SAiVirtualKeyLimits)(nil)), func() gotypes.ISerializable { + return &SAiVirtualKeyLimits{} + }) +} diff --git a/pkg/apis/compute/guestnetwork.go b/pkg/apis/compute/guestnetwork.go index c748d25e0c..75454aaff6 100644 --- a/pkg/apis/compute/guestnetwork.go +++ b/pkg/apis/compute/guestnetwork.go @@ -20,10 +20,10 @@ import ( "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/jsonutils" - billing_api "yunion.io/x/onecloud/pkg/apis/billing" "yunion.io/x/pkg/gotypes" "yunion.io/x/onecloud/pkg/apis" + billing_api "yunion.io/x/onecloud/pkg/apis/billing" ) type GuestnetworkDetails struct { diff --git a/pkg/apis/const.go b/pkg/apis/const.go index 18fdef3f5b..cd8e7aeac9 100644 --- a/pkg/apis/const.go +++ b/pkg/apis/const.go @@ -45,7 +45,8 @@ const ( SERVICE_TYPE_APIMAP = "apimap" - SERVICE_TYPE_LLM = "llm" + SERVICE_TYPE_LLM = "llm" + SERVICE_TYPE_AIPROXY = "aiproxy" STATUS_UPDATE_TAGS = "update_tags" STATUS_UPDATE_TAGS_FAILED = "update_tags_fail" @@ -130,6 +131,7 @@ var ( SERVICE_TYPE_VICTORIA_METRICS, SERVICE_TYPE_LOG, "s3gateway", + SERVICE_TYPE_AIPROXY, "common", "websocket", "echarts-ssr", diff --git a/pkg/compute/hostdrivers/proxmox.go b/pkg/compute/hostdrivers/proxmox.go index 8306db7966..c9e08fc1e3 100644 --- a/pkg/compute/hostdrivers/proxmox.go +++ b/pkg/compute/hostdrivers/proxmox.go @@ -21,12 +21,13 @@ import ( "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/cloudmux/pkg/multicloud/esxi/vcenter" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/httputils" ) type SProxmoxHostDriver struct { diff --git a/pkg/compute/models/guestnetwork_traffic_log.go b/pkg/compute/models/guestnetwork_traffic_log.go index bb052fd396..5849665b86 100644 --- a/pkg/compute/models/guestnetwork_traffic_log.go +++ b/pkg/compute/models/guestnetwork_traffic_log.go @@ -19,6 +19,7 @@ import ( "time" "github.com/golang-plus/errors" + "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/rbacscope" "yunion.io/x/sqlchemy" diff --git a/pkg/compute/models/guestnetworksecgroups.go b/pkg/compute/models/guestnetworksecgroups.go index c3c79d17f5..a2cf1da66c 100644 --- a/pkg/compute/models/guestnetworksecgroups.go +++ b/pkg/compute/models/guestnetworksecgroups.go @@ -21,6 +21,7 @@ import ( "strconv" "gopkg.in/fatih/set.v0" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" diff --git a/pkg/compute/regiondrivers/ecloud.go b/pkg/compute/regiondrivers/ecloud.go index a1bf8eb10c..7525703ed0 100644 --- a/pkg/compute/regiondrivers/ecloud.go +++ b/pkg/compute/regiondrivers/ecloud.go @@ -15,9 +15,10 @@ package regiondrivers import ( + "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/sqlchemy" ) type SEcloudRegionDriver struct { diff --git a/pkg/esxi/handler/proxmox.go b/pkg/esxi/handler/proxmox.go index 9143cb224d..aadad98f97 100644 --- a/pkg/esxi/handler/proxmox.go +++ b/pkg/esxi/handler/proxmox.go @@ -21,14 +21,15 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/appctx" + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/esxi" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/pkg/appctx" - "yunion.io/x/pkg/errors" ) const ( diff --git a/pkg/hostman/storageman/storage_proxmox.go b/pkg/hostman/storageman/storage_proxmox.go index d89febaf82..0b473c87b2 100644 --- a/pkg/hostman/storageman/storage_proxmox.go +++ b/pkg/hostman/storageman/storage_proxmox.go @@ -25,6 +25,10 @@ import ( "yunion.io/x/cloudmux/pkg/multicloud/proxmox" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/seclib" + "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/agent/iagent" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" @@ -32,9 +36,6 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/util/seclib" - "yunion.io/x/pkg/utils" ) type SProxmoxStorage struct { diff --git a/pkg/llm/service/handler.go b/pkg/llm/service/handler.go index 05765edd3f..0f70a2fc1a 100644 --- a/pkg/llm/service/handler.go +++ b/pkg/llm/service/handler.go @@ -9,6 +9,7 @@ import ( "time" "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/appsrv/dispatcher" diff --git a/pkg/mcclient/modules/aiproxy/doc.go b/pkg/mcclient/modules/aiproxy/doc.go new file mode 100644 index 0000000000..40d583283e --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/doc.go @@ -0,0 +1,16 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package aiproxy registers mcclient resource managers for the aiproxy service. +package aiproxy diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_keys.go b/pkg/mcclient/modules/aiproxy/mod_ai_keys.go new file mode 100644 index 0000000000..8764551b18 --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_keys.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiKeyManager struct { + modulebase.ResourceManager +} + +var AiKeys AiKeyManager + +func init() { + AiKeys = AiKeyManager{ + modules.NewAIProxyManager("ai_key", "ai_keys", + []string{}, + []string{}), + } + modules.Register(&AiKeys) +} diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_models.go b/pkg/mcclient/modules/aiproxy/mod_ai_models.go new file mode 100644 index 0000000000..21304aacf0 --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_models.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiModelManager struct { + modulebase.ResourceManager +} + +var AiModels AiModelManager + +func init() { + AiModels = AiModelManager{ + modules.NewAIProxyManager("ai_model", "ai_models", + []string{}, + []string{}), + } + modules.Register(&AiModels) +} diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_providers.go b/pkg/mcclient/modules/aiproxy/mod_ai_providers.go new file mode 100644 index 0000000000..db25d583b2 --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_providers.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiProviderManager struct { + modulebase.ResourceManager +} + +var AiProviders AiProviderManager + +func init() { + AiProviders = AiProviderManager{ + modules.NewAIProxyManager("ai_provider", "ai_providers", + []string{}, + []string{}), + } + modules.Register(&AiProviders) +} diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_proxy_nodes.go b/pkg/mcclient/modules/aiproxy/mod_ai_proxy_nodes.go new file mode 100644 index 0000000000..c542f32b1b --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_proxy_nodes.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiProxyNodeManager struct { + modulebase.ResourceManager +} + +var AiProxyNodes AiProxyNodeManager + +func init() { + AiProxyNodes = AiProxyNodeManager{ + modules.NewAIProxyManager("ai_proxy_node", "ai_proxy_nodes", + []string{"Id", "Name", "Address", "Domain", "Last_seen", "Hb_timeout", "Is_active", "Enabled", "Status"}, + []string{}), + } + modules.Register(&AiProxyNodes) +} diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_routing_models.go b/pkg/mcclient/modules/aiproxy/mod_ai_routing_models.go new file mode 100644 index 0000000000..ab399d88a3 --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_routing_models.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiRoutingModelManager struct { + modulebase.ResourceManager +} + +var AiRoutingModels AiRoutingModelManager + +func init() { + AiRoutingModels = AiRoutingModelManager{ + modules.NewAIProxyManager("ai_routing_model", "ai_routing_models", + []string{}, + []string{}), + } + modules.Register(&AiRoutingModels) +} diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_routings.go b/pkg/mcclient/modules/aiproxy/mod_ai_routings.go new file mode 100644 index 0000000000..900de27472 --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_routings.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiRoutingManager struct { + modulebase.ResourceManager +} + +var AiRoutings AiRoutingManager + +func init() { + AiRoutings = AiRoutingManager{ + modules.NewAIProxyManager("ai_routing", "ai_routings", + []string{}, + []string{}), + } + modules.Register(&AiRoutings) +} diff --git a/pkg/mcclient/modules/aiproxy/mod_ai_virtual_keys.go b/pkg/mcclient/modules/aiproxy/mod_ai_virtual_keys.go new file mode 100644 index 0000000000..8316d33eaa --- /dev/null +++ b/pkg/mcclient/modules/aiproxy/mod_ai_virtual_keys.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type AiVirtualKeyManager struct { + modulebase.ResourceManager +} + +var AiVirtualKeys AiVirtualKeyManager + +func init() { + AiVirtualKeys = AiVirtualKeyManager{ + modules.NewAIProxyManager("ai_virtual_key", "ai_virtual_keys", + []string{}, + []string{}), + } + modules.Register(&AiVirtualKeys) +} diff --git a/pkg/mcclient/modules/llm/mod_llm_model_set.go b/pkg/mcclient/modules/llm/mod_llm_model_set.go index a4c49bbfbb..3f961c352d 100644 --- a/pkg/mcclient/modules/llm/mod_llm_model_set.go +++ b/pkg/mcclient/modules/llm/mod_llm_model_set.go @@ -5,6 +5,7 @@ import ( "net/url" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modulebase" "yunion.io/x/onecloud/pkg/mcclient/modules" diff --git a/pkg/mcclient/modules/loader/loader.go b/pkg/mcclient/modules/loader/loader.go index b69c4d2c97..4a0b10ca88 100644 --- a/pkg/mcclient/modules/loader/loader.go +++ b/pkg/mcclient/modules/loader/loader.go @@ -15,6 +15,7 @@ package loader import ( + _ "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" _ "yunion.io/x/onecloud/pkg/mcclient/modules/ansible" _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudevent" _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudid" diff --git a/pkg/mcclient/modules/managers.go b/pkg/mcclient/modules/managers.go index 160c79b4c7..75e6b6ec4e 100644 --- a/pkg/mcclient/modules/managers.go +++ b/pkg/mcclient/modules/managers.go @@ -186,3 +186,9 @@ func NewLLMManager(keyword, keywordPlural string, columns, adminColumns []string BaseManager: *modulebase.NewBaseManager(apis.SERVICE_TYPE_LLM, "", "", columns, adminColumns), Keyword: keyword, KeywordPlural: keywordPlural} } + +func NewAIProxyManager(keyword, keywordPlural string, columns, adminColumns []string) modulebase.ResourceManager { + return modulebase.ResourceManager{ + BaseManager: *modulebase.NewBaseManager(apis.SERVICE_TYPE_AIPROXY, "", "", columns, adminColumns), + Keyword: keyword, KeywordPlural: keywordPlural} +} diff --git a/pkg/mcclient/options/aiproxy/doc.go b/pkg/mcclient/options/aiproxy/doc.go new file mode 100644 index 0000000000..f3a766a0c5 --- /dev/null +++ b/pkg/mcclient/options/aiproxy/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy // import "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy" diff --git a/pkg/mcclient/options/aiproxy/resources.go b/pkg/mcclient/options/aiproxy/resources.go new file mode 100644 index 0000000000..890fe82250 --- /dev/null +++ b/pkg/mcclient/options/aiproxy/resources.go @@ -0,0 +1,480 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func mergeJSONStringField(params *jsonutils.JSONDict, key, raw string) error { + if raw == "" { + return nil + } + obj, err := jsonutils.ParseString(raw) + if err != nil { + return errors.Wrapf(err, "parse %s", key) + } + params.Set(key, obj) + return nil +} + +// --- ai_provider --- + +type AiProviderListOptions struct { + options.BaseListOptions + + ProviderKey string `help:"filter by provider_key"` +} + +func (o *AiProviderListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiProviderShowOptions struct { + options.BaseShowOptions +} + +type AiProviderCreateOptions struct { + options.BaseCreateOptions + + ProviderKey string `help:"provider key (catalog identifier)" json:"provider_key"` + Config string `help:"provider config as JSON object string" json:"-"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiProviderCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(o).(*jsonutils.JSONDict) + params.Remove("config") + if err := mergeJSONStringField(params, "config", o.Config); err != nil { + return nil, err + } + return params, nil +} + +type AiProviderUpdateOptions struct { + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + ProviderKey string `json:"provider_key,omitempty"` + Config string `help:"provider config JSON" json:"-"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiProviderUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiProviderUpdateOptions) Params() (jsonutils.JSONObject, error) { + d, err := options.StructToParams(o) + if err != nil { + return nil, err + } + if err := mergeJSONStringField(d, "config", o.Config); err != nil { + return nil, err + } + return d, nil +} + +type AiProviderDeleteOptions struct { + options.BaseShowOptions +} + +// --- ai_model --- + +type AiModelListOptions struct { + options.BaseListOptions + + AiProviderId string `help:"filter by ai_provider_id" json:"ai_provider_id"` + ModelKey string `help:"filter by model_key" json:"model_key"` +} + +func (o *AiModelListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiModelShowOptions struct { + options.BaseShowOptions +} + +type AiModelCreateOptions struct { + options.BaseCreateOptions + + AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"` + ModelKey string `help:"model routing key" json:"model_key"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiModelCreateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type AiModelUpdateOptions struct { + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + AiProviderId string `json:"ai_provider_id,omitempty"` + ModelKey string `json:"model_key,omitempty"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiModelUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiModelUpdateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type AiModelDeleteOptions struct { + options.BaseShowOptions +} + +// --- ai_key --- + +type AiKeyListOptions struct { + options.BaseListOptions + + AiProviderId string `help:"filter by ai_provider_id" json:"ai_provider_id"` +} + +func (o *AiKeyListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiKeyShowOptions struct { + options.BaseShowOptions +} + +type AiKeyCreateOptions struct { + options.BaseCreateOptions + + AiProviderId string `help:"optional ai_provider id or name" json:"ai_provider_id"` + Secret string `help:"API key or secret material" json:"secret"` + Weight int `help:"load-balance weight among matching keys (default 1)" json:"weight,omitzero"` + Routing string `help:"routing JSON: allowed_model_keys, blocked_model_keys" json:"-"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiKeyCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(o).(*jsonutils.JSONDict) + params.Remove("routing") + if err := mergeJSONStringField(params, "routing", o.Routing); err != nil { + return nil, err + } + return params, nil +} + +type AiKeyUpdateOptions struct { + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + AiProviderId string `json:"ai_provider_id,omitempty"` + Secret string `json:"secret,omitempty"` + Weight int `json:"weight,omitzero"` + Routing string `json:"-"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiKeyUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiKeyUpdateOptions) Params() (jsonutils.JSONObject, error) { + d, err := options.StructToParams(o) + if err != nil { + return nil, err + } + d.Remove("routing") + if err := mergeJSONStringField(d, "routing", o.Routing); err != nil { + return nil, err + } + return d, nil +} + +type AiKeyDeleteOptions struct { + options.BaseShowOptions +} + +// --- ai_virtual_key --- + +type AiVirtualKeyListOptions struct { + options.BaseListOptions + + VirtualKey string `help:"filter by virtual_key" json:"virtual_key"` + UserId string `help:"filter by owner user id or name (admin)" json:"user_id"` +} + +func (o *AiVirtualKeyListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiVirtualKeyShowOptions struct { + options.BaseShowOptions +} + +type AiVirtualKeyCreateOptions struct { + options.BaseCreateOptions + + VirtualKey string `help:"optional client virtual key; auto-generated sk-... when omitted" json:"virtual_key"` + OwnerId string `help:"owner user id (admin); default current user" json:"owner_id,omitempty"` + Limits string `help:"limits JSON: allowed_ai_provider_ids, max_tokens_per_request, requests_per_minute" json:"-"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiVirtualKeyCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(o).(*jsonutils.JSONDict) + params.Remove("limits") + if err := mergeJSONStringField(params, "limits", o.Limits); err != nil { + return nil, err + } + return params, nil +} + +type AiVirtualKeyUpdateOptions struct { + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + VirtualKey string `json:"virtual_key,omitempty"` + OwnerId string `json:"owner_id,omitempty"` + Limits string `help:"limits JSON object" json:"-"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiVirtualKeyUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiVirtualKeyUpdateOptions) Params() (jsonutils.JSONObject, error) { + d, err := options.StructToParams(o) + if err != nil { + return nil, err + } + d.Remove("limits") + if err := mergeJSONStringField(d, "limits", o.Limits); err != nil { + return nil, err + } + return d, nil +} + +type AiVirtualKeyDeleteOptions struct { + options.BaseShowOptions +} + +// --- ai_routing --- + +type AiRoutingListOptions struct { + options.BaseListOptions + + ModelPattern string `json:"model_pattern"` + AiProxyNodeId string `json:"ai_proxy_node_id"` + Enabled *bool `help:"filter by enabled flag" json:"enabled"` +} + +func (o *AiRoutingListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiRoutingShowOptions struct { + options.BaseShowOptions +} + +type AiRoutingCreateOptions struct { + apis.SharableVirtualResourceCreateInput + + Priority int `json:"priority,omitzero"` + ModelPattern string `json:"model_pattern,omitempty"` + AiProxyNodeId string `json:"ai_proxy_node_id,omitempty"` + Models string `help:"routing models JSON array: [{ai_provider_id,ai_model_id,priority|weight,model_pattern?}]" json:"-"` + Enabled *bool `help:"turn on enabled flag" json:"enabled,omitempty"` + Disabled *bool `help:"turn off enabled flag" json:"disabled,omitempty"` +} + +func (o *AiRoutingCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(o).(*jsonutils.JSONDict) + if err := mergeJSONStringField(params, "models", o.Models); err != nil { + return nil, err + } + return params, nil +} + +type AiRoutingUpdateOptions struct { + apis.SharableVirtualResourceBaseUpdateInput + + ID string `help:"ID or name" json:"-"` + Priority int `json:"priority,omitzero"` + ModelPattern string `json:"model_pattern,omitempty"` + AiProxyNodeId string `json:"ai_proxy_node_id,omitempty"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiRoutingUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiRoutingUpdateOptions) Params() (jsonutils.JSONObject, error) { + d, err := options.StructToParams(o) + if err != nil { + return nil, err + } + baseParams, err := options.StructToParams(&o.SharableVirtualResourceBaseUpdateInput) + if err != nil { + return nil, err + } + if baseParams != nil { + d.Update(baseParams) + } + return d, nil +} + +type AiRoutingDeleteOptions struct { + options.BaseShowOptions +} + +type AiRoutingSetModelsOptions struct { + options.BaseIdOptions + + Models string `help:"routing models JSON array: [{ai_provider_id,ai_model_id,priority|weight,model_pattern?}]" json:"-"` +} + +func (o *AiRoutingSetModelsOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.NewDict() + if err := mergeJSONStringField(params, "models", o.Models); err != nil { + return nil, err + } + return params, nil +} + +// --- ai_routing_model --- + +type AiRoutingModelListOptions struct { + options.BaseListOptions + + AiRoutingId string `help:"filter by ai_routing id or name" json:"ai_routing_id"` + AiProviderId string `json:"ai_provider_id"` + AiModelId string `json:"ai_model_id"` +} + +func (o *AiRoutingModelListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiRoutingModelShowOptions struct { + options.BaseShowOptions +} + +type AiRoutingModelCreateOptions struct { + options.BaseCreateOptions + + AiRoutingId string `help:"parent ai_routing id or name" json:"ai_routing_id"` + AiProviderId string `help:"ai_provider id or name" json:"ai_provider_id"` + AiModelId string `help:"ai_model id or name" json:"ai_model_id"` + Priority int `help:"lower value = higher priority within routing" json:"priority,omitzero"` + ModelPattern string `help:"optional client model glob/prefix" json:"model_pattern,omitempty"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiRoutingModelCreateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type AiRoutingModelUpdateOptions struct { + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + AiRoutingId string `json:"ai_routing_id,omitempty"` + AiProviderId string `json:"ai_provider_id,omitempty"` + AiModelId string `json:"ai_model_id,omitempty"` + Priority int `json:"priority,omitzero"` + ModelPattern string `json:"model_pattern,omitempty"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiRoutingModelUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiRoutingModelUpdateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type AiRoutingModelDeleteOptions struct { + options.BaseShowOptions +} + +// --- ai_proxy_node --- + +type AiProxyNodeListOptions struct { + options.BaseListOptions + + Address string `help:"filter by address" json:"address"` + Domain string `help:"filter by domain" json:"domain"` +} + +func (o *AiProxyNodeListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type AiProxyNodeShowOptions struct { + options.BaseShowOptions +} + +type AiProxyNodeCreateOptions struct { + options.BaseCreateOptions + + Address string `help:"reachable base URL (https://host:port or host:port)" json:"address"` + Domain string `help:"optional hostname without scheme or port" json:"domain,omitempty"` + HbTimeout int `help:"heartbeat timeout in seconds (default 120)" json:"hb_timeout,omitzero"` + Enabled *bool `help:"turn on enabled flag" json:"enabled,omitempty"` +} + +func (o *AiProxyNodeCreateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type AiProxyNodeUpdateOptions struct { + ID string `help:"ID or name" json:"-"` + Name string `json:"name,omitempty"` + Desc string `json:"description,omitempty"` + Address string `help:"reachable base URL" json:"address,omitempty"` + Domain string `help:"hostname without scheme or port; empty string clears" json:"domain,omitempty"` + HbTimeout int `help:"heartbeat timeout in seconds" json:"hb_timeout,omitzero"` + Enabled *bool `json:"enabled,omitempty"` +} + +func (o *AiProxyNodeUpdateOptions) GetId() string { + return o.ID +} + +func (o *AiProxyNodeUpdateOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type AiProxyNodeDeleteOptions struct { + options.BaseShowOptions +} + +// AiProxyNodeRegisterOptions is used by standby instances (PerformClass register). +type AiProxyNodeRegisterOptions struct { + Address string `help:"instance address (https://host:port or host:port)" json:"address"` + HbTimeout int `help:"heartbeat timeout in seconds (default 120)" json:"hb_timeout,omitzero"` +} + +func (o *AiProxyNodeRegisterOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} diff --git a/pkg/mcclient/options/compute/servernetworks.go b/pkg/mcclient/options/compute/servernetworks.go index 1beee5bb49..1696afe4dc 100644 --- a/pkg/mcclient/options/compute/servernetworks.go +++ b/pkg/mcclient/options/compute/servernetworks.go @@ -16,6 +16,7 @@ package compute import ( "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient/options" ) diff --git a/scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh b/scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh new file mode 100755 index 0000000000..fe960b276c --- /dev/null +++ b/scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Test climc ai-provider-create: create a custom ai_provider and verify with show/list. +# +# Usage: +# source /etc/yunion/rcadmin +# bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh +# +# Non-interactive: +# export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1 +# export AIPROXY_PROVIDER_FT_NAME=my-custom-provider +# export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-custom-key +# export AIPROXY_PROVIDER_FT_BASE_URL=https://api.example.com/v1 +# bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh +# +# Or pass full config JSON: +# export AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"https://api.example.com"}' + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/test/aiproxy/aiproxy-functional-test-common.sh +source "${SCRIPT_DIR}/aiproxy-functional-test-common.sh" + +CLIMC_OUTPUT_FORMAT="${CLIMC_OUTPUT_FORMAT:-json}" +export CLIMC_OUTPUT_FORMAT + +aiproxy_ft_need_cmds + +PROVIDER_RESOURCE_NAME="${AIPROXY_PROVIDER_FT_NAME:-}" +PROVIDER_KEY="${AIPROXY_PROVIDER_FT_PROVIDER_KEY:-}" +BASE_URL="${AIPROXY_PROVIDER_FT_BASE_URL:-}" +CONFIG_JSON="${AIPROXY_PROVIDER_FT_CONFIG:-}" +ENABLED_FLAG="${AIPROXY_PROVIDER_FT_ENABLED:-true}" +DELETE_IF_EXISTS="${AIPROXY_PROVIDER_FT_DELETE_EXISTING:-}" + +prompt_line() { + local prompt="$1" default="${2:-}" varname="$3" + local value + if [[ -n "$default" ]]; then + echo -n "${prompt} [${default}]: " >&2 + else + echo -n "${prompt}: " >&2 + fi + if [[ ! -t 0 ]]; then + value="$default" + else + read -r value &2 + read -r ans &2 + echo "将创建自定义 ai_provider（provider_key 不可与 catalog 重复）。" >&2 + echo + + if [[ -z "$PROVIDER_RESOURCE_NAME" ]]; then + prompt_line "资源名称 (climc 第一个参数 NAME)" "aiproxy-provider-ft-${suffix}" PROVIDER_RESOURCE_NAME + fi + if [[ -z "$PROVIDER_KEY" ]]; then + prompt_line "provider_key (唯一标识)" "${PROVIDER_RESOURCE_NAME}" PROVIDER_KEY + fi + if [[ -z "$CONFIG_JSON" && -z "$BASE_URL" ]]; then + prompt_line "config.base_url (OpenAI 兼容上游)" "https://api.openai.com" BASE_URL + fi + if prompt_yes_no "创建后启用 (--enabled)?" 1; then + ENABLED_FLAG=true + else + ENABLED_FLAG=false + fi +} + +delete_existing_provider() { + local name="$1" + if ! climc_json ai-provider-show "$name" >/dev/null 2>&1; then + return 0 + fi + if [[ "$DELETE_IF_EXISTS" != "1" ]]; then + die "ai_provider $name already exists; set AIPROXY_PROVIDER_FT_DELETE_EXISTING=1 to delete first" + fi + echo "deleting existing ai_provider $name" >&2 + climc ai-provider-delete "$name" +} + +create_provider() { + local config enabled_args=() + config="$(build_config_json)" + if [[ "$ENABLED_FLAG" == "true" ]]; then + enabled_args=(--enabled) + fi + climc ai-provider-create \ + "$PROVIDER_RESOURCE_NAME" \ + --provider-key "$PROVIDER_KEY" \ + --config "$config" \ + "${enabled_args[@]}" +} + +verify_provider() { + local row pk base enabled + row="$(climc_json ai-provider-show "$PROVIDER_RESOURCE_NAME")" + pk="$(echo "$row" | jq -r '.provider_key // empty')" + base="$(echo "$row" | jq -r '.config.base_url // empty')" + enabled="$(echo "$row" | jq -r '.enabled // false')" + [[ "$pk" == "$PROVIDER_KEY" ]] || die "provider_key mismatch: got $pk want $PROVIDER_KEY" + if [[ -n "$BASE_URL" ]]; then + [[ "$base" == "$BASE_URL" ]] || die "base_url mismatch: got $base want $BASE_URL" + fi + [[ "$enabled" == "true" ]] || [[ "$ENABLED_FLAG" != "true" ]] || die "expected enabled=true" + echo "$row" | jq '{id, name, provider_key, enabled, config}' +} + +collect_inputs +delete_existing_provider "$PROVIDER_RESOURCE_NAME" + +aiproxy_ft_step "create ai_provider" +create_provider + +aiproxy_ft_step "verify ai-provider-show" +verify_provider + +aiproxy_ft_step "verify ai-provider-list filter" +cnt="$(climc_json ai-provider-list --provider-key "$PROVIDER_KEY" \ + | jq --arg n "$PROVIDER_RESOURCE_NAME" '[.data[] | select(.name == $n)] | length')" +[[ "$cnt" -ge 1 ]] || die "ai-provider-list --provider-key did not return created row" + +echo +echo "OK: ai_provider create test passed." +echo " name: $PROVIDER_RESOURCE_NAME" +echo " provider_key: $PROVIDER_KEY" +echo "Cleanup:" +echo " climc ai-provider-delete $PROVIDER_RESOURCE_NAME" diff --git a/scripts/test/aiproxy/aiproxy-functional-test-common.sh b/scripts/test/aiproxy/aiproxy-functional-test-common.sh new file mode 100755 index 0000000000..5bd3df0c9d --- /dev/null +++ b/scripts/test/aiproxy/aiproxy-functional-test-common.sh @@ -0,0 +1,396 @@ +# Shared helpers for aiproxy functional test scripts (source only, do not execute directly). + +die() { echo "ERROR: $*" >&2; exit 1; } + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing command: $1" +} + +aiproxy_ft_need_cmds() { + need_cmd climc + need_cmd curl + need_cmd jq +} + +climc_json() { + climc --output-format json "$@" +} + +aiproxy_ft_step() { echo; echo "==> $*"; } + +catalog_model_id() { + local provider="$1" model_key="$2" + echo "${provider}-${model_key}" +} + +default_model_for_provider() { + case "$1" in + aliyun) echo "qwen-turbo" ;; + xiaomi) echo "mimo-v2-flash" ;; + openai) echo "gpt-4o-mini" ;; + *) echo "" ;; + esac +} + +default_prompt_for_provider() { + case "$1" in + aliyun) echo "用一句话介绍通义千问" ;; + xiaomi) echo "用一句话介绍小米 MiMo" ;; + *) echo "用一句话介绍这个模型" ;; + esac +} + +resolve_aiproxy_url() { + if [[ -n "${AIPROXY_URL:-}" ]]; then + echo "${AIPROXY_URL%/}" + return + fi + local url + url="$(climc_json endpoint-list --service aiproxy --interface public --limit 1 \ + | jq -r '.data[0].url // empty')" + [[ -n "$url" ]] || die "cannot resolve aiproxy public URL; set AIPROXY_URL" + echo "${url%/}" +} + +list_catalog_provider_keys() { + climc_json ai-provider-list --limit 500 \ + | jq -r '.data[] | .provider_key // empty' | sed '/^$/d' | sort -u +} + +list_catalog_model_keys() { + local provider_key="$1" + climc_json ai-model-list --ai-provider-id "$provider_key" --limit 500 \ + | jq -r '.data[] | .model_key // empty' \ + | sed '/^$/d' | grep -vx 'default' | sort -u +} + +# Resolve API key from env (generic or provider-specific legacy names). +resolve_api_key_from_env() { + local provider_key="$1" + if [[ -n "${AIPROXY_FT_API_KEY:-}" ]]; then + echo "$AIPROXY_FT_API_KEY" + return + fi + case "$provider_key" in + aliyun) + [[ -n "${DASHSCOPE_API_KEY:-}" ]] && echo "$DASHSCOPE_API_KEY" && return + ;; + xiaomi) + [[ -n "${MIMO_API_KEY:-}" ]] && echo "$MIMO_API_KEY" && return + ;; + esac + return 1 +} + +prompt_api_key() { + local provider_key="$1" key + if key="$(resolve_api_key_from_env "$provider_key")"; then + echo "使用环境变量中的 API Key（未回显）" >&2 + echo "$key" + return + fi + if [[ ! -t 0 ]]; then + die "未设置 API Key：export AIPROXY_FT_API_KEY 或 ${provider_key} 对应的环境变量，或使用交互式终端" + fi + echo -n "请输入 ${provider_key} 的 API Key（不回显）: " >&2 + read -r -s key &2 + [[ -n "$key" ]] || die "API Key 不能为空" + echo "$key" +} + +# Populates global AIPROXY_FT_LINES[] (bash 3.2 compatible). +_load_lines_into_array() { + AIPROXY_FT_LINES=() + while IFS= read -r line; do + [[ -n "$line" ]] && AIPROXY_FT_LINES+=("$line") + done +} + +prompt_select_provider() { + local -a keys=() + local k i choice + _load_lines_into_array < <(list_catalog_provider_keys) + keys=("${AIPROXY_FT_LINES[@]}") + [[ ${#keys[@]} -gt 0 ]] || die "catalog 中无 ai_provider，请先执行 aiproxy master InitDB" + + if [[ -n "${AIPROXY_FT_PROVIDER:-}" ]]; then + for k in "${keys[@]}"; do + [[ "$k" == "${AIPROXY_FT_PROVIDER}" ]] && echo "${AIPROXY_FT_PROVIDER}" && return + done + die "ai_provider ${AIPROXY_FT_PROVIDER} 不在 catalog 中" + fi + + if [[ ! -t 0 ]]; then + die "请设置 AIPROXY_FT_PROVIDER 或在交互式终端运行" + fi + + echo "可用模型提供商 (catalog):" >&2 + for i in "${!keys[@]}"; do + printf ' [%d] %s\n' "$((i + 1))" "${keys[$i]}" >&2 + done + while true; do + echo -n "请选择序号 [1-${#keys[@]}] 或直接输入 provider_key: " >&2 + read -r choice = 1 && choice <= ${#keys[@]})); then + echo "${keys[$((choice - 1))]}" + return + fi + for k in "${keys[@]}"; do + [[ "$k" == "$choice" ]] && echo "$choice" && return + done + echo "无效选择，请重试。" >&2 + done +} + +prompt_select_model() { + local provider_key="$1" + local -a models=() + local m i choice default_m found + _load_lines_into_array < <(list_catalog_model_keys "$provider_key") + models=("${AIPROXY_FT_LINES[@]}") + [[ ${#models[@]} -gt 0 ]] || die "provider ${provider_key} 下无可用 model_key（catalog 未 seed？）" + + if [[ -n "${AIPROXY_FT_MODEL:-}" ]]; then + for m in "${models[@]}"; do + [[ "$m" == "${AIPROXY_FT_MODEL}" ]] && echo "${AIPROXY_FT_MODEL}" && return + done + die "model_key ${AIPROXY_FT_MODEL} 不在 provider ${provider_key} 的 catalog 中" + fi + + default_m="$(default_model_for_provider "$provider_key")" + found=0 + if [[ -n "$default_m" ]]; then + for m in "${models[@]}"; do + if [[ "$m" == "$default_m" ]]; then + found=1 + break + fi + done + fi + [[ "$found" -eq 1 ]] || default_m="${models[0]}" + + if [[ ! -t 0 ]]; then + echo "$default_m" + return + fi + + echo "提供商 ${provider_key} 的模型:" >&2 + for i in "${!models[@]}"; do + printf ' [%d] %s\n' "$((i + 1))" "${models[$i]}" >&2 + done + while true; do + echo -n "请选择序号 [1-${#models[@]}] 或输入 model_key [默认: ${default_m}]: " >&2 + read -r choice = 1 && choice <= ${#models[@]})); then + echo "${models[$((choice - 1))]}" + return + fi + for m in "${models[@]}"; do + [[ "$m" == "$choice" ]] && echo "$choice" && return + done + echo "无效选择，请重试。" >&2 + done +} + +prompt_run_stream() { + if [[ "${AIPROXY_FT_SKIP_STREAM:-}" == "1" ]]; then + return 1 + fi + if [[ "${AIPROXY_FT_SKIP_STREAM:-}" == "0" ]]; then + return 0 + fi + if [[ ! -t 0 ]]; then + return 0 + fi + local ans + echo -n "是否执行流式测试 (stream=true)? [Y/n]: " >&2 + read -r ans /dev/null 2>&1; then + echo "ai_key $key_name exists, syncing secret and ai_provider_id" + climc ai-key-update "$key_name" \ + --ai-provider-id "$provider_id" \ + --secret "$api_secret" \ + --weight 10 + else + climc ai-key-create \ + "$key_name" \ + --ai-provider-id "$provider_id" \ + --secret "$api_secret" \ + --weight 10 \ + --enabled + fi + ensure_ai_key_enabled "$key_name" +} + +verify_ai_key_for_provider() { + local provider_key="$1" + local provider_id count + provider_id="$(climc_json ai-provider-show "$provider_key" | jq -r '.id // empty')" + [[ -n "$provider_id" ]] || die "ai_provider $provider_key not found" + count="$(climc_json ai-key-list --ai-provider-id "$provider_id" | jq '[.data[] | select(.enabled == true)] | length')" + [[ "$count" -gt 0 ]] || die "no enabled ai_key bound to ai_provider_id=$provider_id" + echo "enabled ai_key rows for $provider_id: $count" +} + +ensure_virtual_key() { + local vk_name="$1" + if climc_json ai-virtual-key-show "$vk_name" >/dev/null 2>&1; then + echo "virtual key $vk_name already exists" + return + fi + climc ai-virtual-key-create "$vk_name" +} + +ensure_routing() { + local routing_name="$1" provider_key="$2" catalog_model_id="$3" + if climc_json ai-routing-show "$routing_name" >/dev/null 2>&1; then + echo "routing $routing_name already exists" + return + fi + climc ai-routing-create \ + "$routing_name" \ + --priority 10 \ + --models "[{\"ai_provider_id\":\"${provider_key}\",\"ai_model_id\":\"${catalog_model_id}\",\"priority\":1}]" +} + +verify_stream_chat() { + local base_url="$1" vk="$2" model="$3" out="$4" prompt="$5" + local http_code aggregated delta payload + + http_code="$(curl -k -sS -N -o "$out" -w '%{http_code}' \ + "${base_url%/}/openai/v1/chat/completions" \ + -H "Authorization: Bearer ${vk}" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"${model}\",\"stream\":true,\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rn --arg t "$prompt" '$t')}],\"max_tokens\":64}")" + + echo "HTTP $http_code (stream)" + [[ "$http_code" == "200" ]] || { + echo "--- stream body (first 40 lines) ---" >&2 + head -n 40 "$out" >&2 || true + die "stream chat failed with HTTP $http_code" + } + + aggregated="" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ "$line" == data:* ]] || continue + payload="${line#data: }" + payload="${payload//$'\r'/}" + [[ -z "$payload" ]] && continue + [[ "$payload" == "[DONE]" ]] && continue + if echo "$payload" | jq -e '.error // .message' >/dev/null 2>&1; then + echo "upstream error chunk: $payload" >&2 + die "stream returned error event" + fi + delta="$(echo "$payload" | jq -r '.choices[0].delta.content // empty' 2>/dev/null || true)" + aggregated+="$delta" + done <"$out" + + [[ -n "$aggregated" ]] || { + echo "--- stream body ---" >&2 + cat "$out" >&2 + die "empty aggregated stream content (no choices[0].delta.content)" + } + + echo "stream content (${#aggregated} chars): ${aggregated:0:120}..." +} + +# aiproxy_ft_run executes the full functional test for one provider/model/api key. +aiproxy_ft_run() { + local provider_key="$1" chat_model="$2" api_secret="$3" chat_prompt="$4" + local run_stream="${5:-1}" + + local key_name vk_name routing_name catalog_mid + local chat_resp chat_stream_resp aiproxy_url vk http_code content + + key_name="${AIPROXY_FT_KEY_NAME:-aiproxy-ft-${provider_key}}" + vk_name="${AIPROXY_FT_VK_NAME:-aiproxy-ft-${provider_key}-vk}" + routing_name="${AIPROXY_FT_ROUTING_NAME:-aiproxy-ft-${provider_key}-routing}" + chat_resp="${AIPROXY_FT_CHAT_RESP:-/tmp/aiproxy-ft-${provider_key}-chat.json}" + chat_stream_resp="${AIPROXY_FT_STREAM_RESP:-/tmp/aiproxy-ft-${provider_key}-chat-stream.sse}" + catalog_mid="$(catalog_model_id "$provider_key" "$chat_model")" + + echo + echo "=== aiproxy 功能测试 ===" + echo "provider: ${provider_key} model: ${chat_model} catalog_id: ${catalog_mid}" + echo "ai_key: ${key_name} virtual_key: ${vk_name} routing: ${routing_name}" + echo + + aiproxy_ft_step "1. Keystone aiproxy public endpoint" + aiproxy_url="$(resolve_aiproxy_url)" + echo "AIPROXY_URL=$aiproxy_url" + + aiproxy_ft_step "2. Catalog ${provider_key} / ${chat_model}" + climc ai-provider-show "$provider_key" >/dev/null \ + || die "ai_provider $provider_key missing; run aiproxy master InitDB first" + climc ai-model-show "$catalog_mid" >/dev/null \ + || die "ai_model $catalog_mid not in catalog (re-run aiproxy master InitDB)" + + aiproxy_ft_step "3. ai_key" + ensure_ai_key "$provider_key" "$key_name" "$api_secret" + verify_ai_key_for_provider "$provider_key" + + aiproxy_ft_step "4. ai_virtual_key" + ensure_virtual_key "$vk_name" + vk="$(climc_json ai-virtual-key-show "$vk_name" | jq -r '.virtual_key')" + [[ -n "$vk" ]] || die "empty virtual_key from ai-virtual-key-show" + echo "virtual_key=${vk:0:12}..." + + aiproxy_ft_step "5. ai_routing" + ensure_routing "$routing_name" "$provider_key" "$catalog_mid" + + aiproxy_ft_step "6. POST /openai/v1/chat/completions" + http_code="$(curl -k -sS -o "$chat_resp" -w '%{http_code}' \ + "${aiproxy_url}/openai/v1/chat/completions" \ + -H "Authorization: Bearer ${vk}" \ + -H "Content-Type: application/json" \ + -d "{\"model\":\"${chat_model}\",\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rn --arg t "$chat_prompt" '$t')}],\"max_tokens\":128}")" + + echo "HTTP $http_code" + jq . <"$chat_resp" + [[ "$http_code" == "200" ]] || die "chat request failed with HTTP $http_code" + + content="$(jq -r '.choices[0].message.content // empty' <"$chat_resp")" + [[ -n "$content" ]] || die "empty choices[0].message.content" + + if [[ "$run_stream" == "1" ]]; then + aiproxy_ft_step "7. POST /openai/v1/chat/completions (stream=true)" + verify_stream_chat "$aiproxy_url" "$vk" "$chat_model" "$chat_stream_resp" "$chat_prompt" + fi + + echo + echo "OK: aiproxy functional test passed for ${provider_key}/${chat_model} (non-stream$([[ "$run_stream" == "1" ]] && echo ' + stream' || echo ''))." + echo "Cleanup (optional):" + echo " climc ai-routing-delete $routing_name" + echo " climc ai-virtual-key-delete $vk_name" + echo " climc ai-key-delete $key_name" +} diff --git a/scripts/test/aiproxy/aiproxy-functional-test-mimo.sh b/scripts/test/aiproxy/aiproxy-functional-test-mimo.sh new file mode 100755 index 0000000000..91dbd16fba --- /dev/null +++ b/scripts/test/aiproxy/aiproxy-functional-test-mimo.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Legacy entry: pre-select xiaomi provider, then run interactive / env-based test. +# Prefer: bash scripts/test/aiproxy/aiproxy-functional-test.sh + +set -euo pipefail + +export AIPROXY_FT_PROVIDER="${AIPROXY_FT_PROVIDER:-xiaomi}" +[[ -n "${MIMO_API_KEY:-}" && -z "${AIPROXY_FT_API_KEY:-}" ]] && export AIPROXY_FT_API_KEY="${MIMO_API_KEY}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec bash "${SCRIPT_DIR}/aiproxy-functional-test.sh" "$@" diff --git a/scripts/test/aiproxy/aiproxy-functional-test-qwen.sh b/scripts/test/aiproxy/aiproxy-functional-test-qwen.sh new file mode 100755 index 0000000000..7cb818fcd6 --- /dev/null +++ b/scripts/test/aiproxy/aiproxy-functional-test-qwen.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Legacy entry: pre-select aliyun (通义千问) provider, then run interactive / env-based test. +# Prefer: bash scripts/test/aiproxy/aiproxy-functional-test.sh + +set -euo pipefail + +export AIPROXY_FT_PROVIDER="${AIPROXY_FT_PROVIDER:-aliyun}" +[[ -n "${DASHSCOPE_API_KEY:-}" && -z "${AIPROXY_FT_API_KEY:-}" ]] && export AIPROXY_FT_API_KEY="${DASHSCOPE_API_KEY}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec bash "${SCRIPT_DIR}/aiproxy-functional-test.sh" "$@" diff --git a/scripts/test/aiproxy/aiproxy-functional-test.sh b/scripts/test/aiproxy/aiproxy-functional-test.sh new file mode 100755 index 0000000000..6dc42411f8 --- /dev/null +++ b/scripts/test/aiproxy/aiproxy-functional-test.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# aiproxy interactive functional test: select catalog provider/model, enter API key, run chat + stream. +# +# Usage: +# source /etc/yunion/rcadmin +# bash scripts/test/aiproxy/aiproxy-functional-test.sh +# +# Non-interactive (CI): +# export AIPROXY_FT_PROVIDER=aliyun +# export AIPROXY_FT_MODEL=qwen-turbo +# export AIPROXY_FT_API_KEY='...' +# export AIPROXY_FT_NONINTERACTIVE=1 +# bash scripts/test/aiproxy/aiproxy-functional-test.sh +# +# Legacy env (still supported): +# DASHSCOPE_API_KEY + AIPROXY_FT_PROVIDER=aliyun +# MIMO_API_KEY + AIPROXY_FT_PROVIDER=xiaomi + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/test/aiproxy/aiproxy-functional-test-common.sh +source "${SCRIPT_DIR}/aiproxy-functional-test-common.sh" + +CLIMC_OUTPUT_FORMAT="${CLIMC_OUTPUT_FORMAT:-json}" +export CLIMC_OUTPUT_FORMAT + +aiproxy_ft_need_cmds + +if [[ "${AIPROXY_FT_NONINTERACTIVE:-}" == "1" && -z "${AIPROXY_FT_PROVIDER:-}" ]]; then + die "AIPROXY_FT_NONINTERACTIVE=1 时需设置 AIPROXY_FT_PROVIDER" +fi + +PROVIDER_KEY="$(prompt_select_provider)" +CHAT_MODEL="$(prompt_select_model "$PROVIDER_KEY")" +API_SECRET="$(prompt_api_key "$PROVIDER_KEY")" +CHAT_PROMPT="${AIPROXY_FT_PROMPT:-$(default_prompt_for_provider "$PROVIDER_KEY")}" + +RUN_STREAM=1 +if prompt_run_stream; then + RUN_STREAM=1 +else + RUN_STREAM=0 +fi + +aiproxy_ft_run "$PROVIDER_KEY" "$CHAT_MODEL" "$API_SECRET" "$CHAT_PROMPT" "$RUN_STREAM"