From 167682c39b482a5164707b81d61534d9438253e4 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 7 Jul 2026 20:13:14 +0800 Subject: [PATCH] feat(aiproxy): add Anthropic Messages API and migrate functional tests to Go (#25107) Add /v1/messages handler with Anthropic-to-OpenAI translation, upstream failover, and probe endpoints. Replace shell-based functional test scripts with pkg/aiproxy/ft and climc test commands; consolidate documentation. --- cmd/climc/shell/aiproxy/ai_provider.go | 1 + cmd/climc/shell/aiproxy/common.go | 21 + cmd/climc/shell/aiproxy/test_anthropic.go | 26 + cmd/climc/shell/aiproxy/test_chat.go | 26 + .../shell/aiproxy/test_provider_create.go | 26 + docs/aiproxy/functional-test-climc-mimo.md | 114 ---- docs/aiproxy/functional-test-climc.md | 308 ----------- docs/aiproxy/functional-test.md | 369 +++++++++++++ pkg/aiproxy/ft/admin.go | 242 +++++++++ pkg/aiproxy/ft/anthropic.go | 158 ++++++ pkg/aiproxy/ft/catalog.go | 169 ++++++ pkg/aiproxy/ft/chat.go | 166 ++++++ pkg/aiproxy/ft/cleanup.go | 213 ++++++++ pkg/aiproxy/ft/doc.go | 16 + pkg/aiproxy/ft/endpoint.go | 51 ++ pkg/aiproxy/ft/env.go | 89 +++ pkg/aiproxy/ft/ft_test.go | 121 +++++ pkg/aiproxy/ft/httpclient.go | 92 ++++ pkg/aiproxy/ft/interactive.go | 241 +++++++++ pkg/aiproxy/ft/options.go | 55 ++ pkg/aiproxy/ft/provider_create.go | 203 +++++++ pkg/aiproxy/ft/sse.go | 165 ++++++ pkg/aiproxy/handlers/anthropic_probe.go | 47 ++ pkg/aiproxy/handlers/anthropic_probe_test.go | 63 +++ pkg/aiproxy/handlers/chat_completions.go | 101 +--- pkg/aiproxy/handlers/handlers.go | 5 + pkg/aiproxy/handlers/messages.go | 379 +++++++++++++ pkg/aiproxy/handlers/messages_debug.go | 148 +++++ pkg/aiproxy/handlers/upstream_failover.go | 167 ++++++ pkg/aiproxy/models/ai_key_resolve.go | 11 +- pkg/aiproxy/models/ai_providers.go | 116 +++- pkg/aiproxy/models/ai_providers_test.go | 79 +++ pkg/aiproxy/models/ai_routing_models.go | 2 + pkg/aiproxy/models/ai_routings.go | 11 + .../models/aiproxy_catalog_validate.go | 42 +- pkg/aiproxy/models/catalog_seed.go | 180 ++++--- pkg/aiproxy/models/catalog_seed_models.go | 229 ++++---- pkg/aiproxy/models/catalog_seed_test.go | 34 ++ pkg/aiproxy/models/chat_upstream.go | 10 +- pkg/aiproxy/models/provider_connectivity.go | 219 ++++++++ pkg/aiproxy/providerapi/types.go | 22 + pkg/aiproxy/providers/aliyun/aliyun.go | 3 +- pkg/aiproxy/providers/anthropic/anthropic.go | 3 +- .../providers/anthropic/openai_bridge.go | 22 + .../providers/anthropic/openai_bridge_test.go | 34 ++ pkg/aiproxy/providers/azure/azure.go | 3 +- pkg/aiproxy/providers/baidu/baidu.go | 5 +- pkg/aiproxy/providers/chat_provider.go | 39 ++ pkg/aiproxy/providers/cohere/cohere.go | 3 +- pkg/aiproxy/providers/embeddings_test.go | 121 ----- pkg/aiproxy/providers/gemini/gemini.go | 3 +- pkg/aiproxy/providers/images_test.go | 7 +- pkg/aiproxy/providers/messages/doc.go | 1 + .../providers/messages/messages_test.go | 164 ++++++ pkg/aiproxy/providers/messages/passthrough.go | 74 +++ pkg/aiproxy/providers/messages/registry.go | 46 ++ pkg/aiproxy/providers/messages/translation.go | 96 ++++ .../providers/openai/anthropic_compat.go | 508 ++++++++++++++++++ .../providers/openai/anthropic_compat_test.go | 347 ++++++++++++ .../openai/anthropic_stream_compat.go | 340 ++++++++++++ pkg/aiproxy/providers/providers_test.go | 4 +- pkg/aiproxy/providers/registry.go | 37 +- pkg/aiproxy/providers/types.go | 18 +- pkg/aiproxy/providers/vllm/vllm.go | 3 +- pkg/aiproxy/upstream/openai_compat.go | 152 ++++++ .../upstream/openai_compat_models_test.go | 66 +++ pkg/apis/aiproxy/ai_provider.go | 84 ++- pkg/apis/aiproxy/ai_provider_config_test.go | 97 ++++ pkg/apis/aiproxy/ai_provider_secret_test.go | 37 ++ pkg/apis/aiproxy/ai_routing_model.go | 5 + pkg/apis/aiproxy/provider_defaults.go | 51 ++ pkg/apis/aiproxy/provider_keys.go | 147 +++++ pkg/llm/models/llm_aiproxy_sync.go | 5 +- pkg/llm/models/llm_aiproxy_sync_test.go | 183 ------- pkg/mcclient/options/aiproxy/resources.go | 15 + .../aiproxy-ai-provider-create-test.sh | 171 ------ .../aiproxy/aiproxy-functional-test-common.sh | 396 -------------- .../aiproxy/aiproxy-functional-test-mimo.sh | 11 - .../aiproxy/aiproxy-functional-test-qwen.sh | 11 - .../test/aiproxy/aiproxy-functional-test.sh | 46 -- 80 files changed, 6390 insertions(+), 1705 deletions(-) create mode 100644 cmd/climc/shell/aiproxy/common.go create mode 100644 cmd/climc/shell/aiproxy/test_anthropic.go create mode 100644 cmd/climc/shell/aiproxy/test_chat.go create mode 100644 cmd/climc/shell/aiproxy/test_provider_create.go delete mode 100644 docs/aiproxy/functional-test-climc-mimo.md delete mode 100644 docs/aiproxy/functional-test-climc.md create mode 100644 docs/aiproxy/functional-test.md create mode 100644 pkg/aiproxy/ft/admin.go create mode 100644 pkg/aiproxy/ft/anthropic.go create mode 100644 pkg/aiproxy/ft/catalog.go create mode 100644 pkg/aiproxy/ft/chat.go create mode 100644 pkg/aiproxy/ft/cleanup.go create mode 100644 pkg/aiproxy/ft/doc.go create mode 100644 pkg/aiproxy/ft/endpoint.go create mode 100644 pkg/aiproxy/ft/env.go create mode 100644 pkg/aiproxy/ft/ft_test.go create mode 100644 pkg/aiproxy/ft/httpclient.go create mode 100644 pkg/aiproxy/ft/interactive.go create mode 100644 pkg/aiproxy/ft/options.go create mode 100644 pkg/aiproxy/ft/provider_create.go create mode 100644 pkg/aiproxy/ft/sse.go create mode 100644 pkg/aiproxy/handlers/anthropic_probe.go create mode 100644 pkg/aiproxy/handlers/anthropic_probe_test.go create mode 100644 pkg/aiproxy/handlers/messages.go create mode 100644 pkg/aiproxy/handlers/messages_debug.go create mode 100644 pkg/aiproxy/handlers/upstream_failover.go create mode 100644 pkg/aiproxy/models/ai_providers_test.go create mode 100644 pkg/aiproxy/models/catalog_seed_test.go create mode 100644 pkg/aiproxy/models/provider_connectivity.go create mode 100644 pkg/aiproxy/providers/anthropic/openai_bridge.go create mode 100644 pkg/aiproxy/providers/anthropic/openai_bridge_test.go create mode 100644 pkg/aiproxy/providers/chat_provider.go delete mode 100644 pkg/aiproxy/providers/embeddings_test.go create mode 100644 pkg/aiproxy/providers/messages/doc.go create mode 100644 pkg/aiproxy/providers/messages/messages_test.go create mode 100644 pkg/aiproxy/providers/messages/passthrough.go create mode 100644 pkg/aiproxy/providers/messages/registry.go create mode 100644 pkg/aiproxy/providers/messages/translation.go create mode 100644 pkg/aiproxy/providers/openai/anthropic_compat.go create mode 100644 pkg/aiproxy/providers/openai/anthropic_compat_test.go create mode 100644 pkg/aiproxy/providers/openai/anthropic_stream_compat.go create mode 100644 pkg/aiproxy/upstream/openai_compat_models_test.go create mode 100644 pkg/apis/aiproxy/ai_provider_config_test.go create mode 100644 pkg/apis/aiproxy/ai_provider_secret_test.go create mode 100644 pkg/apis/aiproxy/provider_defaults.go create mode 100644 pkg/apis/aiproxy/provider_keys.go delete mode 100644 pkg/llm/models/llm_aiproxy_sync_test.go delete mode 100755 scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh delete mode 100755 scripts/test/aiproxy/aiproxy-functional-test-common.sh delete mode 100755 scripts/test/aiproxy/aiproxy-functional-test-mimo.sh delete mode 100755 scripts/test/aiproxy/aiproxy-functional-test-qwen.sh delete mode 100755 scripts/test/aiproxy/aiproxy-functional-test.sh diff --git a/cmd/climc/shell/aiproxy/ai_provider.go b/cmd/climc/shell/aiproxy/ai_provider.go index b18e17fccb..b6eb329220 100644 --- a/cmd/climc/shell/aiproxy/ai_provider.go +++ b/cmd/climc/shell/aiproxy/ai_provider.go @@ -27,5 +27,6 @@ func init() { cmd.Show(new(apoptions.AiProviderShowOptions)) cmd.Update(new(apoptions.AiProviderUpdateOptions)) cmd.Delete(new(apoptions.AiProviderDeleteOptions)) + cmd.PerformClass("test-connectivity", new(apoptions.AiProviderTestConnectivityOptions)) registerEnableDisable(cmd) } diff --git a/cmd/climc/shell/aiproxy/common.go b/cmd/climc/shell/aiproxy/common.go new file mode 100644 index 0000000000..1a2a053c7d --- /dev/null +++ b/cmd/climc/shell/aiproxy/common.go @@ -0,0 +1,21 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/cmd/climc/shell" +) + +var R = shell.R diff --git a/cmd/climc/shell/aiproxy/test_anthropic.go b/cmd/climc/shell/aiproxy/test_anthropic.go new file mode 100644 index 0000000000..c4dd9926fd --- /dev/null +++ b/cmd/climc/shell/aiproxy/test_anthropic.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/ft" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + R(&ft.AnthropicOptions{}, "aiproxy-test-anthropic", "Run aiproxy Anthropic Messages API E2E test", func(s *mcclient.ClientSession, args *ft.AnthropicOptions) error { + return ft.RunAnthropicTest(s, args) + }) +} diff --git a/cmd/climc/shell/aiproxy/test_chat.go b/cmd/climc/shell/aiproxy/test_chat.go new file mode 100644 index 0000000000..590ec94270 --- /dev/null +++ b/cmd/climc/shell/aiproxy/test_chat.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/ft" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + R(&ft.ChatOptions{}, "aiproxy-test-chat", "Run aiproxy OpenAI chat E2E test", func(s *mcclient.ClientSession, args *ft.ChatOptions) error { + return ft.RunChatTest(s, args) + }) +} diff --git a/cmd/climc/shell/aiproxy/test_provider_create.go b/cmd/climc/shell/aiproxy/test_provider_create.go new file mode 100644 index 0000000000..bd4fdf3667 --- /dev/null +++ b/cmd/climc/shell/aiproxy/test_provider_create.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/ft" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + R(&ft.ProviderCreateOptions{}, "aiproxy-test-provider-create", "Create custom ai_provider and verify", func(s *mcclient.ClientSession, args *ft.ProviderCreateOptions) error { + return ft.RunProviderCreateTest(s, args) + }) +} diff --git a/docs/aiproxy/functional-test-climc-mimo.md b/docs/aiproxy/functional-test-climc-mimo.md deleted file mode 100644 index 1a2a356260..0000000000 --- a/docs/aiproxy/functional-test-climc-mimo.md +++ /dev/null @@ -1,114 +0,0 @@ -# aiproxy 功能测试（climc + 小米 MiMo） - -本文用 **climc** 配置 aiproxy 资源，并通过 **curl** 调用 `POST /v1/chat/completions` 验证 **xiaomi** catalog（`api.xiaomimimo.com`）。 - -> **安全**：请勿将 MiMo API Key 写入脚本或提交到 Git。使用环境变量 `MIMO_API_KEY`。若 Key 曾泄露，请到小米开放平台轮换。 - -通义千问（DashScope）测试见 [functional-test-climc.md](./functional-test-climc.md)。 - -## 前置条件 - -| 项 | 说明 | -|----|------| -| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` public endpoint | -| 数据库 | 主节点已执行 `InitDB`，catalog 含 `xiaomi` 及 `mimo-*` 模型 | -| 客户端 | 已 `source /etc/yunion/rcadmin`，`climc` 可用 | -| 工具 | `jq` | -| 网络 | aiproxy 节点能访问 `https://api.xiaomimimo.com` | - -```bash -source /etc/yunion/rcadmin -export CLIMC_OUTPUT_FORMAT=json -bash scripts/test/aiproxy/aiproxy-functional-test.sh -# 交互菜单中选择 xiaomi，输入 MiMo API Key -``` - -或快捷入口（默认选中 `xiaomi`）： - -```bash -export MIMO_API_KEY='你的 MiMo API Key' -bash scripts/test/aiproxy/aiproxy-functional-test-mimo.sh -``` - -预置模型：`export AIPROXY_FT_PROVIDER=xiaomi AIPROXY_FT_MODEL=mimo-v2.5-pro` - -按 provider 自动命名的资源（可覆盖 `AIPROXY_FT_KEY_NAME` 等）： - -| 变量 | 默认（xiaomi） | -|------|----------------| -| `AIPROXY_FT_KEY_NAME` | `aiproxy-ft-xiaomi` | -| `AIPROXY_FT_VK_NAME` | `aiproxy-ft-xiaomi-vk` | -| `AIPROXY_FT_ROUTING_NAME` | `aiproxy-ft-xiaomi-routing` | -| `AIPROXY_FT_MODEL` | 交互默认 `mimo-v2-flash` | - -## 手动步骤摘要 - -### Catalog - -```bash -climc ai-provider-show xiaomi -climc ai-model-show xiaomi-mimo-v2-flash -``` - -`config.base_url` 应为 `https://api.xiaomimimo.com`。 - -### ai_key - -```bash -climc ai-key-create mimo-ft \ - --ai-provider-id xiaomi \ - --secret "${MIMO_API_KEY}" \ - --weight 10 \ - --enabled -``` - -### 路由与 chat - -```bash -climc ai-virtual-key-create aiproxy-mimo-ft-vk -climc ai-routing-create aiproxy-mimo-ft-routing \ - --priority 10 \ - --models '[{"ai_provider_id":"xiaomi","ai_model_id":"mimo-v2-flash","priority":1}]' - -AIPROXY_URL="$(climc endpoint-list --service aiproxy --interface public --limit 1 \ - --output-format json | jq -r '.data[0].url')" -VK="$(climc ai-virtual-key-show aiproxy-mimo-ft-vk --output-format json | jq -r '.virtual_key')" - -curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \ - -H "Authorization: Bearer ${VK}" \ - -H "Content-Type: application/json" \ - -d '{"model":"mimo-v2-flash","messages":[{"role":"user","content":"hi"}],"max_tokens":64}' | jq . -``` - -### 流式 - -`scripts/test/aiproxy/aiproxy-functional-test-mimo.sh` 在非流式通过后默认执行 step 7（`stream: true`）。跳过：`export AIPROXY_FT_SKIP_STREAM=1`。 - -```bash -curl -k -sS -N -o /tmp/aiproxy-mimo-stream.sse \ - "${AIPROXY_URL%/}/v1/chat/completions" \ - -H "Authorization: Bearer ${VK}" \ - -H "Content-Type: application/json" \ - -d '{"model":"mimo-v2-flash","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' -``` - -catalog 中其它模型：`mimo-v2.5-pro`、`mimo-v2-pro`、`mimo-v2.5`、`mimo-v2-omni`（id 形如 `xiaomi-mimo-v2.5-pro`）。 - -## 常见问题 - -**上游 401** -检查 `MIMO_API_KEY` 是否有效；确认 `ai_key` 已 `--enabled` 且 `ai_provider_id=xiaomi`。 - -**`no ai_routing matched`** -virtual key 与 routing 须在同一 climc 项目下创建。 - -**与 DashScope 脚本冲突** -MiMo 脚本使用独立的 vk/routing/key 名称；勿与 `aiproxy-ft-vk` 混用同一 routing 的 model 列表。 - -## 清理 - -```bash -climc ai-routing-delete aiproxy-mimo-ft-routing -climc ai-virtual-key-delete aiproxy-mimo-ft-vk -climc ai-key-delete mimo-ft -``` diff --git a/docs/aiproxy/functional-test-climc.md b/docs/aiproxy/functional-test-climc.md deleted file mode 100644 index 9f59426c89..0000000000 --- a/docs/aiproxy/functional-test-climc.md +++ /dev/null @@ -1,308 +0,0 @@ -# aiproxy 功能测试（climc + 通义千问 DashScope） - -本文用 **climc** 配置 aiproxy 资源，并用 **curl** 调用 OpenAI 兼容接口 `POST /v1/chat/completions` 做端到端验证。 - -> **安全**：请勿把 DashScope API Key 写进脚本、文档或提交到 Git。在 shell 里用环境变量 `DASHSCOPE_API_KEY` 传入。若 Key 曾在聊天/工单中泄露，请到阿里云控制台轮换。 - -## 前置条件 - -| 项 | 说明 | -|----|------| -| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` 服务及 public endpoint | -| 数据库 | 主节点已执行 `InitDB`，catalog 中已有 `aliyun` provider 及 `qwen-*` 模型（首次启动 master 会自动 seed） | -| 客户端 | 已 `source /etc/yunion/rcadmin`（或等价 rc 文件），`climc` 能正常 list | -| 工具 | `jq`（脚本与下文 curl 示例用于解析 JSON） | -| 网络 | aiproxy 节点能访问 `https://dashscope.aliyuncs.com` | - -### 一键脚本（交互式，推荐） - -从 catalog 选择 **模型提供商** 与 **model_key**，终端输入 API Key（或使用环境变量跳过输入），自动完成非流式 + 流式 chat： - -```bash -source /etc/yunion/rcadmin -export CLIMC_OUTPUT_FORMAT=json -bash scripts/test/aiproxy/aiproxy-functional-test.sh -``` - -也可预置后减少交互（仍会选择模型、是否流式，除非全部用环境变量）： - -```bash -export DASHSCOPE_API_KEY='你的 DashScope API Key' # 或 AIPROXY_FT_API_KEY -export AIPROXY_FT_PROVIDER=aliyun -export AIPROXY_FT_MODEL=qwen-turbo -bash scripts/test/aiproxy/aiproxy-functional-test.sh -``` - -通义快捷入口：`bash scripts/test/aiproxy/aiproxy-functional-test-qwen.sh`（默认 `aliyun`）。 -小米 MiMo 见 [functional-test-climc-mimo.md](./functional-test-climc-mimo.md)（`aiproxy-functional-test-mimo.sh`）。 - -非交互（CI）： - -```bash -export AIPROXY_FT_NONINTERACTIVE=1 -export AIPROXY_FT_PROVIDER=aliyun -export AIPROXY_FT_MODEL=qwen-turbo -export AIPROXY_FT_API_KEY='...' -export AIPROXY_FT_SKIP_STREAM=1 # 可选，跳过流式 -bash scripts/test/aiproxy/aiproxy-functional-test.sh -``` - -| 环境变量 | 说明 | -|----------|------| -| `AIPROXY_FT_PROVIDER` | `provider_key`（如 `aliyun`、`xiaomi`） | -| `AIPROXY_FT_MODEL` | `model_key`（如 `qwen-turbo`） | -| `AIPROXY_FT_API_KEY` | 上游 API Key（通用） | -| `DASHSCOPE_API_KEY` / `MIMO_API_KEY` | 按提供商兼容的旧变量名 | -| `AIPROXY_FT_SKIP_STREAM` | `1` 跳过流式；`0` 强制流式 | -| `AIPROXY_URL` | 留空则从 endpoint-list 解析 | - -## 测试流程概览 - -```mermaid -flowchart LR - VK[ai_virtual_key] --> RT[ai_routing] - RT --> RM[ai_routing_model] - RM --> P[ai_provider aliyun] - RM --> M[ai_model qwen-turbo] - P --> K[ai_key secret] - K --> DS[DashScope API] -``` - -## 0a. ai_provider 创建测试脚本 - -自定义 provider（非 catalog seed）创建与校验： - -```bash -source /etc/yunion/rcadmin -bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh -``` - -交互输入：资源名、`provider_key`、`base_url`、是否 `--enabled`。非交互示例： - -```bash -export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1 -export AIPROXY_PROVIDER_FT_NAME=my-vllm -export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-vllm -export AIPROXY_PROVIDER_FT_BASE_URL=http://127.0.0.1:8000/v1 -bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh -``` - -`provider_key` 须全局唯一；与 InitDB catalog（如 `aliyun`）重复会失败。完整 config 可用 `AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"..."}'`。 - -## 0. ai_proxy_node（多副本 / 路由绑定） - -列出 aiproxy 实例节点（InitDB 后默认有 `primary`）： - -```bash -climc ai-proxy-node-list -climc ai-proxy-node-show primary -``` - -注册 standby 节点（与进程内 `register` 心跳相同，一般由 standby 自动调用；手工测试可用）： - -```bash -climc ai-proxy-node-register --address https://standby-host:30938 --hb-timeout 120 -``` - -手工创建/更新节点（需具备写权限策略）： - -```bash -climc ai-proxy-node-create standby-1 \ - --address https://standby-host:30938 \ - --access-address https://aiproxy-standby.example.com:443 \ - --hb-timeout 120 \ - --enabled - -climc ai-proxy-node-update primary --address https://primary-host:30938 --access-address https://aiproxy.example.com:443 -climc ai-proxy-node-enable primary -climc ai-proxy-node-disable -``` - -将 `ai_routing` 绑定到指定节点（chat 须走该节点 public endpoint）： - -```bash -climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary -``` - -创建 `ai_routing` 时若省略 `--ai-proxy-node-id`，默认绑定 `primary` 节点；更新时显式传空值仍可清空绑定（任意 aiproxy 节点均可匹配）。 - -## 1. 检查 Keystone endpoint - -```bash -climc endpoint-list --service aiproxy --interface public -``` - -应能看到当前 region 的 public URL（脚本会取第一条用于 curl）。 - -## 2. 检查 catalog（InitDB seed） - -```bash -climc ai-provider-list --provider-key aliyun -climc ai-provider-show aliyun -climc ai-model-list --ai-provider-id aliyun --model-key qwen-turbo -``` - -确认 `provider_key=aliyun`，`config.base_url` 含 `https://dashscope.aliyuncs.com/compatible-mode`，且存在模型 `model_key=qwen-turbo`（catalog 固定 id / name：`aliyun-qwen-turbo`）。 - -## 3. 注册上游 API Key（ai_key） - -将 DashScope Key 存为 `ai_key`，供 chat 时按 provider 加权选取： - -```bash -climc ai-key-create qwen-dashscope-ft \ - --ai-provider-id aliyun \ - --secret "${DASHSCOPE_API_KEY}" \ - --weight 10 \ - --enabled -``` - -校验： - -```bash -climc ai-key-list --ai-provider-id aliyun -climc ai-key-show qwen-dashscope-ft -``` - -确认 `ai_provider_id` 为 `aliyun`，且 **`enabled=true`**（`ai_key` 默认 disabled，创建时需 `--enabled`；若已存在但被禁用，执行 `climc ai-key-enable qwen-dashscope-ft`）。若曾用错误参数创建过同名 key，可更新： - -```bash -climc ai-key-update qwen-dashscope-ft \ - --ai-provider-id aliyun \ - --secret "${DASHSCOPE_API_KEY}" \ - --weight 10 -climc ai-key-enable qwen-dashscope-ft -``` - -（`secret` 在 API 中通常不回显，仅用于上游调用。） - -## 4. 创建 Virtual Key（客户端鉴权） - -```bash -climc ai-virtual-key-create aiproxy-ft-vk -``` - -记下返回的 `virtual_key`（形如 `sk-...`）。查看： - -```bash -climc ai-virtual-key-list -climc ai-virtual-key-show aiproxy-ft-vk -``` - -Virtual key 归属当前 climc 用户的 **项目**；后续 `ai_routing` 须在同一项目（或共享到该项目）下。 - -## 5. 创建项目路由（ai_routing + models） - -将项目内请求 `model=qwen-turbo` 指到 catalog 的 aliyun/qwen-turbo。 - -**精确匹配**：`--model-key qwen-turbo` 与请求 body 中 `model` 完全一致时命中（优先于 `--model-pattern` 通配规则）。 - -`models` 里 `ai_model_id` 使用 catalog 固定 id（与 name 相同，如 `aliyun-qwen-turbo`），或在指定 `ai_provider_id` 时也可填 **model_key**（如 `qwen-turbo`）： - -```bash -climc ai-routing-create aiproxy-ft-routing \ - --priority 10 \ - --model-key qwen-turbo \ - --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-turbo","priority":1}]' -``` - -或手工指定 name： - -```bash -climc ai-routing-create aiproxy-ft-routing \ - --priority 10 \ - --models '[{"ai_provider_id":"aliyun","ai_model_id":"aliyun-qwen-turbo","priority":1}]' -``` - -查看绑定模型： - -```bash -climc ai-routing-show aiproxy-ft-routing -``` - -也可事后调整： - -```bash -climc ai-routing-set-models aiproxy-ft-routing \ - --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-plus","priority":1}]' -``` - -可选：将规则绑定到指定 aiproxy 实例（多副本时）： - -```bash -# 仅当需要固定到 primary 等节点时 -climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary -``` - -## 6. Chat completions（curl） - -climc 暂无 chat 子命令，用 public endpoint + virtual key 调用（`-k` 跳过 TLS 证书校验，适用于自签或内网 HTTPS）： - -```bash -AIPROXY_URL="${AIPROXY_URL:-$(climc endpoint-list --service aiproxy --interface public --limit 1 \ - --output-format json | jq -r '.data[0].url // empty')}" - -VK="$(climc ai-virtual-key-show aiproxy-ft-vk --output-format json \ - | jq -r '.virtual_key')" - -curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \ - -H "Authorization: Bearer ${VK}" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "qwen-turbo", - "messages": [{"role": "user", "content": "用一句话介绍通义千问"}], - "max_tokens": 128 - }' | jq . -``` - -**期望**：HTTP 200，JSON 含 `choices[0].message.content` 及 `usage`。 - -## 6b. 流式 Chat（curl / 脚本 step 7） - -一键脚本在步骤 6 非流式成功后，默认继续执行流式校验（聚合 `choices[0].delta.content`）。跳过流式： - -```bash -export AIPROXY_FT_SKIP_STREAM=1 -``` - -手动 curl（SSE，`data: [DONE]` 结束）： - -```bash -curl -k -sS -N -o /tmp/aiproxy-ft-stream.sse \ - "${AIPROXY_URL%/}/v1/chat/completions" \ - -H "Authorization: Bearer ${VK}" \ - -H "Content-Type: application/json" \ - -d '{"model":"qwen-turbo","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' -``` - -期望：HTTP 200，响应体含 `data: {...}` 行且至少一条 `delta.content` 非空；最后为 `data: [DONE]`。 - -## 7. 负向用例（可选） - -| 场景 | 操作 | 期望 | -|------|------|------| -| 错误 virtual key | `Authorization: Bearer sk-invalid` | 4xx，virtual key 无效 | -| 无路由 | `climc ai-routing-disable aiproxy-ft-routing` 或删除后再 chat | 404，无匹配 routing | -| 禁用 virtual key | `climc ai-virtual-key-disable aiproxy-ft-vk` | 4xx | -| provider 限制 | create vk 时 `--limits '{"allowed_ai_provider_ids":["openai"]}'` | 4xx，provider 不允许 | - -## 8. 清理（可选） - -```bash -climc ai-routing-delete aiproxy-ft-routing -climc ai-virtual-key-delete aiproxy-ft-vk -climc ai-key-delete qwen-dashscope-ft -``` - -## 常见问题 - -**`no ai_routing matched for virtual key project`** -Virtual key 的 `project_id` 与 routing 所在项目不一致，或 routing 未 `enabled`、未共享到该项目。用同一 `climc` 项目上下文创建两者。 - -**`no api_key for ai_provider`** -未创建 `ai_key`，且 `ai_provider.config` 里也没有 `api_key`。按步骤 3 创建 `ai_key`。 - -**DashScope 401/403** -检查 `DASHSCOPE_API_KEY` 是否有效、是否开通对应模型。 - -**多副本 `ai_routing` 绑定其它节点** -若 routing 指定了 `ai_proxy_node_id`，须访问该节点的 public endpoint，或去掉绑定。 diff --git a/docs/aiproxy/functional-test.md b/docs/aiproxy/functional-test.md new file mode 100644 index 0000000000..fbe3528cbe --- /dev/null +++ b/docs/aiproxy/functional-test.md @@ -0,0 +1,369 @@ +# aiproxy 功能测试（climc） + +本文用 **climc** 配置 aiproxy 资源，并通过 **`climc aiproxy-test-*`** 子命令或 **curl** 做端到端验证。 + +| 子命令 | 数据面路径 | 用途 | +|--------|------------|------| +| `aiproxy-test-chat` | `POST /ai/openai/v1/chat/completions` | OpenAI 兼容 chat（非流式 + 流式） | +| `aiproxy-test-anthropic` | `POST /ai/anthropic/v1/messages` | Anthropic Messages API | +| `aiproxy-test-provider-create` | — | 创建自定义 `ai_provider` 并校验 | + +> **安全**：请勿将上游 API Key 写入文档或提交到 Git。使用环境变量传入；若 Key 曾泄露，请到对应云平台控制台轮换。 + +## 前置条件 + +| 项 | 说明 | +|----|------| +| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` 服务及 public endpoint | +| 数据库 | 主节点已执行 `InitDB`，catalog 已 seed 对应 provider / model | +| 客户端 | 已 `source /etc/yunion/rcadmin`（或等价 rc 文件），`climc` 能正常 list | +| 网络 | aiproxy 节点能访问目标上游（DashScope、MiMo、Anthropic 等） | + +## 一键 E2E（交互式，推荐） + +从 catalog 选择 **模型提供商** 与 **model_key**，终端输入 API Key（或使用环境变量跳过输入），自动完成 `ai_key` / `ai_virtual_key` / `ai_routing` 配置及 chat 校验： + +```bash +source /etc/yunion/rcadmin +climc aiproxy-test-chat +``` + +非交互（CI）： + +```bash +export AIPROXY_TEST_NONINTERACTIVE=1 +export AIPROXY_TEST_PROVIDER=aliyun +export AIPROXY_TEST_MODEL=qwen-turbo +export AIPROXY_TEST_API_KEY='...' +export AIPROXY_TEST_SKIP_STREAM=1 # 可选，跳过流式 +climc aiproxy-test-chat +``` + +### 环境变量 + +| 变量 | 说明 | +|------|------| +| `AIPROXY_TEST_PROVIDER` | `provider_key`（如 `aliyun`、`xiaomi`） | +| `AIPROXY_TEST_MODEL` | `model_key`（如 `qwen-turbo`） | +| `AIPROXY_TEST_API_KEY` | 上游 API Key（通用） | +| `AIPROXY_FT_*` | 同上（兼容旧变量名） | +| `DASHSCOPE_API_KEY` | 通义千问（`provider=aliyun`） | +| `MIMO_API_KEY` | 小米 MiMo（`provider=xiaomi`） | +| `ANTHROPIC_API_KEY` | Anthropic 直通 | +| `DEEPSEEK_API_KEY` | DeepSeek（Anthropic 兼容场景） | +| `AIPROXY_TEST_SKIP_STREAM` | `1` 跳过流式；`0` 强制流式 | +| `AIPROXY_TEST_KEEP_RESOURCES` | `1` 测试结束后**保留**本次创建的资源（默认自动清理） | +| `AIPROXY_URL` | 留空则从 `endpoint-list` 解析 | + +`aiproxy-test-*` 会在测试过程中自动创建缺失的依赖（`ai_model`、`ai_key`、`ai_virtual_key`、`ai_routing` 等），**测试结束（成功或失败）后自动删除本次创建的资源**。临时修改的 `ai_provider.config.base_url` 会还原。仅删除本次新建项，测试前已存在的同名资源不会被删。 + +保留资源以便排查：`climc aiproxy-test-chat --keep-resources` 或 `export AIPROXY_TEST_KEEP_RESOURCES=1`。 + +`aiproxy-test-chat` 按 provider 自动命名资源（可用 `--key-name`、`--vk-name`、`--routing-name` 覆盖），默认形如 `aiproxy-test-{provider}`。 + +## 按模型提供商快速开始 + +### 通义千问（DashScope / aliyun） + +catalog 需含 `aliyun` 及 `qwen-*` 模型；上游 `https://dashscope.aliyuncs.com/compatible-mode`。 + +```bash +export DASHSCOPE_API_KEY='你的 DashScope API Key' +climc aiproxy-test-chat --provider aliyun --model qwen-turbo --api-key "$DASHSCOPE_API_KEY" +``` + +### 小米 MiMo（xiaomi） + +catalog 需含 `xiaomi` 及 `mimo-*` 模型；上游 `https://api.xiaomimimo.com`。 + +```bash +export MIMO_API_KEY='你的 MiMo API Key' +climc aiproxy-test-chat --provider xiaomi --model mimo-v2-flash --api-key "$MIMO_API_KEY" +``` + +其它 catalog 模型：`mimo-v2.5-pro`、`mimo-v2-pro`、`mimo-v2.5`、`mimo-v2-omni`（id 形如 `xiaomi-mimo-v2.5-pro`）。 + +```bash +export AIPROXY_TEST_PROVIDER=xiaomi AIPROXY_TEST_MODEL=mimo-v2.5-pro +climc aiproxy-test-chat --api-key "$MIMO_API_KEY" +``` + +MiMo 与 DashScope 测试应使用独立的 vk/routing/key 名称，避免混用同一 routing 的 model 列表。 + +### Anthropic Messages API + +数据面 **`POST /ai/anthropic/v1/messages`**，认证为 `Authorization: Bearer `（**不是**上游 Anthropic/DeepSeek API Key）。 + +Claude Code / Anthropic SDK 在正式请求前会对 base URL 发 **`HEAD /ai/anthropic/`** 做连通性探测；aiproxy 已返回 `204`。对 **`HEAD /ai/anthropic/v1/messages`** 无 virtual key 时返回 `401`（表示路由存在、需鉴权）。 + +**Anthropic 直通**（catalog `provider_key=anthropic`）： + +```bash +export ANTHROPIC_API_KEY='sk-ant-...' +climc aiproxy-test-anthropic --provider anthropic --model claude-sonnet-4-5 --api-key "$ANTHROPIC_API_KEY" +``` + +**OpenAI 兼容后端（DeepSeek，翻译模式）**：`config.api_mode=openai`（默认）；客户端仍用 Anthropic SDK；aiproxy 转换为 OpenAI `chat/completions` 转发。 + +| 资源 | 示例 | +|------|------| +| `ai_provider.provider_key` | `deepseek` 或 `openai` | +| `ai_provider.config.base_url` | `https://api.deepseek.com` | +| `ai_provider.config.api_mode` | `openai`（可省略） | +| `ai_model.model_key` | `deepseek-chat` | + +```bash +export DEEPSEEK_API_KEY='...' +climc aiproxy-test-anthropic --provider deepseek --model deepseek-chat \ + --api-key "$DEEPSEEK_API_KEY" --upstream-base-url https://api.deepseek.com +``` + +**DeepSeek 原生 Anthropic 模式**：`provider_key=deepseek` 且 `config.api_mode=anthropic`；aiproxy 将 Anthropic SDK 请求直通 DeepSeek `https://api.deepseek.com/anthropic/v1/messages`（`base_url` 可仍填 `https://api.deepseek.com`，由 aiproxy 自动补 `/anthropic`）。 + +创建 provider 时在顶层 `secret` 写入上游密钥（PostCreate 自动创建关联 `ai_key`）；`config` 仅保留 `base_url` / `api_mode`： + +```json +{ + "generate_name": "my-deepseek", + "provider_key": "deepseek", + "secret": "", + "config": { + "base_url": "https://api.deepseek.com", + "api_mode": "anthropic" + } +} +``` + +`config.api_key` 已不再支持；请在「供应商密钥」Tab 或独立 `ai_key` 资源中管理密钥。 + +OpenAI SDK 经 `/ai/openai/v1/chat/completions` 访问同一 provider 时，也会按 `api_mode=anthropic` 转为 Anthropic Messages 上游。 + +Anthropic SDK / Claude Code 配置（`base_url` 指向 aiproxy，**不要**加 `/v1`；`api_key` 为 **virtual_key**）： + +```python +import anthropic +client = anthropic.Anthropic( + base_url=f"{AIPROXY_URL}/ai/anthropic", # 正确：SDK 自行拼 /v1/messages + api_key=VIRTUAL_KEY, # aiproxy virtual_key，不是上游 Key +) +client.messages.create(model="claude-sonnet-4-5", max_tokens=128, messages=[...]) +``` + +环境变量等价配置： + +```bash +export ANTHROPIC_BASE_URL="${AIPROXY_URL}/ai/anthropic" # 勿写成 .../ai/anthropic/v1 +export ANTHROPIC_API_KEY="${VIRTUAL_KEY}" +``` + +| 配置项 | 正确 | 错误 | +|--------|------|------| +| `ANTHROPIC_BASE_URL` | `https://host/ai/anthropic` | `.../ai/anthropic/v1`（会变成 `/v1/v1/messages`） | +| API Key | aiproxy **virtual_key** | 上游 Anthropic / DeepSeek key | + +## 测试流程概览 + +```mermaid +flowchart LR + VK[ai_virtual_key] --> RT[ai_routing] + RT --> RM[ai_routing_model] + RM --> P[ai_provider] + RM --> M[ai_model] + P --> K[ai_key secret] + K --> UP[上游 API] +``` + +## ai_provider 创建测试 + +### 自定义供应商（provider_key=custom） + +用户自建网关，需填写完整 `base_url`、顶层 `secret` 与 `api_mode`（openai / anthropic）： + +```json +{ + "generate_name": "my-gateway", + "provider_key": "custom", + "secret": "sk-xxx", + "config": { + "base_url": "https://llm.example.com/v1", + "api_mode": "openai" + } +} +``` + +Anthropic Messages 上游示例： + +```json +{ + "generate_name": "my-anthropic-gateway", + "provider_key": "custom", + "secret": "sk-ant-xxx", + "config": { + "base_url": "https://llm.example.com/anthropic", + "api_mode": "anthropic" + } +} +``` + +创建后不会自动注入 catalog 模型；须手动创建 `ai_model` 并配置路由。 + +### 自托管 provider（非 catalog seed） + +```bash +climc aiproxy-test-provider-create +``` + +非交互示例： + +```bash +export AIPROXY_PROVIDER_TEST_NONINTERACTIVE=1 +climc aiproxy-test-provider-create \ + --name my-vllm --provider-key my-vllm \ + --base-url http://127.0.0.1:8000/v1 --enabled +``` + +`provider_key` 须全局唯一；与 InitDB catalog 重复会失败。完整 config 可用 `--config '{"base_url":"..."}'` 或 `AIPROXY_PROVIDER_TEST_CONFIG`。 + +## ai_proxy_node（多副本 / 路由绑定） + +```bash +climc ai-proxy-node-list +climc ai-proxy-node-show primary +climc ai-proxy-node-register --address https://standby-host:30938 --hb-timeout 120 +``` + +将 `ai_routing` 绑定到指定节点（chat 须走该节点 public endpoint）： + +```bash +climc ai-routing-update aiproxy-test-routing --ai-proxy-node-id primary +``` + +创建 `ai_routing` 时若省略 `--ai-proxy-node-id`，默认绑定 `primary` 节点。 + +## 手动步骤（以 aliyun / qwen-turbo 为例） + +以下步骤与 `climc aiproxy-test-chat` 等价，便于理解各资源关系；其它 provider 替换 `aliyun`、`qwen-turbo` 及对应 API Key 即可。 + +### 1. 检查 Keystone endpoint + +```bash +climc endpoint-list --service aiproxy --interface public +``` + +### 2. 检查 catalog + +```bash +climc ai-provider-show aliyun +climc ai-model-show aliyun-qwen-turbo +``` + +小米 MiMo：`climc ai-provider-show xiaomi`、`climc ai-model-show xiaomi-mimo-v2-flash`。 + +### 3. 注册上游 API Key（ai_key） + +```bash +climc ai-key-create qwen-dashscope-test \ + --ai-provider-id aliyun \ + --secret "${DASHSCOPE_API_KEY}" \ + --weight 10 \ + --enabled +``` + +`ai_key` 默认 disabled，创建时需 `--enabled`。 + +### 4. 创建 Virtual Key + +```bash +climc ai-virtual-key-create aiproxy-test-vk +climc ai-virtual-key-show aiproxy-test-vk +``` + +Virtual key 归属当前 climc 用户的 **项目**；`ai_routing` 须在同一项目（或共享到该项目）下。 + +### 5. 创建项目路由 + +```bash +climc ai-routing-create aiproxy-test-routing \ + --priority 10 \ + --model-key qwen-turbo \ + --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-turbo","priority":1}]' +``` + +### 6. Chat completions（curl） + +```bash +AIPROXY_URL="${AIPROXY_URL:-$(climc endpoint-list --service aiproxy --interface public --limit 1 \ + --output-format json | jq -r '.data[0].url // empty')}" + +VK="$(climc ai-virtual-key-show aiproxy-test-vk --output-format json | jq -r '.virtual_key')" + +curl -k -sS "${AIPROXY_URL%/}/ai/openai/v1/chat/completions" \ + -H "Authorization: Bearer ${VK}" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen-turbo", + "messages": [{"role": "user", "content": "用一句话介绍通义千问"}], + "max_tokens": 128 + }' | jq . +``` + +**期望**：HTTP 200，JSON 含 `choices[0].message.content` 及 `usage`。 + +### 6b. 流式 Chat + +`climc aiproxy-test-chat` 默认在非流式成功后继续流式校验。跳过：`climc aiproxy-test-chat --skip-stream`。 + +```bash +curl -k -sS -N "${AIPROXY_URL%/}/ai/openai/v1/chat/completions" \ + -H "Authorization: Bearer ${VK}" \ + -H "Content-Type: application/json" \ + -d '{"model":"qwen-turbo","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}' +``` + +Anthropic 流式与非流式均走 `/ai/anthropic/v1/messages`，请求体设置 `"stream": true` 即可。 + +## 负向用例（可选） + +| 场景 | 操作 | 期望 | +|------|------|------| +| 错误 virtual key | `Authorization: Bearer sk-invalid` | 4xx | +| 无路由 | disable 或删除 routing 后再 chat | 404 | +| 禁用 virtual key | `climc ai-virtual-key-disable aiproxy-test-vk` | 4xx | +| provider 限制 | vk `--limits '{"allowed_ai_provider_ids":["openai"]}'` | 4xx | + +## 清理 + +`aiproxy-test-*` 默认在结束时自动清理（见上文 `AIPROXY_TEST_KEEP_RESOURCES`）。手动清理示例（仅在使用 `--keep-resources` 或清理失败时需要）： + +DashScope： + +```bash +climc ai-routing-delete aiproxy-test-aliyun-routing +climc ai-virtual-key-delete aiproxy-test-aliyun-vk +climc ai-key-delete aiproxy-test-aliyun +``` + +MiMo 示例（若使用独立资源名）： + +```bash +climc ai-routing-delete aiproxy-test-xiaomi-routing +climc ai-virtual-key-delete aiproxy-test-xiaomi-vk +climc ai-key-delete aiproxy-test-xiaomi +``` + +## 常见问题 + +**`no ai_routing matched for virtual key project`** +Virtual key 与 routing 的项目不一致，或 routing 未 `enabled`、未共享到该项目。 + +**`add an enabled ai_key with secret for this provider`** +未创建启用的 `ai_key`，或密钥为空。创建 provider 时使用顶层 `secret`，或在「供应商密钥」Tab 手动添加。 + +**DashScope / MiMo 401/403** +检查对应环境变量中的 API Key 是否有效、模型是否已开通。 + +**多副本 `ai_routing` 绑定其它节点** +若 routing 指定了 `ai_proxy_node_id`，须访问该节点的 public endpoint，或去掉绑定。 + +**MiMo 与 DashScope 资源冲突** +各 provider 使用独立的 vk/routing/key 名称，勿共用同一 routing 的 model 列表。 diff --git a/pkg/aiproxy/ft/admin.go b/pkg/aiproxy/ft/admin.go new file mode 100644 index 0000000000..3b6fb4acb9 --- /dev/null +++ b/pkg/aiproxy/ft/admin.go @@ -0,0 +1,242 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" +) + +type AdminNames struct { + KeyName string + VkName string + RoutingName string +} + +func DefaultAdminNames(providerKey string, suffix string) AdminNames { + base := fmt.Sprintf("aiproxy-test-%s", providerKey) + if suffix != "" { + base = fmt.Sprintf("aiproxy-test-%s-%s", providerKey, suffix) + } + return AdminNames{ + KeyName: base, + VkName: base + "-vk", + RoutingName: base + "-routing", + } +} + +func DefaultAnthropicAdminNames(providerKey string) AdminNames { + if providerKey == "anthropic" { + return AdminNames{ + KeyName: "aiproxy-test-anthropic", + VkName: "aiproxy-test-anthropic-vk", + RoutingName: "aiproxy-test-anthropic-routing", + } + } + return AdminNames{ + KeyName: fmt.Sprintf("aiproxy-test-%s-anthropic", providerKey), + VkName: "aiproxy-test-anthropic-vk", + RoutingName: "aiproxy-test-anthropic-routing", + } +} + +func ensureAiKey(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, keyName, apiSecret string) error { + provider, err := apmodules.AiProviders.Get(session, providerKey, nil) + if err != nil { + return errors.Wrapf(err, "ai_provider %s not found", providerKey) + } + providerID, _ := provider.GetString("id") + if providerID == "" { + return errors.Errorf("ai_provider %s has empty id", providerKey) + } + + if _, err := apmodules.AiKeys.Get(session, keyName, nil); err == nil { + fmt.Printf("ai_key %s exists, syncing secret and ai_provider_id\n", keyName) + params := jsonutils.NewDict() + params.Set("ai_provider_id", jsonutils.NewString(providerID)) + params.Set("secret", jsonutils.NewString(apiSecret)) + params.Set("weight", jsonutils.NewInt(10)) + if _, err := apmodules.AiKeys.Update(session, keyName, params); err != nil { + return errors.Wrap(err, "ai-key-update") + } + } else { + params := jsonutils.NewDict() + params.Set("name", jsonutils.NewString(keyName)) + params.Set("ai_provider_id", jsonutils.NewString(providerID)) + params.Set("secret", jsonutils.NewString(apiSecret)) + params.Set("weight", jsonutils.NewInt(10)) + params.Set("enabled", jsonutils.JSONTrue) + if _, err := apmodules.AiKeys.Create(session, params); err != nil { + return errors.Wrap(err, "ai-key-create") + } + if tracker != nil { + tracker.createdAiKey = keyName + } + } + return ensureAiKeyEnabled(session, keyName) +} + +func ensureAiKeyEnabled(session *mcclient.ClientSession, keyName string) error { + obj, err := apmodules.AiKeys.Get(session, keyName, nil) + if err != nil { + return err + } + enabled, _ := obj.Bool("enabled") + if enabled { + return nil + } + fmt.Printf("ai_key %s is disabled, enabling\n", keyName) + _, err = apmodules.AiKeys.PerformAction(session, keyName, "enable", nil) + return err +} + +func VerifyAiKeyForProvider(session *mcclient.ClientSession, providerKey string) error { + provider, err := apmodules.AiProviders.Get(session, providerKey, nil) + if err != nil { + return errors.Wrapf(err, "ai_provider %s not found", providerKey) + } + providerID, _ := provider.GetString("id") + query := jsonutils.NewDict() + query.Set("ai_provider_id", jsonutils.NewString(providerID)) + result, err := apmodules.AiKeys.List(session, query) + if err != nil { + return err + } + count := 0 + for _, item := range result.Data { + ok, _ := item.Bool("enabled") + if ok { + count++ + } + } + if count == 0 { + return errors.Errorf("no enabled ai_key bound to ai_provider_id=%s", providerID) + } + fmt.Printf("enabled ai_key rows for %s: %d\n", providerID, count) + return nil +} + +func ensureVirtualKey(session *mcclient.ClientSession, tracker *ResourceTracker, vkName string) (string, error) { + if _, err := apmodules.AiVirtualKeys.Get(session, vkName, nil); err != nil { + params := jsonutils.NewDict() + params.Set("name", jsonutils.NewString(vkName)) + if _, err := apmodules.AiVirtualKeys.Create(session, params); err != nil { + return "", errors.Wrap(err, "ai-virtual-key-create") + } + if tracker != nil { + tracker.createdVirtualKey = vkName + } + } else { + fmt.Printf("virtual key %s already exists\n", vkName) + } + obj, err := apmodules.AiVirtualKeys.Get(session, vkName, nil) + if err != nil { + return "", err + } + vk, _ := obj.GetString("virtual_key") + if vk == "" { + return "", errors.Error("empty virtual_key from ai-virtual-key-show") + } + return vk, nil +} + +func ensureRouting(session *mcclient.ClientSession, tracker *ResourceTracker, routingName, providerKey, routingModelRef string) error { + if _, err := apmodules.AiRoutings.Get(session, routingName, nil); err == nil { + fmt.Printf("routing %s already exists\n", routingName) + return nil + } + models := jsonutils.NewArray() + models.Add(jsonutils.Marshal(map[string]interface{}{ + "ai_provider_id": providerKey, + "ai_model_id": routingModelRef, + "priority": 1, + })) + params := jsonutils.NewDict() + params.Set("name", jsonutils.NewString(routingName)) + params.Set("priority", jsonutils.NewInt(10)) + params.Set("models", models) + if _, err := apmodules.AiRoutings.Create(session, params); err != nil { + return errors.Wrap(err, "ai-routing-create") + } + if tracker != nil { + tracker.createdRouting = routingName + } + return nil +} + +func SetupAdminResources(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, modelKey, apiSecret string, names AdminNames) (virtualKey string, catalogModelID string, err error) { + catalogModelID = CatalogModelID(providerKey, modelKey) + routingModelRef, _, err := ensureAiModel(session, tracker, providerKey, modelKey) + if err != nil { + return "", "", err + } + if err = ensureAiKey(session, tracker, providerKey, names.KeyName, apiSecret); err != nil { + return "", "", err + } + if err = VerifyAiKeyForProvider(session, providerKey); err != nil { + return "", "", err + } + virtualKey, err = ensureVirtualKey(session, tracker, names.VkName) + if err != nil { + return "", "", err + } + if err = ensureRouting(session, tracker, names.RoutingName, providerKey, routingModelRef); err != nil { + return "", "", err + } + return virtualKey, catalogModelID, nil +} + +func EnsureAiProviderBaseURL(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, baseURL string) error { + baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/") + if baseURL == "" { + return nil + } + obj, err := apmodules.AiProviders.Get(session, providerKey, nil) + if err != nil { + return errors.Wrapf(err, "ai_provider %s not found", providerKey) + } + current, _ := obj.GetString("config", "base_url") + current = strings.TrimRight(strings.TrimSpace(current), "/") + if current == baseURL { + fmt.Printf("ai_provider %s config.base_url=%s\n", providerKey, baseURL) + return nil + } + if tracker != nil && tracker.providerConfigRestore == nil { + snap, err := snapshotProviderConfig(session, providerKey) + if err != nil { + return err + } + tracker.providerConfigRestore = snap + } + configDict := jsonutils.NewDict() + if obj.Contains("config") { + cfg, _ := obj.Get("config") + if cfgDict, ok := cfg.(*jsonutils.JSONDict); ok { + configDict = cfgDict + } + } + configDict.Set("base_url", jsonutils.NewString(baseURL)) + params := jsonutils.NewDict() + params.Set("config", configDict) + fmt.Printf("updating ai_provider %s config.base_url -> %s\n", providerKey, baseURL) + _, err = apmodules.AiProviders.Update(session, providerKey, params) + return errors.Wrap(err, "ai-provider-update base_url") +} diff --git a/pkg/aiproxy/ft/anthropic.go b/pkg/aiproxy/ft/anthropic.go new file mode 100644 index 0000000000..712e2da3c6 --- /dev/null +++ b/pkg/aiproxy/ft/anthropic.go @@ -0,0 +1,158 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "fmt" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +func RunAnthropicTest(session *mcclient.ClientSession, opts *AnthropicOptions) error { + tracker := NewResourceTracker(envKeepResources(opts.KeepResources)) + defer tracker.Cleanup(session) + + providerKey := strings.TrimSpace(opts.Provider) + if providerKey == "" { + providerKey = "anthropic" + } + + modelKey := strings.TrimSpace(opts.Model) + if modelKey == "" { + modelKey = resolveModelFromEnv() + } + if modelKey == "" { + modelKey = DefaultModelForProvider(providerKey) + } + + apiSecret := strings.TrimSpace(opts.ApiKey) + if apiSecret == "" { + var err error + apiSecret, err = promptApiKey(providerKey, "", envNonInteractive(opts.NonInteractive)) + if err != nil { + return err + } + } + + prompt := strings.TrimSpace(opts.Prompt) + if prompt == "" { + prompt = resolvePromptFromEnv() + } + if prompt == "" { + prompt = "Say hi in one short sentence." + } + + skipStream := envSkipStream(opts.SkipStream) + names := DefaultAnthropicAdminNames(providerKey) + if opts.KeyName != "" { + names.KeyName = opts.KeyName + } + if opts.VkName != "" { + names.VkName = opts.VkName + } + if opts.RoutingName != "" { + names.RoutingName = opts.RoutingName + } + + catalogModelID := CatalogModelID(providerKey, modelKey) + fmt.Println() + fmt.Println("=== aiproxy Anthropic Messages 测试 ===") + fmt.Printf("provider: %s model: %s catalog_id: %s\n", providerKey, modelKey, catalogModelID) + fmt.Println() + + Step("1. Resolve aiproxy URL") + aiproxyURL, err := ResolveAiproxyURL(session, opts.AiproxyURL) + if err != nil { + return err + } + fmt.Printf("AIPROXY_URL=%s\n", aiproxyURL) + upstreamBase := strings.TrimSpace(opts.UpstreamBaseURL) + if upstreamBase == "" { + upstreamBase = envFirst("AIPROXY_TEST_BASE_URL", "AIPROXY_FT_BASE_URL") + } + if upstreamBase != "" { + if err := EnsureAiProviderBaseURL(session, tracker, providerKey, upstreamBase); err != nil { + return err + } + } + + Step(fmt.Sprintf("2. Catalog %s / %s", providerKey, modelKey)) + if err := VerifyCatalog(session, providerKey, modelKey, true); err != nil { + return err + } + + Step("3. ai_key / ai_virtual_key / ai_routing") + vk, _, err := SetupAdminResources(session, tracker, providerKey, modelKey, apiSecret, names) + if err != nil { + return err + } + fmt.Printf("virtual_key=%s...\n", previewText(vk, 12)) + + Step("4. POST /ai/anthropic/v1/messages") + client := httpClientFromSession(session) + payload := map[string]interface{}{ + "model": modelKey, + "max_tokens": 128, + "messages": []map[string]string{ + {"role": "user", "content": prompt}, + }, + } + code, body, err := postJSON(client, anthropicMessagesURL(aiproxyURL), vk, payload) + if err != nil { + return err + } + fmt.Printf("HTTP %d\n", code) + if err := printJSONBody(body); err != nil { + return err + } + if code != 200 { + return errors.Errorf("anthropic messages request failed with HTTP %d", code) + } + content, err := extractAnthropicTextContent(body) + if err != nil { + return err + } + fmt.Printf("text: %s\n", previewText(content, 120)) + + if !skipStream { + Step("5. POST /ai/anthropic/v1/messages (stream=true)") + streamPayload := map[string]interface{}{ + "model": modelKey, + "stream": true, + "max_tokens": 128, + "messages": []map[string]string{ + {"role": "user", "content": prompt}, + }, + } + streamCode, streamBody, err := postJSONStream(client, anthropicMessagesURL(aiproxyURL), vk, streamPayload) + if err != nil { + return err + } + defer streamBody.Close() + fmt.Printf("HTTP %d (anthropic stream)\n", streamCode) + aggregated, err := aggregateSSEStream(streamBody, parseAnthropicStreamDelta) + if err != nil { + return err + } + fmt.Printf("stream text: %s\n", previewText(aggregated, 120)) + } + + fmt.Println() + fmt.Printf("OK: anthropic messages test passed for %s/%s.\n", providerKey, modelKey) + return nil +} diff --git a/pkg/aiproxy/ft/catalog.go b/pkg/aiproxy/ft/catalog.go new file mode 100644 index 0000000000..26f86f61fa --- /dev/null +++ b/pkg/aiproxy/ft/catalog.go @@ -0,0 +1,169 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "fmt" + "sort" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/mcclient" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" +) + +func CatalogModelID(providerKey, modelKey string) string { + return fmt.Sprintf("%s-%s", providerKey, modelKey) +} + +func DefaultModelForProvider(providerKey string) string { + switch providerKey { + case api.ProviderKeyAliyun: + return "qwen-turbo" + case api.ProviderKeyXiaomi: + return "mimo-v2-flash" + case api.ProviderKeyDeepseek: + return "deepseek-v4-flash" + case api.ProviderKeyOpenAI: + return "gpt-4o-mini" + case api.ProviderKeyAnthropic: + return "claude-sonnet-4-5" + default: + return "" + } +} + +func DefaultPromptForProvider(providerKey string) string { + switch providerKey { + case api.ProviderKeyAliyun: + return "用一句话介绍通义千问" + case api.ProviderKeyXiaomi: + return "用一句话介绍小米 MiMo" + default: + return "用一句话介绍这个模型" + } +} + +func ListCatalogProviderKeys(session *mcclient.ClientSession) ([]string, error) { + query := jsonutils.NewDict() + query.Set("limit", jsonutils.NewInt(500)) + result, err := apmodules.AiProviders.List(session, query) + if err != nil { + return nil, err + } + keys := make([]string, 0, len(result.Data)) + seen := map[string]struct{}{} + for _, item := range result.Data { + pk, _ := item.GetString("provider_key") + pk = strings.TrimSpace(pk) + if pk == "" { + continue + } + if _, ok := seen[pk]; ok { + continue + } + seen[pk] = struct{}{} + keys = append(keys, pk) + } + sort.Strings(keys) + return keys, nil +} + +func ListCatalogModelKeys(session *mcclient.ClientSession, providerKey string) ([]string, error) { + query := jsonutils.NewDict() + query.Set("limit", jsonutils.NewInt(500)) + query.Set("ai_provider_id", jsonutils.NewString(providerKey)) + result, err := apmodules.AiModels.List(session, query) + if err != nil { + return nil, err + } + models := make([]string, 0, len(result.Data)) + seen := map[string]struct{}{} + for _, item := range result.Data { + mk, _ := item.GetString("model_key") + mk = strings.TrimSpace(mk) + if mk == "" || mk == "default" { + continue + } + if _, ok := seen[mk]; ok { + continue + } + seen[mk] = struct{}{} + models = append(models, mk) + } + sort.Strings(models) + return models, nil +} + +func VerifyCatalog(session *mcclient.ClientSession, providerKey, modelKey string, warnMissingModel bool) error { + if _, err := apmodules.AiProviders.Get(session, providerKey, nil); err != nil { + return errors.Wrapf(err, "ai_provider %s missing; run aiproxy master InitDB first", providerKey) + } + if _, err := findAiModelByKey(session, providerKey, modelKey); err == nil { + return nil + } + catalogID := CatalogModelID(providerKey, modelKey) + if _, err := apmodules.AiModels.Get(session, catalogID, nil); err == nil { + return nil + } + if warnMissingModel { + fmt.Printf("WARN: ai_model %s not in catalog; will create for test if needed\n", catalogID) + return nil + } + return errors.Errorf("ai_model %s not in catalog (re-run aiproxy master InitDB)", catalogID) +} + +func findAiModelByKey(session *mcclient.ClientSession, providerKey, modelKey string) (jsonutils.JSONObject, error) { + query := jsonutils.NewDict() + query.Set("limit", jsonutils.NewInt(10)) + query.Set("ai_provider_id", jsonutils.NewString(providerKey)) + query.Set("model_key", jsonutils.NewString(modelKey)) + result, err := apmodules.AiModels.List(session, query) + if err != nil { + return nil, err + } + if len(result.Data) == 0 { + return nil, errors.Errorf("model_key %s not found under provider %s", modelKey, providerKey) + } + return result.Data[0], nil +} + +// EnsureAiModel guarantees an ai_model row exists for providerKey/modelKey. +// Returns the model reference for ai_routing and the created resource name (if any). +func ensureAiModel(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, modelKey string) (routingModelRef string, createdName string, err error) { + catalogID := CatalogModelID(providerKey, modelKey) + if _, err := apmodules.AiModels.Get(session, catalogID, nil); err == nil { + return modelKey, "", nil + } + if _, err := findAiModelByKey(session, providerKey, modelKey); err == nil { + return modelKey, "", nil + } + fmt.Printf("ai_model %s not in catalog, creating for test\n", catalogID) + params := jsonutils.NewDict() + params.Set("name", jsonutils.NewString(catalogID)) + params.Set("ai_provider_id", jsonutils.NewString(providerKey)) + params.Set("model_key", jsonutils.NewString(modelKey)) + params.Set("enabled", jsonutils.JSONTrue) + if _, err := apmodules.AiModels.Create(session, params); err != nil { + return "", "", errors.Wrapf(err, "ai-model-create %s/%s", providerKey, modelKey) + } + if tracker != nil { + tracker.createdAiModel = catalogID + } + return modelKey, catalogID, nil +} diff --git a/pkg/aiproxy/ft/chat.go b/pkg/aiproxy/ft/chat.go new file mode 100644 index 0000000000..e1ff5d71a2 --- /dev/null +++ b/pkg/aiproxy/ft/chat.go @@ -0,0 +1,166 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "fmt" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +func RunChatTest(session *mcclient.ClientSession, opts *ChatOptions) error { + tracker := NewResourceTracker(envKeepResources(opts.KeepResources)) + defer tracker.Cleanup(session) + + nonInteractive := envNonInteractive(opts.NonInteractive) + + providers, err := ListCatalogProviderKeys(session) + if err != nil { + return err + } + if len(providers) == 0 { + return errors.Error("catalog 中无 ai_provider，请先执行 aiproxy master InitDB") + } + + providerKey, err := promptSelectProvider(providers, opts.Provider, nonInteractive) + if err != nil { + return err + } + + models, err := ListCatalogModelKeys(session, providerKey) + if err != nil { + return err + } + if len(models) == 0 { + return errors.Errorf("provider %s 下无可用 model_key（catalog 未 seed？）", providerKey) + } + + modelKey, err := promptSelectModel(models, providerKey, opts.Model, nonInteractive) + if err != nil { + return err + } + + apiSecret, err := promptApiKey(providerKey, opts.ApiKey, nonInteractive) + if err != nil { + return err + } + + prompt := strings.TrimSpace(opts.Prompt) + if prompt == "" { + prompt = resolvePromptFromEnv() + } + if prompt == "" { + prompt = DefaultPromptForProvider(providerKey) + } + + runStream := promptRunStream(opts.SkipStream, nonInteractive) + + names := DefaultAdminNames(providerKey, "") + if opts.KeyName != "" { + names.KeyName = opts.KeyName + } + if opts.VkName != "" { + names.VkName = opts.VkName + } + if opts.RoutingName != "" { + names.RoutingName = opts.RoutingName + } + + catalogModelID := CatalogModelID(providerKey, modelKey) + fmt.Println() + fmt.Println("=== aiproxy OpenAI chat 测试 ===") + fmt.Printf("provider: %s model: %s catalog_id: %s\n", providerKey, modelKey, catalogModelID) + fmt.Printf("ai_key: %s virtual_key: %s routing: %s\n", names.KeyName, names.VkName, names.RoutingName) + fmt.Println() + + Step("1. Keystone aiproxy public endpoint") + aiproxyURL, err := ResolveAiproxyURL(session, opts.AiproxyURL) + if err != nil { + return err + } + fmt.Printf("AIPROXY_URL=%s\n", aiproxyURL) + + Step(fmt.Sprintf("2. Catalog %s / %s", providerKey, modelKey)) + if err := VerifyCatalog(session, providerKey, modelKey, false); err != nil { + return err + } + + Step("3. ai_key") + vk, _, err := SetupAdminResources(session, tracker, providerKey, modelKey, apiSecret, names) + if err != nil { + return err + } + fmt.Printf("virtual_key=%s...\n", previewText(vk, 12)) + + Step("4. POST /ai/openai/v1/chat/completions") + client := httpClientFromSession(session) + payload := map[string]interface{}{ + "model": modelKey, + "messages": []map[string]string{ + {"role": "user", "content": prompt}, + }, + "max_tokens": 128, + } + code, body, err := postJSON(client, openAIChatURL(aiproxyURL), vk, payload) + if err != nil { + return err + } + fmt.Printf("HTTP %d\n", code) + if err := printJSONBody(body); err != nil { + return err + } + if code != 200 { + return errors.Errorf("chat request failed with HTTP %d", code) + } + content, err := extractOpenAIChatContent(body) + if err != nil { + return err + } + fmt.Printf("content (%d chars): %s\n", len(content), previewText(content, 120)) + + if runStream { + Step("5. POST /ai/openai/v1/chat/completions (stream=true)") + streamPayload := map[string]interface{}{ + "model": modelKey, + "stream": true, + "messages": []map[string]string{ + {"role": "user", "content": prompt}, + }, + "max_tokens": 64, + } + streamCode, streamBody, err := postJSONStream(client, openAIChatURL(aiproxyURL), vk, streamPayload) + if err != nil { + return err + } + defer streamBody.Close() + fmt.Printf("HTTP %d (stream)\n", streamCode) + aggregated, err := aggregateSSEStream(streamBody, parseOpenAIStreamDelta) + if err != nil { + return err + } + fmt.Printf("stream content (%d chars): %s\n", len(aggregated), previewText(aggregated, 120)) + } + + streamNote := "" + if runStream { + streamNote = " + stream" + } + fmt.Println() + fmt.Printf("OK: aiproxy chat test passed for %s/%s (non-stream%s).\n", providerKey, modelKey, streamNote) + return nil +} diff --git a/pkg/aiproxy/ft/cleanup.go b/pkg/aiproxy/ft/cleanup.go new file mode 100644 index 0000000000..843c6f49ef --- /dev/null +++ b/pkg/aiproxy/ft/cleanup.go @@ -0,0 +1,213 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" +) + +type providerConfigSnapshot struct { + providerKey string + config *jsonutils.JSONDict +} + +// ResourceTracker records resources created during a test run for automatic cleanup. +type ResourceTracker struct { + KeepResources bool + + createdRouting string + createdVirtualKey string + createdAiKey string + createdAiModel string + createdProvider string + + providerConfigRestore *providerConfigSnapshot +} + +func NewResourceTracker(keepResources bool) *ResourceTracker { + return &ResourceTracker{KeepResources: keepResources} +} + +func envKeepResources(explicit bool) bool { + if explicit { + return true + } + return envTruthy("AIPROXY_TEST_KEEP_RESOURCES", "AIPROXY_FT_KEEP_RESOURCES") +} + +func (t *ResourceTracker) hasCreated() bool { + if t == nil { + return false + } + return t.createdRouting != "" || + t.createdVirtualKey != "" || + t.createdAiKey != "" || + t.createdAiModel != "" || + t.createdProvider != "" || + t.providerConfigRestore != nil +} + +func (t *ResourceTracker) Cleanup(session *mcclient.ClientSession) { + if t == nil { + return + } + if t.KeepResources { + if t.hasCreated() { + fmt.Println() + fmt.Println("Keeping test resources (--keep-resources / AIPROXY_TEST_KEEP_RESOURCES=1)") + t.printKeepHint() + } + return + } + if !t.hasCreated() { + return + } + + fmt.Println() + Step("cleanup test resources") + + if t.createdRouting != "" { + t.deleteRouting(session, t.createdRouting) + } + if t.createdVirtualKey != "" { + t.deleteVirtualKey(session, t.createdVirtualKey) + } + if t.createdAiKey != "" { + t.deleteAiKey(session, t.createdAiKey) + } + if t.createdAiModel != "" { + t.deleteAiModel(session, t.createdAiModel) + } + if t.providerConfigRestore != nil { + t.restoreProviderConfig(session) + } + if t.createdProvider != "" { + t.deleteProvider(session, t.createdProvider) + } +} + +func (t *ResourceTracker) printKeepHint() { + if t.createdRouting != "" { + fmt.Printf(" climc ai-routing-delete %s\n", t.createdRouting) + } + if t.createdVirtualKey != "" { + fmt.Printf(" climc ai-virtual-key-delete %s\n", t.createdVirtualKey) + } + if t.createdAiKey != "" { + fmt.Printf(" climc ai-key-delete %s\n", t.createdAiKey) + } + if t.createdAiModel != "" { + fmt.Printf(" climc ai-model-delete %s\n", t.createdAiModel) + } + if t.createdProvider != "" { + fmt.Printf(" climc ai-provider-delete %s\n", t.createdProvider) + } +} + +func (t *ResourceTracker) deleteRouting(session *mcclient.ClientSession, name string) { + if _, err := apmodules.AiRoutings.Delete(session, name, nil); err != nil { + fmt.Printf("WARN: delete ai_routing %s: %v\n", name, err) + return + } + fmt.Printf("deleted ai_routing %s\n", name) +} + +func (t *ResourceTracker) deleteVirtualKey(session *mcclient.ClientSession, name string) { + if _, err := apmodules.AiVirtualKeys.Delete(session, name, nil); err != nil { + fmt.Printf("WARN: delete ai_virtual_key %s: %v\n", name, err) + return + } + fmt.Printf("deleted ai_virtual_key %s\n", name) +} + +func (t *ResourceTracker) deleteAiKey(session *mcclient.ClientSession, name string) { + if _, err := apmodules.AiKeys.Delete(session, name, nil); err != nil { + fmt.Printf("WARN: delete ai_key %s: %v\n", name, err) + return + } + fmt.Printf("deleted ai_key %s\n", name) +} + +func (t *ResourceTracker) deleteAiModel(session *mcclient.ClientSession, name string) { + if _, err := apmodules.AiModels.Delete(session, name, nil); err != nil { + fmt.Printf("WARN: delete ai_model %s: %v\n", name, err) + return + } + fmt.Printf("deleted ai_model %s\n", name) +} + +func (t *ResourceTracker) deleteProvider(session *mcclient.ClientSession, name string) { + if _, err := apmodules.AiProviders.Delete(session, name, nil); err != nil { + fmt.Printf("WARN: delete ai_provider %s: %v\n", name, err) + return + } + fmt.Printf("deleted ai_provider %s\n", name) +} + +func (t *ResourceTracker) restoreProviderConfig(session *mcclient.ClientSession) { + snap := t.providerConfigRestore + if snap == nil { + return + } + params := jsonutils.NewDict() + if snap.config != nil { + params.Set("config", snap.config) + } else { + params.Set("config", jsonutils.NewDict()) + } + if _, err := apmodules.AiProviders.Update(session, snap.providerKey, params); err != nil { + fmt.Printf("WARN: restore ai_provider %s config: %v\n", snap.providerKey, err) + return + } + fmt.Printf("restored ai_provider %s config\n", snap.providerKey) +} + +func cloneJSONDict(obj jsonutils.JSONObject) *jsonutils.JSONDict { + if obj == nil { + return nil + } + if d, ok := obj.(*jsonutils.JSONDict); ok { + out := jsonutils.NewDict() + out.Update(d) + return out + } + parsed, err := jsonutils.Parse([]byte(obj.String())) + if err != nil { + return nil + } + if d, ok := parsed.(*jsonutils.JSONDict); ok { + return d + } + return nil +} + +func snapshotProviderConfig(session *mcclient.ClientSession, providerKey string) (*providerConfigSnapshot, error) { + obj, err := apmodules.AiProviders.Get(session, providerKey, nil) + if err != nil { + return nil, errors.Wrapf(err, "ai_provider %s not found", providerKey) + } + snap := &providerConfigSnapshot{providerKey: providerKey} + if obj.Contains("config") { + cfg, _ := obj.Get("config") + snap.config = cloneJSONDict(cfg) + } + return snap, nil +} diff --git a/pkg/aiproxy/ft/doc.go b/pkg/aiproxy/ft/doc.go new file mode 100644 index 0000000000..6a5f402ae2 --- /dev/null +++ b/pkg/aiproxy/ft/doc.go @@ -0,0 +1,16 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ft implements aiproxy end-to-end functional tests for climc subcommands. +package ft // import "yunion.io/x/onecloud/pkg/aiproxy/ft" diff --git a/pkg/aiproxy/ft/endpoint.go b/pkg/aiproxy/ft/endpoint.go new file mode 100644 index 0000000000..1f4ba105ce --- /dev/null +++ b/pkg/aiproxy/ft/endpoint.go @@ -0,0 +1,51 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/identity" +) + +func ResolveAiproxyURL(session *mcclient.ClientSession, override string) (string, error) { + if override = strings.TrimRight(strings.TrimSpace(override), "/"); override != "" { + return override, nil + } + if url := resolveAiproxyURLFromEnv(); url != "" { + return url, nil + } + query := jsonutils.NewDict() + query.Set("service", jsonutils.NewString("aiproxy")) + query.Set("interface", jsonutils.NewString("public")) + query.Set("limit", jsonutils.NewInt(1)) + result, err := modules.EndpointsV3.List(session, query) + if err != nil { + return "", errors.Wrap(err, "endpoint-list aiproxy public") + } + if len(result.Data) == 0 { + return "", errors.Error("cannot resolve aiproxy public URL; set AIPROXY_URL") + } + url, _ := result.Data[0].GetString("url") + url = strings.TrimRight(strings.TrimSpace(url), "/") + if url == "" { + return "", errors.Error("cannot resolve aiproxy public URL; set AIPROXY_URL") + } + return url, nil +} diff --git a/pkg/aiproxy/ft/env.go b/pkg/aiproxy/ft/env.go new file mode 100644 index 0000000000..de7a89f6af --- /dev/null +++ b/pkg/aiproxy/ft/env.go @@ -0,0 +1,89 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "os" + "strings" +) + +func envFirst(keys ...string) string { + for _, k := range keys { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + return v + } + } + return "" +} + +func envTruthy(keys ...string) bool { + for _, k := range keys { + switch strings.TrimSpace(os.Getenv(k)) { + case "1", "true", "TRUE", "yes", "YES": + return true + } + } + return false +} + +func envSkipStream(explicit bool) bool { + if explicit { + return true + } + return envTruthy("AIPROXY_TEST_SKIP_STREAM", "AIPROXY_FT_SKIP_STREAM") +} + +func envNonInteractive(explicit bool) bool { + if explicit { + return true + } + return envTruthy("AIPROXY_TEST_NONINTERACTIVE", "AIPROXY_FT_NONINTERACTIVE") +} + +func resolveProviderFromEnv() string { + return envFirst("AIPROXY_TEST_PROVIDER", "AIPROXY_FT_PROVIDER") +} + +func resolveModelFromEnv() string { + return envFirst("AIPROXY_TEST_MODEL", "AIPROXY_FT_MODEL") +} + +func resolvePromptFromEnv() string { + return envFirst("AIPROXY_TEST_PROMPT", "AIPROXY_FT_PROMPT") +} + +func resolveApiKeyFromEnv(providerKey string) string { + if v := envFirst("AIPROXY_TEST_API_KEY", "AIPROXY_FT_API_KEY"); v != "" { + return v + } + switch providerKey { + case "aliyun": + return os.Getenv("DASHSCOPE_API_KEY") + case "xiaomi": + return os.Getenv("MIMO_API_KEY") + case "anthropic": + return os.Getenv("ANTHROPIC_API_KEY") + case "openai": + if v := os.Getenv("DEEPSEEK_API_KEY"); v != "" { + return v + } + return os.Getenv("OPENAI_API_KEY") + } + return "" +} + +func resolveAiproxyURLFromEnv() string { + return strings.TrimRight(envFirst("AIPROXY_URL"), "/") +} diff --git a/pkg/aiproxy/ft/ft_test.go b/pkg/aiproxy/ft/ft_test.go new file mode 100644 index 0000000000..8ecd14b0bb --- /dev/null +++ b/pkg/aiproxy/ft/ft_test.go @@ -0,0 +1,121 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "strings" + "testing" +) + +func TestCatalogModelID(t *testing.T) { + if got := CatalogModelID("aliyun", "qwen-turbo"); got != "aliyun-qwen-turbo" { + t.Fatalf("got %q", got) + } +} + +func TestDefaultModelForProvider(t *testing.T) { + cases := map[string]string{ + "aliyun": "qwen-turbo", + "xiaomi": "mimo-v2-flash", + "anthropic": "claude-sonnet-4-5", + "unknown": "", + } + for provider, want := range cases { + if got := DefaultModelForProvider(provider); got != want { + t.Fatalf("%s: got %q want %q", provider, got, want) + } + } +} + +func TestParseOpenAIStreamDelta(t *testing.T) { + payload := `{"choices":[{"delta":{"content":"hello"}}]}` + delta, err := parseOpenAIStreamDelta(payload) + if err != nil || delta != "hello" { + t.Fatalf("delta=%q err=%v", delta, err) + } + _, err = parseOpenAIStreamDelta(`{"error":{"message":"fail"}}`) + if err == nil { + t.Fatal("expected error event") + } +} + +func TestParseAnthropicStreamDelta(t *testing.T) { + payload := `{"delta":{"text":"hi"}}` + delta, err := parseAnthropicStreamDelta(payload) + if err != nil || delta != "hi" { + t.Fatalf("delta=%q err=%v", delta, err) + } +} + +func TestExtractOpenAIChatContent(t *testing.T) { + body := []byte(`{"choices":[{"message":{"content":"answer"}}]}`) + content, err := extractOpenAIChatContent(body) + if err != nil || content != "answer" { + t.Fatalf("content=%q err=%v", content, err) + } +} + +func TestExtractAnthropicTextContent(t *testing.T) { + body := []byte(`{"content":[{"type":"text","text":"hello"}]}`) + content, err := extractAnthropicTextContent(body) + if err != nil || content != "hello" { + t.Fatalf("content=%q err=%v", content, err) + } +} + +func TestAggregateSSEStreamOpenAI(t *testing.T) { + input := strings.Join([]string{ + "data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"}}]}", + "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}", + "data: [DONE]", + }, "\n") + out, err := aggregateSSEStream(strings.NewReader(input), parseOpenAIStreamDelta) + if err != nil || out != "Hello" { + t.Fatalf("out=%q err=%v", out, err) + } +} + +func TestOpenAIChatURL(t *testing.T) { + got := openAIChatURL("https://aiproxy.example.com/") + if got != "https://aiproxy.example.com/ai/openai/v1/chat/completions" { + t.Fatalf("got %q", got) + } +} + +func TestAnthropicMessagesURL(t *testing.T) { + got := anthropicMessagesURL("https://aiproxy.example.com") + if got != "https://aiproxy.example.com/ai/anthropic/v1/messages" { + t.Fatalf("got %q", got) + } +} + +func TestResourceTrackerHasCreated(t *testing.T) { + t.Parallel() + tr := NewResourceTracker(false) + if tr.hasCreated() { + t.Fatal("expected empty tracker") + } + tr.createdAiKey = "k1" + if !tr.hasCreated() { + t.Fatal("expected created") + } +} + +func TestDefaultAdminNames(t *testing.T) { + names := DefaultAdminNames("aliyun", "") + if names.KeyName != "aiproxy-test-aliyun" || names.VkName != "aiproxy-test-aliyun-vk" { + t.Fatalf("%+v", names) + } +} diff --git a/pkg/aiproxy/ft/httpclient.go b/pkg/aiproxy/ft/httpclient.go new file mode 100644 index 0000000000..6c39a26b9d --- /dev/null +++ b/pkg/aiproxy/ft/httpclient.go @@ -0,0 +1,92 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +func httpClientFromSession(session *mcclient.ClientSession) *http.Client { + return session.GetClient().GetClient() +} + +func postJSON(client *http.Client, url, bearer string, payload interface{}) (int, []byte, error) { + body, err := json.Marshal(payload) + if err != nil { + return 0, nil, err + } + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return 0, nil, err + } + req.Header.Set("Authorization", "Bearer "+bearer) + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return resp.StatusCode, nil, err + } + return resp.StatusCode, respBody, nil +} + +func postJSONStream(client *http.Client, url, bearer string, payload interface{}) (int, io.ReadCloser, error) { + body, err := json.Marshal(payload) + if err != nil { + return 0, nil, err + } + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return 0, nil, err + } + req.Header.Set("Authorization", "Bearer "+bearer) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "text/event-stream") + resp, err := client.Do(req) + if err != nil { + return 0, nil, err + } + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, nil, errors.Errorf("HTTP %d: %s", resp.StatusCode, truncateBody(b)) + } + return resp.StatusCode, resp.Body, nil +} + +func openAIChatURL(baseURL string) string { + return strings.TrimRight(baseURL, "/") + "/ai/openai/v1/chat/completions" +} + +func anthropicMessagesURL(baseURL string) string { + return strings.TrimRight(baseURL, "/") + "/ai/anthropic/v1/messages" +} + +func Step(msg string) { + fmt.Println() + fmt.Println("==>", msg) +} diff --git a/pkg/aiproxy/ft/interactive.go b/pkg/aiproxy/ft/interactive.go new file mode 100644 index 0000000000..d8177070db --- /dev/null +++ b/pkg/aiproxy/ft/interactive.go @@ -0,0 +1,241 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" + + "golang.org/x/term" + "yunion.io/x/pkg/errors" +) + +func isInteractive() bool { + return term.IsTerminal(int(os.Stdin.Fd())) +} + +func promptSelectProvider(keys []string, providerOverride string, nonInteractive bool) (string, error) { + if providerOverride = strings.TrimSpace(providerOverride); providerOverride != "" { + for _, k := range keys { + if k == providerOverride { + return providerOverride, nil + } + } + return "", errors.Errorf("ai_provider %s not in catalog", providerOverride) + } + providerOverride = resolveProviderFromEnv() + if providerOverride != "" { + for _, k := range keys { + if k == providerOverride { + return providerOverride, nil + } + } + return "", errors.Errorf("ai_provider %s not in catalog", providerOverride) + } + if nonInteractive || !isInteractive() { + return "", errors.Error("set --provider or AIPROXY_TEST_PROVIDER (or run in interactive terminal)") + } + fmt.Println("可用模型提供商 (catalog):") + for i, k := range keys { + fmt.Printf(" [%d] %s\n", i+1, k) + } + reader := bufio.NewReader(os.Stdin) + for { + fmt.Printf("请选择序号 [1-%d] 或直接输入 provider_key: ", len(keys)) + line, err := reader.ReadString('\n') + if err != nil { + return "", err + } + choice := strings.TrimSpace(line) + if choice == "" { + continue + } + if n, err := strconv.Atoi(choice); err == nil && n >= 1 && n <= len(keys) { + return keys[n-1], nil + } + for _, k := range keys { + if k == choice { + return choice, nil + } + } + fmt.Println("无效选择，请重试。") + } +} + +func promptSelectModel(models []string, providerKey, modelOverride string, nonInteractive bool) (string, error) { + if modelOverride = strings.TrimSpace(modelOverride); modelOverride != "" { + for _, m := range models { + if m == modelOverride { + return modelOverride, nil + } + } + return "", errors.Errorf("model_key %s not in provider %s catalog", modelOverride, providerKey) + } + modelOverride = resolveModelFromEnv() + if modelOverride != "" { + for _, m := range models { + if m == modelOverride { + return modelOverride, nil + } + } + return "", errors.Errorf("model_key %s not in provider %s catalog", modelOverride, providerKey) + } + + defaultM := DefaultModelForProvider(providerKey) + found := false + for _, m := range models { + if m == defaultM { + found = true + break + } + } + if !found { + defaultM = models[0] + } + if nonInteractive || !isInteractive() { + return defaultM, nil + } + + fmt.Printf("提供商 %s 的模型:\n", providerKey) + for i, m := range models { + fmt.Printf(" [%d] %s\n", i+1, m) + } + reader := bufio.NewReader(os.Stdin) + for { + fmt.Printf("请选择序号 [1-%d] 或输入 model_key [默认: %s]: ", len(models), defaultM) + line, err := reader.ReadString('\n') + if err != nil { + return "", err + } + choice := strings.TrimSpace(line) + if choice == "" { + return defaultM, nil + } + if n, err := strconv.Atoi(choice); err == nil && n >= 1 && n <= len(models) { + return models[n-1], nil + } + for _, m := range models { + if m == choice { + return choice, nil + } + } + fmt.Println("无效选择，请重试。") + } +} + +func promptApiKey(providerKey, apiKeyOverride string, nonInteractive bool) (string, error) { + if apiKeyOverride = strings.TrimSpace(apiKeyOverride); apiKeyOverride != "" { + return apiKeyOverride, nil + } + if v := resolveApiKeyFromEnv(providerKey); v != "" { + fmt.Println("使用环境变量中的 API Key（未回显）") + return v, nil + } + if nonInteractive || !isInteractive() { + return "", errors.Errorf("未设置 API Key：--api-key 或 AIPROXY_TEST_API_KEY 或 %s 对应的环境变量", providerKey) + } + fmt.Printf("请输入 %s 的 API Key（不回显）: ", providerKey) + b, err := term.ReadPassword(int(os.Stdin.Fd())) + fmt.Println() + if err != nil { + return "", err + } + key := strings.TrimSpace(string(b)) + if key == "" { + return "", errors.Error("API Key 不能为空") + } + return key, nil +} + +func promptRunStream(skipStream bool, nonInteractive bool) bool { + if skipStream || envSkipStream(false) { + return false + } + if v := os.Getenv("AIPROXY_TEST_SKIP_STREAM"); v == "0" { + return true + } + if v := os.Getenv("AIPROXY_FT_SKIP_STREAM"); v == "0" { + return true + } + if nonInteractive || !isInteractive() { + return true + } + fmt.Print("是否执行流式测试 (stream=true)? [Y/n]: ") + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + return true + } + switch strings.TrimSpace(strings.ToLower(line)) { + case "n", "no": + return false + default: + return true + } +} + +func promptLine(prompt, defaultVal string, nonInteractive bool) (string, error) { + if defaultVal != "" { + fmt.Printf("%s [%s]: ", prompt, defaultVal) + } else { + fmt.Printf("%s: ", prompt) + } + if nonInteractive || !isInteractive() { + if defaultVal == "" { + return "", errors.Errorf("empty input for %s in non-interactive mode", prompt) + } + return defaultVal, nil + } + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + return "", err + } + val := strings.TrimSpace(line) + if val == "" { + val = defaultVal + } + if val == "" { + return "", errors.Errorf("empty input for %s", prompt) + } + return val, nil +} + +func promptYesNo(prompt string, defaultYes bool, nonInteractive bool) bool { + if nonInteractive || !isInteractive() { + return defaultYes + } + if defaultYes { + fmt.Printf("%s [Y/n]: ", prompt) + } else { + fmt.Printf("%s [y/N]: ", prompt) + } + reader := bufio.NewReader(os.Stdin) + line, err := reader.ReadString('\n') + if err != nil { + return defaultYes + } + switch strings.TrimSpace(strings.ToLower(line)) { + case "n", "no": + return false + case "y", "yes": + return true + default: + return defaultYes + } +} diff --git a/pkg/aiproxy/ft/options.go b/pkg/aiproxy/ft/options.go new file mode 100644 index 0000000000..3d37a39449 --- /dev/null +++ b/pkg/aiproxy/ft/options.go @@ -0,0 +1,55 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +type ChatOptions struct { + Provider string `help:"provider_key from catalog"` + Model string `help:"model_key from catalog"` + ApiKey string `help:"upstream API key (or use env AIPROXY_TEST_API_KEY / provider-specific env)"` + Prompt string `help:"user message content"` + KeyName string `help:"ai_key resource name override"` + VkName string `help:"ai_virtual_key resource name override"` + RoutingName string `help:"ai_routing resource name override"` + AiproxyURL string `help:"aiproxy public base URL (default: AIPROXY_URL or endpoint-list)"` + SkipStream bool `help:"skip streaming test"` + NonInteractive bool `help:"fail instead of prompting (also AIPROXY_TEST_NONINTERACTIVE=1)"` + KeepResources bool `help:"keep created test resources after run (AIPROXY_TEST_KEEP_RESOURCES=1)"` +} + +type AnthropicOptions struct { + Provider string `help:"provider_key (default anthropic; use openai for DeepSeek)"` + Model string `help:"model_key"` + ApiKey string `help:"upstream API key"` + Prompt string `help:"user message content"` + KeyName string `help:"ai_key resource name override"` + VkName string `help:"ai_virtual_key resource name override"` + RoutingName string `help:"ai_routing resource name override"` + AiproxyURL string `help:"aiproxy public base URL"` + UpstreamBaseURL string `help:"optional reminder: ensure ai_provider config.base_url is set"` + SkipStream bool `help:"skip streaming test"` + NonInteractive bool `help:"fail instead of prompting"` + KeepResources bool `help:"keep created test resources after run (AIPROXY_TEST_KEEP_RESOURCES=1)"` +} + +type ProviderCreateOptions struct { + Name string `help:"ai_provider resource name"` + ProviderKey string `help:"provider_key (unique catalog identifier)"` + BaseURL string `help:"config.base_url for OpenAI-compatible upstream"` + Config string `help:"full provider config JSON (overrides --base-url)"` + Enabled bool `help:"create with --enabled"` + DeleteExisting bool `help:"delete existing resource with same name before create"` + NonInteractive bool `help:"fail instead of prompting (AIPROXY_PROVIDER_TEST_NONINTERACTIVE=1)"` + KeepResources bool `help:"keep created ai_provider after test (AIPROXY_TEST_KEEP_RESOURCES=1)"` +} diff --git a/pkg/aiproxy/ft/provider_create.go b/pkg/aiproxy/ft/provider_create.go new file mode 100644 index 0000000000..9345825d4c --- /dev/null +++ b/pkg/aiproxy/ft/provider_create.go @@ -0,0 +1,203 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" + apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy" +) + +func providerCreateNonInteractive(explicit bool) bool { + if explicit { + return true + } + return envTruthy("AIPROXY_PROVIDER_TEST_NONINTERACTIVE", "AIPROXY_PROVIDER_FT_NONINTERACTIVE") +} + +func buildProviderConfigJSON(configJSON, baseURL string) (jsonutils.JSONObject, error) { + if strings.TrimSpace(configJSON) != "" { + obj, err := jsonutils.ParseString(configJSON) + if err != nil { + return nil, errors.Wrap(err, "parse config JSON") + } + return obj, nil + } + baseURL = strings.TrimSpace(baseURL) + if baseURL == "" { + return nil, errors.Error("set --base-url or --config (or AIPROXY_PROVIDER_TEST_BASE_URL)") + } + return jsonutils.Marshal(map[string]string{"base_url": baseURL}), nil +} + +func collectProviderCreateInputs(opts *ProviderCreateOptions) error { + nonInteractive := providerCreateNonInteractive(opts.NonInteractive) + suffix := time.Now().Format("20060102150405") + + if nonInteractive { + if opts.Name == "" { + opts.Name = fmt.Sprintf("aiproxy-provider-test-%s", suffix) + } + if opts.ProviderKey == "" { + opts.ProviderKey = fmt.Sprintf("custom-test-%s", suffix) + } + if opts.Config == "" && opts.BaseURL == "" { + opts.BaseURL = envFirst("AIPROXY_PROVIDER_TEST_BASE_URL", "AIPROXY_PROVIDER_FT_BASE_URL") + } + if opts.Config == "" { + opts.Config = envFirst("AIPROXY_PROVIDER_TEST_CONFIG", "AIPROXY_PROVIDER_FT_CONFIG") + } + if !opts.Enabled { + opts.Enabled = envTruthy("AIPROXY_PROVIDER_TEST_ENABLED") || !envTruthy("AIPROXY_PROVIDER_TEST_DISABLED") + } + if !opts.DeleteExisting { + opts.DeleteExisting = envTruthy("AIPROXY_PROVIDER_TEST_DELETE_EXISTING", "AIPROXY_PROVIDER_FT_DELETE_EXISTING") + } + return nil + } + + fmt.Println("=== ai_provider 创建测试 ===") + fmt.Println("将创建自定义 ai_provider（provider_key 不可与 catalog 重复）。") + fmt.Println() + + var err error + if opts.Name == "" { + opts.Name, err = promptLine("资源名称 (climc 第一个参数 NAME)", fmt.Sprintf("aiproxy-provider-test-%s", suffix), false) + if err != nil { + return err + } + } + if opts.ProviderKey == "" { + opts.ProviderKey, err = promptLine("provider_key (唯一标识)", opts.Name, false) + if err != nil { + return err + } + } + if opts.Config == "" && opts.BaseURL == "" { + opts.BaseURL, err = promptLine("config.base_url (OpenAI 兼容上游)", "https://api.openai.com", false) + if err != nil { + return err + } + } + if !opts.Enabled && !envTruthy("AIPROXY_PROVIDER_TEST_DISABLED") { + opts.Enabled = promptYesNo("创建后启用 (--enabled)?", true, false) + } + return nil +} + +func deleteExistingProvider(session *mcclient.ClientSession, name string, deleteIfExists bool) error { + if _, err := apmodules.AiProviders.Get(session, name, nil); err != nil { + return nil + } + if !deleteIfExists { + return errors.Errorf("ai_provider %s already exists; use --delete-existing or AIPROXY_PROVIDER_TEST_DELETE_EXISTING=1", name) + } + fmt.Printf("deleting existing ai_provider %s\n", name) + _, err := apmodules.AiProviders.Delete(session, name, nil) + return err +} + +func RunProviderCreateTest(session *mcclient.ClientSession, opts *ProviderCreateOptions) error { + tracker := NewResourceTracker(envKeepResources(opts.KeepResources)) + defer tracker.Cleanup(session) + + if err := collectProviderCreateInputs(opts); err != nil { + return err + } + + configObj, err := buildProviderConfigJSON(opts.Config, opts.BaseURL) + if err != nil { + return err + } + + if err := deleteExistingProvider(session, opts.Name, opts.DeleteExisting); err != nil { + return err + } + + Step("create ai_provider") + params := jsonutils.NewDict() + params.Set("name", jsonutils.NewString(opts.Name)) + params.Set("provider_key", jsonutils.NewString(opts.ProviderKey)) + params.Set("config", configObj) + if opts.Enabled { + params.Set("enabled", jsonutils.JSONTrue) + } + if _, err := apmodules.AiProviders.Create(session, params); err != nil { + return errors.Wrap(err, "ai-provider-create") + } + tracker.createdProvider = opts.Name + + Step("verify ai-provider-show") + row, err := apmodules.AiProviders.Get(session, opts.Name, nil) + if err != nil { + return err + } + pk, _ := row.GetString("provider_key") + if pk != opts.ProviderKey { + return errors.Errorf("provider_key mismatch: got %s want %s", pk, opts.ProviderKey) + } + if opts.BaseURL != "" { + base, _ := row.GetString("config", "base_url") + if base != opts.BaseURL { + return errors.Errorf("base_url mismatch: got %s want %s", base, opts.BaseURL) + } + } + enabled, _ := row.Bool("enabled") + if opts.Enabled && !enabled { + return errors.Error("expected enabled=true") + } + summary := jsonutils.NewDict() + for _, k := range []string{"id", "name", "provider_key", "enabled"} { + if row.Contains(k) { + val, _ := row.Get(k) + summary.Set(k, val) + } + } + if row.Contains("config") { + val, _ := row.Get("config") + summary.Set("config", val) + } + fmt.Println(summary.PrettyString()) + + Step("verify ai-provider-list filter") + query := jsonutils.NewDict() + query.Set("provider_key", jsonutils.NewString(opts.ProviderKey)) + result, err := apmodules.AiProviders.List(session, query) + if err != nil { + return err + } + count := 0 + for _, item := range result.Data { + name, _ := item.GetString("name") + if name == opts.Name { + count++ + } + } + if count < 1 { + return errors.Error("ai-provider-list --provider-key did not return created row") + } + + fmt.Println() + fmt.Println("OK: ai_provider create test passed.") + fmt.Printf(" name: %s\n", opts.Name) + fmt.Printf(" provider_key: %s\n", opts.ProviderKey) + return nil +} diff --git a/pkg/aiproxy/ft/sse.go b/pkg/aiproxy/ft/sse.go new file mode 100644 index 0000000000..e7332c8bae --- /dev/null +++ b/pkg/aiproxy/ft/sse.go @@ -0,0 +1,165 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ft + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +func openAIChoiceDelta(obj jsonutils.JSONObject, field string) (string, error) { + arr, err := obj.GetArray("choices") + if err != nil || len(arr) == 0 { + return "", err + } + if field == "delta" { + return arr[0].GetString("delta", "content") + } + return arr[0].GetString("message", "content") +} + +func parseOpenAIStreamDelta(payload string) (string, error) { + payload = strings.TrimSpace(payload) + if payload == "" || payload == "[DONE]" { + return "", nil + } + obj, err := jsonutils.ParseString(payload) + if err != nil { + return "", err + } + if obj.Contains("error") { + return "", errors.Errorf("stream error event: %s", payload) + } + return openAIChoiceDelta(obj, "delta") +} + +func parseAnthropicStreamDelta(payload string) (string, error) { + payload = strings.TrimSpace(payload) + if payload == "" || payload == "[DONE]" { + return "", nil + } + obj, err := jsonutils.ParseString(payload) + if err != nil { + return "", err + } + delta, _ := obj.GetString("delta", "text") + return delta, nil +} + +func aggregateSSEStream(r io.Reader, parseDelta func(string) (string, error)) (string, error) { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + var aggregated strings.Builder + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "data:") { + continue + } + payload := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + delta, err := parseDelta(payload) + if err != nil { + return "", err + } + aggregated.WriteString(delta) + } + if err := scanner.Err(); err != nil { + return "", err + } + out := aggregated.String() + if out == "" { + return "", errors.Error("empty aggregated stream content") + } + return out, nil +} + +func extractOpenAIChatContent(body []byte) (string, error) { + obj, err := jsonutils.Parse(body) + if err != nil { + return "", err + } + content, err := openAIChoiceDelta(obj, "message") + if err != nil { + return "", errors.Wrap(err, "parse choices") + } + if content == "" { + return "", errors.Errorf("empty choices[0].message.content: %s", truncateBody(body)) + } + return content, nil +} + +func extractAnthropicTextContent(body []byte) (string, error) { + obj, err := jsonutils.Parse(body) + if err != nil { + return "", err + } + arr, err := obj.GetArray("content") + if err != nil { + return "", errors.Wrap(err, "parse content array") + } + for _, block := range arr { + typ, _ := block.GetString("type") + if typ == "text" { + text, _ := block.GetString("text") + if text != "" { + return text, nil + } + } + } + return "", errors.Errorf("empty anthropic text content block: %s", truncateBody(body)) +} + +func truncateBody(body []byte) string { + const max = 512 + if len(body) <= max { + return string(body) + } + return string(body[:max]) + "..." +} + +func printJSONBody(body []byte) error { + obj, err := jsonutils.Parse(body) + if err != nil { + fmt.Println(string(body)) + return nil + } + fmt.Println(obj.PrettyString()) + return nil +} + +func previewText(s string, max int) string { + if max <= 0 || len(s) <= max { + return s + } + return s[:max] + "..." +} + +func dumpStreamPreview(body []byte) { + lines := bytes.Split(body, []byte("\n")) + limit := 40 + if len(lines) < limit { + limit = len(lines) + } + fmt.Println("--- stream body (first lines) ---") + for i := 0; i < limit; i++ { + fmt.Println(string(lines[i])) + } +} diff --git a/pkg/aiproxy/handlers/anthropic_probe.go b/pkg/aiproxy/handlers/anthropic_probe.go new file mode 100644 index 0000000000..b7ae4ae854 --- /dev/null +++ b/pkg/aiproxy/handlers/anthropic_probe.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" +) + +const anthropicBasePrefix = "/ai/anthropic" + +// anthropicBaseProbeHandler answers HEAD on the Anthropic base URL (e.g. /ai/anthropic or /ai/anthropic/). +// appsrv SplitPath normalizes trailing slashes; one route covers both. Claude / Anthropic SDK probes +// base URL connectivity before POST /v1/messages. +func anthropicBaseProbeHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// anthropicMessagesHeadHandler answers HEAD on /ai/anthropic/v1/messages for path-existence probes. +// Returns 401 without virtual key (route exists, auth required); 204 when a key is present. +func anthropicMessagesHeadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if extractVirtualKey(r) == "" { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/pkg/aiproxy/handlers/anthropic_probe_test.go b/pkg/aiproxy/handlers/anthropic_probe_test.go new file mode 100644 index 0000000000..5c6e0cb554 --- /dev/null +++ b/pkg/aiproxy/handlers/anthropic_probe_test.go @@ -0,0 +1,63 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestAnthropicBaseProbeHandler(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, anthropicBasePrefix+"/", nil) + rec := httptest.NewRecorder() + anthropicBaseProbeHandler(context.Background(), rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status=%d want %d", rec.Code, http.StatusNoContent) + } + if rec.Body.Len() != 0 { + t.Fatalf("expected empty body, got %q", rec.Body.String()) + } +} + +func TestAnthropicMessagesHeadHandlerNoAuth(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, anthropicCompatAPIPrefix+"/messages", nil) + rec := httptest.NewRecorder() + anthropicMessagesHeadHandler(context.Background(), rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status=%d want %d", rec.Code, http.StatusUnauthorized) + } +} + +func TestAnthropicMessagesHeadHandlerWithAuth(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, anthropicCompatAPIPrefix+"/messages", nil) + req.Header.Set("Authorization", "Bearer sk-test-vk") + rec := httptest.NewRecorder() + anthropicMessagesHeadHandler(context.Background(), rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status=%d want %d", rec.Code, http.StatusNoContent) + } +} + +func TestAnthropicMessagesHeadHandlerWithXAiVirtualKey(t *testing.T) { + req := httptest.NewRequest(http.MethodHead, anthropicCompatAPIPrefix+"/messages", nil) + req.Header.Set(headerAiVirtualKey, "sk-test-vk") + rec := httptest.NewRecorder() + anthropicMessagesHeadHandler(context.Background(), rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status=%d want %d", rec.Code, http.StatusNoContent) + } +} diff --git a/pkg/aiproxy/handlers/chat_completions.go b/pkg/aiproxy/handlers/chat_completions.go index 67da057bd0..e0bbdac327 100644 --- a/pkg/aiproxy/handlers/chat_completions.go +++ b/pkg/aiproxy/handlers/chat_completions.go @@ -149,13 +149,9 @@ func chatCompletionsHandler(ctx context.Context, w http.ResponseWriter, r *http. } isStream, _ := dict.Bool("stream") - prov := providers.Get(up.ProviderKey) - if _, err := prov.BuildUpstreamRequest(&providers.ChatContext{ - ProviderKey: up.ProviderKey, - BaseURL: up.BaseURL, - APIKey: up.APIKey, - UpstreamModel: up.UpstreamModel, - }, dict, isStream); err != nil { + prov := providers.ChatProviderForUpstream(up.ProviderKey, up.APIMode) + chatCtx := providers.ChatContextFromUpstream(up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode) + if _, err := prov.BuildUpstreamRequest(chatCtx, dict, isStream); err != nil { httperrors.InvalidInputError(ctx, w, "provider request: %v", err) return } @@ -250,37 +246,9 @@ func chatCompletionWithKeyFailover( stream bool, timeout time.Duration, ) (*upstream.Response, *upstream.Error) { - tried := make(map[string]bool) - if up.AiKeyId != "" { - tried[up.AiKeyId] = true - } - var last *upstream.Error - for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { - upReq, err := buildProviderUpstream(up, dict, stream) - if err != nil { - return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} - } - reqCtx, cancel := context.WithTimeout(ctx, timeout) - resp, uerr := upstream.ChatCompletion(reqCtx, upReq) - cancel() - if uerr == nil { - models.RecordAiKeySuccess(up.AiKeyId) - return resp, nil - } - last = uerr - status := upstreamErrorStatusCode(uerr) - models.RecordAiKeyFailure(up.AiKeyId, status) - if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { - break - } - if err := models.RepickUpstreamAPIKey(up, tried); err != nil { - break - } - if up.AiKeyId != "" { - tried[up.AiKeyId] = true - } - } - return nil, last + return upstreamWithKeyFailover(ctx, up, timeout, func() (*upstream.Request, error) { + return buildProviderUpstream(up, dict, stream) + }) } func chatCompletionStreamWithKeyFailover( @@ -291,50 +259,18 @@ func chatCompletionStreamWithKeyFailover( prov providers.Provider, timeout time.Duration, ) (<-chan upstream.StreamChunk, *upstream.Error) { - tried := make(map[string]bool) - if up.AiKeyId != "" { - tried[up.AiKeyId] = true - } - var last *upstream.Error - for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { - upReq, err := buildProviderUpstream(up, dict, stream) - if err != nil { - return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} - } - reqCtx, cancel := context.WithTimeout(ctx, timeout) - ch, uerr := providerStreamChunks(reqCtx, up, upReq, prov) - if uerr != nil { - cancel() - } else { - ch = streamChunksWithCancel(ch, cancel) - } - if uerr == nil { - return ch, nil - } - last = uerr - status := upstreamErrorStatusCode(uerr) - models.RecordAiKeyFailure(up.AiKeyId, status) - if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { - break - } - if err := models.RepickUpstreamAPIKey(up, tried); err != nil { - break - } - if up.AiKeyId != "" { - tried[up.AiKeyId] = true - } - } - return nil, last + return upstreamStreamWithKeyFailover(ctx, up, timeout, func() (*upstream.Request, error) { + return buildProviderUpstream(up, dict, stream) + }, func(reqCtx context.Context, upReq *upstream.Request) (<-chan upstream.StreamChunk, *upstream.Error) { + return providerStreamChunks(reqCtx, up, upReq, prov) + }) } func buildProviderUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict, isStream bool) (*upstream.Request, error) { - prov := providers.Get(up.ProviderKey) - httpReq, err := prov.BuildUpstreamRequest(&providers.ChatContext{ - ProviderKey: up.ProviderKey, - BaseURL: up.BaseURL, - APIKey: up.APIKey, - UpstreamModel: up.UpstreamModel, - }, dict, isStream) + prov := providers.ChatProviderForUpstream(up.ProviderKey, up.APIMode) + httpReq, err := prov.BuildUpstreamRequest(providers.ChatContextFromUpstream( + up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode, + ), dict, isStream) if err != nil { return nil, err } @@ -347,12 +283,7 @@ func providerStreamChunks( upReq *upstream.Request, prov providers.Provider, ) (<-chan upstream.StreamChunk, *upstream.Error) { - chatCtx := &providers.ChatContext{ - ProviderKey: up.ProviderKey, - BaseURL: up.BaseURL, - APIKey: up.APIKey, - UpstreamModel: up.UpstreamModel, - } + chatCtx := providers.ChatContextFromUpstream(up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode) if providers.OpenAIStreamPassthrough(prov, chatCtx) { return upstream.ChatCompletionStream(ctx, upReq) } diff --git a/pkg/aiproxy/handlers/handlers.go b/pkg/aiproxy/handlers/handlers.go index 921b615edf..96a5c8e368 100644 --- a/pkg/aiproxy/handlers/handlers.go +++ b/pkg/aiproxy/handlers/handlers.go @@ -28,6 +28,7 @@ import ( const ( openaiCompatAPIPrefix = "/ai/openai/v1" + anthropicCompatAPIPrefix = "/ai/anthropic/v1" openaiLongProcessTimeout = 2 * time.Hour openaiShortProcessTimeout = 5 * time.Minute ) @@ -44,6 +45,10 @@ func InitHandlers(app *appsrv.Application, isSlave bool) { app.AddHandler2("POST", openaiCompatAPIPrefix+"/chat/completions", chatCompletionsHandler, nil, "aiproxy_openai_v1_chat_completions", nil). SetProcessTimeout(openaiLongProcessTimeout) + app.AddHandler2("POST", anthropicCompatAPIPrefix+"/messages", messagesHandler, nil, "aiproxy_anthropic_v1_messages", nil). + SetProcessTimeout(openaiLongProcessTimeout) + app.AddHandler2("HEAD", anthropicBasePrefix, anthropicBaseProbeHandler, nil, "aiproxy_anthropic_base_probe", nil) + app.AddHandler2("HEAD", anthropicCompatAPIPrefix+"/messages", anthropicMessagesHeadHandler, nil, "aiproxy_anthropic_v1_messages_head", nil) app.AddHandler2("POST", openaiCompatAPIPrefix+"/completions", completionsHandler, nil, "aiproxy_openai_v1_completions", nil). SetProcessTimeout(openaiLongProcessTimeout) app.AddHandler2("POST", openaiCompatAPIPrefix+"/embeddings", embeddingsHandler, nil, "aiproxy_openai_v1_embeddings", nil). diff --git a/pkg/aiproxy/handlers/messages.go b/pkg/aiproxy/handlers/messages.go new file mode 100644 index 0000000000..6d8bd7ba3a --- /dev/null +++ b/pkg/aiproxy/handlers/messages.go @@ -0,0 +1,379 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/providers/messages" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +// messagesHandler implements Anthropic-compatible POST /ai/anthropic/v1/messages. +func messagesHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "only POST is supported") + return + } + + defer r.Body.Close() + raw, err := io.ReadAll(r.Body) + if err != nil { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "read body: %v", err) + return + } + + body, err := jsonutils.Parse(raw) + if err != nil { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "invalid JSON body: %v", err) + return + } + dict, ok := body.(*jsonutils.JSONDict) + if !ok { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "body must be a JSON object") + return + } + + reqID := newMessagesReqID() + + vk := extractVirtualKey(r) + userCred := auth.AdminCredential() + up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + + if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + var vkLim *api.SAiVirtualKeyLimits + if up.MaxTokensPerRequest > 0 { + vkLim = &api.SAiVirtualKeyLimits{ + MaxTokensPerRequest: up.MaxTokensPerRequest, + } + } + if err := models.EnforceVirtualKeyMaxTokens(dict, vkLim); err != nil { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err) + return + } + + adapter, err := messages.GetAdapter(up.ProviderKey, up.APIMode) + if err != nil { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err) + return + } + + isStream, _ := dict.Bool("stream") + logMessagesClientRequest(reqID, r, dict, up, isStream) + chatCtx := providers.ChatContextFromUpstream(up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode) + if _, err := adapter.BuildUpstreamRequest(chatCtx, dict, isStream); err != nil { + writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "provider request: %v", err) + return + } + + timeout := 120 * time.Second + if isStream { + timeout = 2 * time.Hour + } + + build := func() (*upstream.Request, error) { + req, err := buildMessagesUpstream(up, adapter, dict, isStream) + if err == nil { + logMessagesUpstreamRequest(reqID, req) + } + return req, err + } + + if !isStream { + prov := providers.Get(up.ProviderKey) + resp, uerr := upstreamWithKeyFailover(ctx, up, timeout, build) + if uerr != nil { + logMessagesError(reqID, "upstream error status=%d body=%s", uerr.StatusCode, truncateLogBytes(uerr.Body, messagesDebugLogMax)) + writeMessagesUpstreamError(ctx, w, adapter, uerr) + return + } + bodyOut := resp.Body + logMessagesUpstreamResponse(reqID, bodyOut) + if norm, nerr := adapter.NormalizeResponse(prov, bodyOut); nerr == nil && len(norm) > 0 { + bodyOut = norm + } + logMessagesClientResponse(reqID, bodyOut) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(bodyOut) + return + } + + if adapter.AnthropicStreamPassthrough() { + ch, uerr := upstreamRawStreamWithKeyFailover(ctx, up, timeout, build, upstream.ChatCompletionStreamRaw) + if uerr != nil { + logMessagesError(reqID, "upstream stream error status=%d body=%s", uerr.StatusCode, truncateLogBytes(uerr.Body, messagesDebugLogMax)) + writeMessagesUpstreamError(ctx, w, adapter, uerr) + return + } + writeAnthropicPassthroughStream(ctx, w, ch, up.AiKeyId, reqID) + return + } + + prov := providers.Get(up.ProviderKey) + ch, uerr := upstreamStreamWithKeyFailover(ctx, up, timeout, build, func(reqCtx context.Context, upReq *upstream.Request) (<-chan upstream.StreamChunk, *upstream.Error) { + return messagesOpenAIStreamChunks(reqCtx, up, upReq, prov, reqID) + }) + if uerr != nil { + logMessagesError(reqID, "upstream stream error status=%d body=%s", uerr.StatusCode, truncateLogBytes(uerr.Body, messagesDebugLogMax)) + writeMessagesUpstreamError(ctx, w, adapter, uerr) + return + } + writeAnthropicTranslatedStream(ctx, w, ch, adapter, up.UpstreamModel, up.AiKeyId, reqID) +} + +func buildMessagesUpstream( + up *models.ChatUpstream, + adapter providerapi.MessagesAdapter, + dict *jsonutils.JSONDict, + isStream bool, +) (*upstream.Request, error) { + httpReq, err := adapter.BuildUpstreamRequest(providers.ChatContextFromUpstream( + up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode, + ), dict, isStream) + if err != nil { + return nil, err + } + return providers.ToUpstreamRequest(httpReq, up.APIKey), nil +} + +func messagesOpenAIStreamChunks( + ctx context.Context, + up *models.ChatUpstream, + upReq *upstream.Request, + prov providers.Provider, + reqID string, +) (<-chan upstream.StreamChunk, *upstream.Error) { + chatCtx := &providers.ChatContext{ + ProviderKey: up.ProviderKey, + BaseURL: up.BaseURL, + APIKey: up.APIKey, + UpstreamModel: up.UpstreamModel, + } + if providers.OpenAIStreamPassthrough(prov, chatCtx) { + ch, uerr := upstream.ChatCompletionStream(ctx, upReq) + if uerr != nil { + return nil, uerr + } + out := make(chan upstream.StreamChunk, 16) + go func() { + defer close(out) + seq := 0 + for chunk := range ch { + if len(chunk.Data) > 0 { + seq++ + logMessagesUpstreamStreamChunk(reqID, seq, chunk.Data) + } + out <- chunk + } + }() + return out, nil + } + rawCh, uerr := upstream.ChatCompletionStreamRaw(ctx, upReq) + if uerr != nil { + return nil, uerr + } + out := make(chan upstream.StreamChunk, 16) + go func() { + defer close(out) + state := &providers.StreamState{Model: up.UpstreamModel} + seq := 0 + for evt := range rawCh { + seq++ + logMessagesUpstreamStreamChunk(reqID, seq, evt.Data) + chunks, err := prov.ConvertStreamEvent(evt.Event, evt.Data, state) + if err != nil { + msg, _ := json.Marshal(map[string]interface{}{ + "error": map[string]interface{}{"message": err.Error()}, + }) + out <- upstream.StreamChunk{Data: msg} + return + } + for _, c := range chunks { + if len(c.Data) > 0 { + out <- upstream.StreamChunk{Data: c.Data} + } + if c.Done { + out <- upstream.StreamChunk{Done: true} + return + } + } + } + }() + return out, nil +} + +func writeAnthropicError(ctx context.Context, w http.ResponseWriter, status int, errType, format string, args ...interface{}) { + if ctx.Err() != nil { + return + } + msg := fmt.Sprintf(format, args...) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write(openai.NewAnthropicErrorBody(errType, msg)) +} + +func writeMessagesUpstreamError(ctx context.Context, w http.ResponseWriter, adapter providerapi.MessagesAdapter, uerr *upstream.Error) { + if ctx.Err() != nil { + return + } + status := http.StatusBadGateway + if uerr != nil && uerr.StatusCode > 0 { + status = uerr.StatusCode + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if uerr != nil && len(uerr.Body) > 0 { + if adapter.AnthropicStreamPassthrough() { + _, _ = w.Write(uerr.Body) + return + } + _, _ = w.Write(openai.OpenAIErrorToAnthropic(uerr.Body, status)) + return + } + msg := "upstream request failed" + if uerr != nil { + msg = uerr.Error() + } + _, _ = w.Write(openai.NewAnthropicErrorBody("api_error", msg)) +} + +func writeAnthropicPassthroughStream(ctx context.Context, w http.ResponseWriter, ch <-chan upstream.RawSSEEvent, aiKeyId string, reqID string) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flushIf(w) + + streamOK := true + seq := 0 + for evt := range ch { + seq++ + logMessagesClientStreamPassthrough(reqID, seq, evt.Event, evt.Data) + if evt.Event != "" { + _, _ = fmt.Fprintf(w, "event: %s\n", evt.Event) + } + if len(evt.Data) > 0 { + _, _ = fmt.Fprintf(w, "data: %s\n\n", evt.Data) + } else if evt.Event != "" { + _, _ = fmt.Fprint(w, "\n") + } + flushIf(w) + } + if streamOK && aiKeyId != "" { + models.RecordAiKeySuccess(aiKeyId) + } +} + +func writeAnthropicTranslatedStream( + ctx context.Context, + w http.ResponseWriter, + ch <-chan upstream.StreamChunk, + adapter providerapi.MessagesAdapter, + requestModel string, + aiKeyId string, + reqID string, +) { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flushIf(w) + + state := adapter.NewStreamState(requestModel) + streamOK := true + outSeq := 0 + for chunk := range ch { + if chunk.Done { + events, err := adapter.ConvertStreamPayload(state, nil, true) + if err != nil { + logMessagesError(reqID, "stream convert end error: %v", err) + streamOK = false + break + } + outSeq++ + logMessagesClientStreamEvents(reqID, outSeq, events) + writeAnthropicSSEEvents(w, events) + break + } + if len(chunk.Data) == 0 { + continue + } + if isAnthropicUpstreamErrorChunk(chunk.Data) { + logMessagesError(reqID, "upstream stream error chunk: %s", truncateLogBytes(chunk.Data, messagesDebugLogMax)) + streamOK = false + if aiKeyId != "" { + models.RecordAiKeyFailure(aiKeyId, parseUpstreamErrorStatus(chunk.Data)) + } + _, _ = fmt.Fprintf(w, "event: error\ndata: %s\n\n", openai.OpenAIErrorToAnthropic(chunk.Data, http.StatusBadGateway)) + flushIf(w) + break + } + events, err := adapter.ConvertStreamPayload(state, chunk.Data, false) + if err != nil { + logMessagesError(reqID, "stream convert error: %v upstream_chunk=%s", err, truncateLogBytes(chunk.Data, messagesDebugLogMax)) + streamOK = false + break + } + outSeq++ + logMessagesClientStreamEvents(reqID, outSeq, events) + writeAnthropicSSEEvents(w, events) + } + if streamOK && aiKeyId != "" { + models.RecordAiKeySuccess(aiKeyId) + } +} + +func writeAnthropicSSEEvents(w http.ResponseWriter, events []providerapi.AnthropicStreamChunk) { + for _, evt := range events { + if evt.Event != "" { + _, _ = fmt.Fprintf(w, "event: %s\n", evt.Event) + } + if len(evt.Data) > 0 { + _, _ = fmt.Fprintf(w, "data: %s\n\n", string(evt.Data)) + } else if evt.Event != "" { + _, _ = fmt.Fprint(w, "\n") + } + flushIf(w) + } +} + +func isAnthropicUpstreamErrorChunk(data []byte) bool { + var wrap struct { + Error interface{} `json:"error"` + } + return json.Unmarshal(data, &wrap) == nil && wrap.Error != nil +} diff --git a/pkg/aiproxy/handlers/messages_debug.go b/pkg/aiproxy/handlers/messages_debug.go new file mode 100644 index 0000000000..bd5f217626 --- /dev/null +++ b/pkg/aiproxy/handlers/messages_debug.go @@ -0,0 +1,148 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "fmt" + "net/http" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" +) + +const messagesDebugLogMax = 4096 + +func newMessagesReqID() string { + return fmt.Sprintf("%08x", time.Now().UnixNano()) +} + +func logMessagesClientRequest(reqID string, r *http.Request, body *jsonutils.JSONDict, up *models.ChatUpstream, stream bool) { + model, _ := body.GetString("model") + log.Debugf( + "aiproxy messages [%s] client request method=%s path=%s query=%s stream=%v provider=%s upstream_model=%s vk=%s body=%s", + reqID, + r.Method, + r.URL.Path, + r.URL.RawQuery, + stream, + up.ProviderKey, + up.UpstreamModel, + maskSecret(extractVirtualKey(r)), + truncateLogBytes([]byte(body.String()), messagesDebugLogMax), + ) + if model != "" && model != up.UpstreamModel { + log.Debugf("aiproxy messages [%s] client model=%q routed upstream_model=%q", reqID, model, up.UpstreamModel) + } +} + +func logMessagesUpstreamRequest(reqID string, req *upstream.Request) { + if req == nil { + return + } + url := strings.TrimSpace(req.URL) + if url == "" { + url = strings.TrimSpace(req.BaseURL) + } + log.Debugf( + "aiproxy messages [%s] upstream request url=%s body=%s", + reqID, + url, + truncateLogBytes(req.Body, messagesDebugLogMax), + ) +} + +func logMessagesUpstreamStreamChunk(reqID string, seq int, data []byte) { + log.Debugf( + "aiproxy messages [%s] upstream stream chunk #%d: %s", + reqID, + seq, + truncateLogBytes(data, messagesDebugLogMax), + ) +} + +func logMessagesClientStreamEvents(reqID string, seq int, events []providerapi.AnthropicStreamChunk) { + if len(events) == 0 { + log.Debugf("aiproxy messages [%s] client stream out #%d: (no anthropic events)", reqID, seq) + return + } + for i, evt := range events { + log.Debugf( + "aiproxy messages [%s] client stream out #%d.%d event=%s data=%s", + reqID, + seq, + i, + evt.Event, + truncateLogBytes(evt.Data, messagesDebugLogMax), + ) + } +} + +func logMessagesUpstreamResponse(reqID string, body []byte) { + log.Debugf( + "aiproxy messages [%s] upstream response body=%s", + reqID, + truncateLogBytes(body, messagesDebugLogMax), + ) +} + +func logMessagesClientResponse(reqID string, body []byte) { + log.Debugf( + "aiproxy messages [%s] client response body=%s", + reqID, + truncateLogBytes(body, messagesDebugLogMax), + ) +} + +func logMessagesClientStreamPassthrough(reqID string, seq int, event string, data []byte) { + log.Debugf( + "aiproxy messages [%s] client stream passthrough #%d event=%s data=%s", + reqID, + seq, + event, + truncateLogBytes(data, messagesDebugLogMax), + ) +} + +func logMessagesError(reqID string, format string, args ...interface{}) { + log.Debugf("aiproxy messages [%s] "+format, append([]interface{}{reqID}, args...)...) +} + +func truncateLogBytes(b []byte, max int) string { + if len(b) == 0 { + return "" + } + s := strings.TrimSpace(string(b)) + if max <= 0 || len(s) <= max { + return s + } + return s[:max] + fmt.Sprintf("...(truncated, total=%d)", len(s)) +} + +func maskSecret(s string) string { + s = strings.TrimSpace(s) + if s == "" { + return "" + } + if len(s) <= 8 { + return "***" + } + return s[:4] + "..." + s[len(s)-4:] +} diff --git a/pkg/aiproxy/handlers/upstream_failover.go b/pkg/aiproxy/handlers/upstream_failover.go new file mode 100644 index 0000000000..e614ab725c --- /dev/null +++ b/pkg/aiproxy/handlers/upstream_failover.go @@ -0,0 +1,167 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + "time" + + "yunion.io/x/onecloud/pkg/aiproxy/models" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" +) + +type upstreamRequestBuilder func() (*upstream.Request, error) + +func upstreamWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + timeout time.Duration, + build upstreamRequestBuilder, +) (*upstream.Response, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := build() + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + resp, uerr := upstream.ChatCompletion(reqCtx, upReq) + cancel() + if uerr == nil { + models.RecordAiKeySuccess(up.AiKeyId) + return resp, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +type streamChunkProducer func(ctx context.Context, upReq *upstream.Request) (<-chan upstream.StreamChunk, *upstream.Error) + +func upstreamStreamWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + timeout time.Duration, + build upstreamRequestBuilder, + produce streamChunkProducer, +) (<-chan upstream.StreamChunk, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := build() + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + ch, uerr := produce(reqCtx, upReq) + if uerr != nil { + cancel() + } else { + ch = streamChunksWithCancel(ch, cancel) + } + if uerr == nil { + return ch, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +type rawSSEProducer func(ctx context.Context, upReq *upstream.Request) (<-chan upstream.RawSSEEvent, *upstream.Error) + +func upstreamRawStreamWithKeyFailover( + ctx context.Context, + up *models.ChatUpstream, + timeout time.Duration, + build upstreamRequestBuilder, + produce rawSSEProducer, +) (<-chan upstream.RawSSEEvent, *upstream.Error) { + tried := make(map[string]bool) + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + var last *upstream.Error + for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ { + upReq, err := build() + if err != nil { + return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()} + } + reqCtx, cancel := context.WithTimeout(ctx, timeout) + ch, uerr := produce(reqCtx, upReq) + if uerr != nil { + cancel() + } else { + ch = rawSSEWithCancel(ch, cancel) + } + if uerr == nil { + return ch, nil + } + last = uerr + status := upstreamErrorStatusCode(uerr) + models.RecordAiKeyFailure(up.AiKeyId, status) + if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts { + break + } + if err := models.RepickUpstreamAPIKey(up, tried); err != nil { + break + } + if up.AiKeyId != "" { + tried[up.AiKeyId] = true + } + } + return nil, last +} + +func rawSSEWithCancel(ch <-chan upstream.RawSSEEvent, cancel context.CancelFunc) <-chan upstream.RawSSEEvent { + out := make(chan upstream.RawSSEEvent, 16) + go func() { + defer cancel() + defer close(out) + for evt := range ch { + out <- evt + } + }() + return out +} diff --git a/pkg/aiproxy/models/ai_key_resolve.go b/pkg/aiproxy/models/ai_key_resolve.go index 04a690a255..1ac97b9ae3 100644 --- a/pkg/aiproxy/models/ai_key_resolve.go +++ b/pkg/aiproxy/models/ai_key_resolve.go @@ -122,7 +122,7 @@ type resolvedUpstreamAPIKey struct { // MaxAiKeyFailoverAttempts is how many alternate ai_key rows to try per chat request. const MaxAiKeyFailoverAttempts = 8 -// resolveUpstreamAPIKey picks an ai_key (weighted + dynamic penalty) or provider.config api_key. +// resolveUpstreamAPIKey picks an enabled ai_key for the provider (weighted + dynamic penalty). func resolveUpstreamAPIKey(prov *SAiProvider, modelKey string) (*resolvedUpstreamAPIKey, error) { return resolveUpstreamAPIKeyExcluding(prov, modelKey, nil) } @@ -175,14 +175,7 @@ func resolveUpstreamAPIKeyExcluding(prov *SAiProvider, modelKey string, exclude if hasSecretKey { return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "no available ai_key for catalog model %q (check weight, cooldown, allowed_model_keys)", modelKey) } - if prov.Config == nil { - return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty") - } - apiKey := strings.TrimSpace(prov.Config.ResolvedAPIKey()) - if apiKey == "" { - return nil, errors.Wrap(httperrors.ErrInvalidStatus, "set api_key on ai_provider or add an enabled ai_key with secret for this provider") - } - return &resolvedUpstreamAPIKey{Secret: apiKey}, nil + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "add an enabled ai_key with secret for this provider") } // RepickUpstreamAPIKey selects another ai_key for the same provider/model, excluding already tried ids. diff --git a/pkg/aiproxy/models/ai_providers.go b/pkg/aiproxy/models/ai_providers.go index 6d8da98c70..b3bb892726 100644 --- a/pkg/aiproxy/models/ai_providers.go +++ b/pkg/aiproxy/models/ai_providers.go @@ -16,14 +16,17 @@ package models import ( "context" + "fmt" "strings" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/aiproxy" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -35,7 +38,7 @@ type SAiProvider struct { // ProviderKey selects the upstream adapter implementation (e.g. openai, vllm, aliyun). // Multiple ai_provider rows may share the same provider_key with different config. ProviderKey string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` - // Config is a JSON snapshot of provider connectivity (base_url, optional api_key). + // Config is a JSON snapshot of provider connectivity (base_url, api_mode). Config *api.SAiProviderConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"` // LlmDeploymentId and LlmId link this provider to an llm_deployment replica (set by llm sync). LlmDeploymentId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user" index:"true"` @@ -125,10 +128,25 @@ func (manager *SAiProviderManager) ValidateCreateData( input.ProviderKey = pk input.Config = normalizeAiProviderConfig(input.Config) - if err := validateAiProviderConfig(input.Config); err != nil { + if err := validateAiProviderConfig(input.Config, input.ProviderKey); err != nil { return input, err } + input.Secret = strings.TrimSpace(input.Secret) + + if api.IsCustomProviderKey(input.ProviderKey) { + if input.Secret == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "secret is required for provider_key custom") + } + if input.Config == nil || strings.TrimSpace(input.Config.ResolvedBaseURL()) == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required for provider_key custom") + } + } + + if input.Enabled == nil && input.Disabled == nil { + input.SetEnabled() + } + if strings.TrimSpace(input.Name) == "" { input.Name = pk } @@ -139,9 +157,97 @@ func (manager *SAiProviderManager) ValidateCreateData( return input, err } + if strings.TrimSpace(input.Secret) != "" { + var err error + input.ModelKeys, err = normalizeProviderModelKeys(input.ModelKeys) + if err != nil { + return input, err + } + if len(input.ModelKeys) == 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "model_keys is required when secret is provided") + } + if err := probeProviderConnectivity(ctx, input.ProviderKey, input.Secret, input.Config); err != nil { + return input, err + } + } + return input, nil } +func (p *SAiProvider) CustomizeCreate( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) error { + if err := rejectProviderConfigAPIKeyInJSON(data); err != nil { + return err + } + return p.SEnabledStatusStandaloneResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) +} + +func (p *SAiProvider) PostCreate( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) { + p.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + + input := api.AiProviderCreateInput{} + if err := data.Unmarshal(&input); err != nil { + log.Errorf("ai_provider PostCreate unmarshal: %v", err) + return + } + if len(input.ModelKeys) > 0 { + if err := createSelectedProviderModels(ctx, userCred, ownerId, p, input.ModelKeys); err != nil { + log.Errorf("ai_provider %s create selected models: %v", p.Id, err) + } + } else if err := createCatalogModelsForUserProvider(ctx, userCred, ownerId, p); err != nil { + log.Errorf("ai_provider %s create catalog models: %v", p.Id, err) + } + + secret := strings.TrimSpace(input.Secret) + if secret == "" { + secret, _ = data.GetString("secret") + secret = strings.TrimSpace(secret) + } + if secret == "" { + return + } + if err := createInitialAiKey(ctx, userCred, ownerId, p, secret); err != nil { + log.Errorf("ai_provider %s create initial ai_key: %v", p.Id, err) + } +} + +func createInitialAiKey( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + prov *SAiProvider, + secret string, +) error { + if prov == nil || strings.TrimSpace(prov.Id) == "" { + return errors.Error("ai_provider is nil or has no id") + } + secret = strings.TrimSpace(secret) + if secret == "" { + return nil + } + dataDict := jsonutils.NewDict() + dataDict.Set("ai_provider_id", jsonutils.NewString(prov.Id)) + dataDict.Set("secret", jsonutils.NewString(secret)) + dataDict.Set("weight", jsonutils.NewInt(1)) + dataDict.Set("enabled", jsonutils.JSONTrue) + dataDict.Set("generate_name", jsonutils.NewString(fmt.Sprintf("%s-key", prov.Name))) + if _, err := db.DoCreate(AiKeyManager, ctx, userCred, nil, dataDict, ownerId); err != nil { + return errors.Wrap(err, "create ai_key for provider") + } + return nil +} + func (p *SAiProvider) ValidateUpdateData( ctx context.Context, userCred mcclient.TokenCredential, @@ -164,7 +270,11 @@ func (p *SAiProvider) ValidateUpdateData( if input.Config != nil { input.Config = normalizeAiProviderConfig(input.Config) - if err := validateAiProviderConfig(input.Config); err != nil { + pk := strings.TrimSpace(input.ProviderKey) + if pk == "" { + pk = p.ProviderKey + } + if err := validateAiProviderConfig(input.Config, pk); err != nil { return input, err } } diff --git a/pkg/aiproxy/models/ai_providers_test.go b/pkg/aiproxy/models/ai_providers_test.go new file mode 100644 index 0000000000..e50c4a6b66 --- /dev/null +++ b/pkg/aiproxy/models/ai_providers_test.go @@ -0,0 +1,79 @@ +package models + +import ( + "strings" + "testing" + + "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) + +func TestRejectProviderConfigAPIKeyInJSON(t *testing.T) { + obj, err := jsonutils.Parse([]byte(`{"config":{"api_key":"sk-test"}}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + err = rejectProviderConfigAPIKeyInJSON(obj) + if err == nil { + t.Fatal("expected error for config.api_key") + } + if !strings.Contains(err.Error(), "config.api_key is not supported") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestRejectProviderConfigAPIKeyInJSONAllowsValidConfig(t *testing.T) { + obj, err := jsonutils.Parse([]byte(`{"config":{"base_url":"https://api.openai.com"}}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + err = rejectProviderConfigAPIKeyInJSON(obj) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestAiProviderCreateInputDefaultsEnabled(t *testing.T) { + input := api.AiProviderCreateInput{ + ProviderKey: "deepseek", + Secret: "sk-test", + } + if input.Enabled != nil { + t.Fatal("expected enabled unset before defaulting") + } + if input.Disabled != nil { + t.Fatal("expected disabled unset before defaulting") + } + input.SetEnabled() + if input.Enabled == nil || !*input.Enabled { + t.Fatal("expected enabled=true after SetEnabled") + } +} + +func TestResolveUpstreamAPIKeyNilProvider(t *testing.T) { + _, err := resolveUpstreamAPIKey(nil, "gpt-4o") + if err == nil { + t.Fatal("expected error for nil provider") + } +} + +func TestValidateAiProviderConfigCustomRequiresBaseURL(t *testing.T) { + err := validateAiProviderConfig(nil, api.ProviderKeyCustom) + if err == nil { + t.Fatal("expected error when custom has no config") + } + err = validateAiProviderConfig(&api.SAiProviderConfig{}, api.ProviderKeyCustom) + if err == nil { + t.Fatal("expected error when custom base_url empty") + } +} + +func TestValidateAiProviderConfigCustomAnthropic(t *testing.T) { + cfg := &api.SAiProviderConfig{ + BaseURL: "https://llm.example.com/anthropic", + APIMode: api.ProviderAPIModeAnthropic, + } + if err := validateAiProviderConfig(cfg, api.ProviderKeyCustom); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/pkg/aiproxy/models/ai_routing_models.go b/pkg/aiproxy/models/ai_routing_models.go index 9c339c7cb9..a3102fcdfd 100644 --- a/pkg/aiproxy/models/ai_routing_models.go +++ b/pkg/aiproxy/models/ai_routing_models.go @@ -109,6 +109,8 @@ func (manager *SAiRoutingModelManager) FetchCustomizeColumns( for i := range objs { rows[i].StandaloneResourceDetails = baseRows[i] rm := objs[i].(*SAiRoutingModel) + rows[i].Id = rm.Id + rows[i].Name = rm.Name rows[i].AiRoutingId = rm.AiRoutingId rows[i].AiProviderId = rm.AiProviderId rows[i].AiModelId = rm.AiModelId diff --git a/pkg/aiproxy/models/ai_routings.go b/pkg/aiproxy/models/ai_routings.go index 064b012454..1af52e6461 100644 --- a/pkg/aiproxy/models/ai_routings.go +++ b/pkg/aiproxy/models/ai_routings.go @@ -140,6 +140,8 @@ func (manager *SAiRoutingManager) FetchCustomizeColumns( for j := range entries { e := entries[j] rows[i].RoutingModels[j] = api.AiRoutingModelDetails{ + Id: e.Id, + Name: e.Name, AiRoutingId: e.AiRoutingId, AiProviderId: e.AiProviderId, AiModelId: e.AiModelId, @@ -367,3 +369,12 @@ func (routing *SAiRouting) PerformSetModels( } return nil, nil } + +func (routing *SAiRouting) CustomizeDelete( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) error { + return deleteAiRoutingModels(ctx, routing.Id) +} diff --git a/pkg/aiproxy/models/aiproxy_catalog_validate.go b/pkg/aiproxy/models/aiproxy_catalog_validate.go index 34861d204c..1a7cc9a486 100644 --- a/pkg/aiproxy/models/aiproxy_catalog_validate.go +++ b/pkg/aiproxy/models/aiproxy_catalog_validate.go @@ -20,6 +20,7 @@ import ( "regexp" "strings" + "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/aiproxy" @@ -110,10 +111,23 @@ func catalogModelKeySlug(modelKey string) string { return s } -func validateAiProviderConfig(cfg *api.SAiProviderConfig) error { +func validateAiProviderConfig(cfg *api.SAiProviderConfig, providerKey string) error { + pk := strings.ToLower(strings.TrimSpace(providerKey)) + if api.IsCustomProviderKey(pk) { + if cfg == nil || cfg.IsZero() || cfg.ResolvedBaseURL() == "" { + return errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required for provider_key custom") + } + } if cfg == nil || cfg.IsZero() { return nil } + if !api.IsValidProviderAPIMode(cfg.APIMode) { + return errors.Wrap(httperrors.ErrInputParameter, "config.api_mode must be openai or anthropic") + } + mode := cfg.ResolvedAPIMode() + if mode == api.ProviderAPIModeAnthropic && !api.SupportsDualAPIMode(pk) { + return errors.Wrapf(httperrors.ErrInputParameter, "config.api_mode=anthropic is not supported for provider_key %q", providerKey) + } baseURL := cfg.ResolvedBaseURL() if baseURL == "" { return nil @@ -139,12 +153,34 @@ func normalizeAiProviderConfig(cfg *api.SAiProviderConfig) *api.SAiProviderConfi if base := cfg.ResolvedBaseURL(); base != "" { out.BaseURL = base } - if key := cfg.ResolvedAPIKey(); key != "" { - out.APIKey = key + if mode := strings.TrimSpace(cfg.APIMode); mode != "" { + out.APIMode = strings.ToLower(mode) } return out } +func rejectProviderConfigAPIKeyInJSON(data jsonutils.JSONObject) error { + if data == nil { + return nil + } + dict, ok := data.(*jsonutils.JSONDict) + if !ok { + return nil + } + cfgVal, err := dict.Get("config") + if err != nil || cfgVal == nil { + return nil + } + cfgDict, ok := cfgVal.(*jsonutils.JSONDict) + if !ok { + return nil + } + if cfgDict.Contains("api_key") { + return errors.Wrap(httperrors.ErrInputParameter, "config.api_key is not supported, use secret and ai_keys") + } + return nil +} + func ensureAiModelKeyUniquePerProvider(ctx context.Context, providerId, modelKey, excludeId string) error { q := AiModelManager.Query().Equals("ai_provider_id", providerId).Equals("model_key", modelKey) if excludeId != "" { diff --git a/pkg/aiproxy/models/catalog_seed.go b/pkg/aiproxy/models/catalog_seed.go index 63a9213346..d998fa844b 100644 --- a/pkg/aiproxy/models/catalog_seed.go +++ b/pkg/aiproxy/models/catalog_seed.go @@ -19,100 +19,21 @@ import ( "fmt" "strings" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" ) // standardCatalogProviderKeys lists built-in provider_key values seeded at InitDB. -var standardCatalogProviderKeys = []string{ - "anthropic", - "azure", - "bedrock", - "cerebras", - "cohere", - "gemini", - "groq", - "mistral", - "ollama", - "openai", - "parasail", - "perplexity", - "sgl", - "vertex", - "openrouter", - "elevenlabs", - "huggingface", - "nebius", - "xai", - "replicate", - "vllm", - "runway", - "fireworks", - "aliyun", - "baidu", - "xiaomi", -} - -// defaultPublicBaseURL returns a well-known public API base for OpenAI-compatible upstreams. -// Empty string means no default in catalog (user must set base_url in provider config). -func defaultPublicBaseURL(providerKey string) string { - switch strings.ToLower(strings.TrimSpace(providerKey)) { - case "openai": - return "https://api.openai.com" - case "anthropic": - return "https://api.anthropic.com" - case "azure", "bedrock", "sgl", "vertex": - return "" - case "cerebras": - return "https://api.cerebras.ai" - case "cohere": - return "https://api.cohere.ai" - case "gemini": - return "https://generativelanguage.googleapis.com/v1beta" - case "groq": - return "https://api.groq.com/openai" - case "mistral": - return "https://api.mistral.ai" - case "ollama": - return "http://127.0.0.1:11434" - case "vllm": - return "http://127.0.0.1:8000" - case "parasail": - return "https://api.parasail.io" - case "perplexity": - return "https://api.perplexity.ai" - case "openrouter": - return "https://openrouter.ai/api" - case "elevenlabs": - return "https://api.elevenlabs.io" - case "huggingface": - return "https://router.huggingface.co" - case "nebius": - return "https://api.tokenfactory.nebius.com" - case "xai": - return "https://api.x.ai" - case "replicate": - return "https://api.replicate.com" - case "runway": - return "https://api.dev.runwayml.com" - case "fireworks": - return "https://api.fireworks.ai/inference" - case "aliyun": - return "https://dashscope.aliyuncs.com/compatible-mode" - case "baidu": - return "https://qianfan.baidubce.com/v2" - case "xiaomi": - return "https://api.xiaomimimo.com" - default: - return "" - } -} +var standardCatalogProviderKeys = api.StandardCatalogProviderKeys func standardProviderConfig(providerKey string) *api.SAiProviderConfig { - if u := defaultPublicBaseURL(providerKey); u != "" { + if u := api.DefaultPublicBaseURL(providerKey); u != "" { return &api.SAiProviderConfig{BaseURL: u} } return nil @@ -216,6 +137,97 @@ func ensureSeedProvider(ctx context.Context, providerKey string) error { return ensureSeedModelsEntries(ctx, providerId, providerKey, catalogSeedModelsForProvider(providerKey)) } +func providerModelExists(providerId, modelKey string) (bool, error) { + cnt, err := AiModelManager.Query(). + Equals("ai_provider_id", providerId). + Equals("model_key", modelKey). + CountWithError() + if err != nil { + return false, errors.Wrap(err, "count ai_model for provider") + } + return cnt > 0, nil +} + +func createUserProviderModel( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + prov *SAiProvider, + modelKey, description string, +) error { + if prov == nil || strings.TrimSpace(prov.Id) == "" { + return errors.Error("ai_provider is nil or has no id") + } + modelKey = strings.TrimSpace(modelKey) + if modelKey == "" { + return errors.Error("model_key is empty") + } + exists, err := providerModelExists(prov.Id, modelKey) + if err != nil { + return err + } + if exists { + return nil + } + dataDict := jsonutils.NewDict() + dataDict.Set("ai_provider_id", jsonutils.NewString(prov.Id)) + dataDict.Set("model_key", jsonutils.NewString(modelKey)) + dataDict.Set("enabled", jsonutils.JSONTrue) + dataDict.Set("generate_name", jsonutils.NewString(defaultAiModelName(prov.Name, modelKey))) + if desc := strings.TrimSpace(description); desc != "" { + dataDict.Set("description", jsonutils.NewString(desc)) + } + if _, err := db.DoCreate(AiModelManager, ctx, userCred, nil, dataDict, ownerId); err != nil { + return errors.Wrapf(err, "create ai_model %q for provider %s", modelKey, prov.Id) + } + return nil +} + +func createSelectedProviderModels( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + prov *SAiProvider, + modelKeys []string, +) error { + if prov == nil { + return nil + } + for _, modelKey := range modelKeys { + if err := createUserProviderModel(ctx, userCred, ownerId, prov, modelKey, ""); err != nil { + return err + } + } + return nil +} + +// createCatalogModelsForUserProvider inserts built-in catalog model rows for a newly created public SaaS provider. +func createCatalogModelsForUserProvider( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + prov *SAiProvider, +) error { + if prov == nil { + return nil + } + pk := strings.TrimSpace(prov.ProviderKey) + if !api.HasDefaultPublicBaseURL(pk) { + return nil + } + entries := catalogSeedModelsForProvider(pk) + if len(entries) == 0 { + return createUserProviderModel(ctx, userCred, ownerId, prov, placeholderCatalogModelKey, + "Catalog seed placeholder; replace with concrete model_key values or use a provider with a built-in catalog.") + } + for i := range entries { + if err := createUserProviderModel(ctx, userCred, ownerId, prov, entries[i].ModelKey, entries[i].Description); err != nil { + return err + } + } + return nil +} + // SeedStandardCatalog inserts built-in ai_provider / ai_model catalog rows on first init only. // Existing rows are left unchanged so user config survives service restarts. func SeedStandardCatalog(ctx context.Context) error { diff --git a/pkg/aiproxy/models/catalog_seed_models.go b/pkg/aiproxy/models/catalog_seed_models.go index d36f9ce0ba..b1b2a26fdc 100644 --- a/pkg/aiproxy/models/catalog_seed_models.go +++ b/pkg/aiproxy/models/catalog_seed_models.go @@ -14,6 +14,8 @@ package models +import api "yunion.io/x/onecloud/pkg/apis/aiproxy" + // catalogSeedModel is one row to insert into ai_models when seeding a standard provider. // ModelKey is the id sent to the upstream API (no "provider/" prefix). type catalogSeedModel struct { @@ -26,7 +28,7 @@ type catalogSeedModel struct { // Providers without a list return nil and the seeder inserts model_key "default". func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { switch providerKey { - case "anthropic": + case api.ProviderKeyAnthropic: return []catalogSeedModel{ {ModelKey: "claude-opus-4-20250514", Description: "Anthropic Claude Opus 4"}, {ModelKey: "claude-sonnet-4-20250514", Description: "Anthropic Claude Sonnet 4"}, @@ -36,58 +38,69 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "claude-3-opus-20240229", Description: "Anthropic Claude 3 Opus"}, {ModelKey: "claude-3-haiku-20240307", Description: "Anthropic Claude 3 Haiku"}, } - case "azure": - // Azure OpenAI uses deployment names; these match common Azure OpenAI deployment ids. + // disabled: uncommon provider + // case api.ProviderKeyAzure: + // // Azure OpenAI uses deployment names; these match common Azure OpenAI deployment ids. + // return []catalogSeedModel{ + // {ModelKey: "gpt-4o", Description: "Azure OpenAI GPT-4o deployment"}, + // {ModelKey: "gpt-4o-mini", Description: "Azure OpenAI GPT-4o mini deployment"}, + // {ModelKey: "gpt-4", Description: "Azure OpenAI GPT-4 deployment"}, + // {ModelKey: "gpt-35-turbo", Description: "Azure OpenAI GPT-3.5 Turbo deployment"}, + // {ModelKey: "o3-mini", Description: "Azure OpenAI o3-mini deployment"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyBedrock: + // return []catalogSeedModel{ + // {ModelKey: "anthropic.claude-3-5-sonnet-20241022-v2:0", Description: "Bedrock Claude 3.5 Sonnet"}, + // {ModelKey: "anthropic.claude-3-5-haiku-20241022-v1:0", Description: "Bedrock Claude 3.5 Haiku"}, + // {ModelKey: "anthropic.claude-3-opus-20240229-v1:0", Description: "Bedrock Claude 3 Opus"}, + // {ModelKey: "anthropic.claude-3-sonnet-20240229-v1:0", Description: "Bedrock Claude 3 Sonnet"}, + // {ModelKey: "anthropic.claude-3-haiku-20240307-v1:0", Description: "Bedrock Claude 3 Haiku"}, + // {ModelKey: "meta.llama3-70b-instruct-v1:0", Description: "Bedrock Llama 3 70B Instruct"}, + // {ModelKey: "meta.llama3-8b-instruct-v1:0", Description: "Bedrock Llama 3 8B Instruct"}, + // {ModelKey: "mistral.mistral-large-2402-v1:0", Description: "Bedrock Mistral Large"}, + // {ModelKey: "amazon.titan-text-express-v1", Description: "Bedrock Amazon Titan Text Express"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyCerebras: + // return []catalogSeedModel{ + // {ModelKey: "llama3.1-8b", Description: "Cerebras Llama 3.1 8B"}, + // {ModelKey: "llama3.1-70b", Description: "Cerebras Llama 3.1 70B"}, + // {ModelKey: "llama-3.3-70b", Description: "Cerebras Llama 3.3 70B"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyCohere: + // return []catalogSeedModel{ + // {ModelKey: "command-r-plus", Description: "Cohere Command R+"}, + // {ModelKey: "command-r", Description: "Cohere Command R"}, + // {ModelKey: "command-a", Description: "Cohere Command A"}, + // {ModelKey: "command", Description: "Cohere Command"}, + // {ModelKey: "command-light", Description: "Cohere Command Light"}, + // {ModelKey: "embed-english-v3.0", Description: "Cohere Embed English v3"}, + // {ModelKey: "embed-multilingual-v3.0", Description: "Cohere Embed Multilingual v3"}, + // } + case api.ProviderKeyDeepseek: return []catalogSeedModel{ - {ModelKey: "gpt-4o", Description: "Azure OpenAI GPT-4o deployment"}, - {ModelKey: "gpt-4o-mini", Description: "Azure OpenAI GPT-4o mini deployment"}, - {ModelKey: "gpt-4", Description: "Azure OpenAI GPT-4 deployment"}, - {ModelKey: "gpt-35-turbo", Description: "Azure OpenAI GPT-3.5 Turbo deployment"}, - {ModelKey: "o3-mini", Description: "Azure OpenAI o3-mini deployment"}, + {ModelKey: "deepseek-v4-flash", Description: "DeepSeek-V4-Flash; 1M context; high concurrency (2500); cost-efficient default"}, + {ModelKey: "deepseek-v4-pro", Description: "DeepSeek-V4-Pro; 1M context; frontier reasoning/coding/agents (500 concurrency)"}, } - case "bedrock": - return []catalogSeedModel{ - {ModelKey: "anthropic.claude-3-5-sonnet-20241022-v2:0", Description: "Bedrock Claude 3.5 Sonnet"}, - {ModelKey: "anthropic.claude-3-5-haiku-20241022-v1:0", Description: "Bedrock Claude 3.5 Haiku"}, - {ModelKey: "anthropic.claude-3-opus-20240229-v1:0", Description: "Bedrock Claude 3 Opus"}, - {ModelKey: "anthropic.claude-3-sonnet-20240229-v1:0", Description: "Bedrock Claude 3 Sonnet"}, - {ModelKey: "anthropic.claude-3-haiku-20240307-v1:0", Description: "Bedrock Claude 3 Haiku"}, - {ModelKey: "meta.llama3-70b-instruct-v1:0", Description: "Bedrock Llama 3 70B Instruct"}, - {ModelKey: "meta.llama3-8b-instruct-v1:0", Description: "Bedrock Llama 3 8B Instruct"}, - {ModelKey: "mistral.mistral-large-2402-v1:0", Description: "Bedrock Mistral Large"}, - {ModelKey: "amazon.titan-text-express-v1", Description: "Bedrock Amazon Titan Text Express"}, - } - case "cerebras": - return []catalogSeedModel{ - {ModelKey: "llama3.1-8b", Description: "Cerebras Llama 3.1 8B"}, - {ModelKey: "llama3.1-70b", Description: "Cerebras Llama 3.1 70B"}, - {ModelKey: "llama-3.3-70b", Description: "Cerebras Llama 3.3 70B"}, - } - case "cohere": - return []catalogSeedModel{ - {ModelKey: "command-r-plus", Description: "Cohere Command R+"}, - {ModelKey: "command-r", Description: "Cohere Command R"}, - {ModelKey: "command-a", Description: "Cohere Command A"}, - {ModelKey: "command", Description: "Cohere Command"}, - {ModelKey: "command-light", Description: "Cohere Command Light"}, - {ModelKey: "embed-english-v3.0", Description: "Cohere Embed English v3"}, - {ModelKey: "embed-multilingual-v3.0", Description: "Cohere Embed Multilingual v3"}, - } - case "elevenlabs": - return []catalogSeedModel{ - {ModelKey: "eleven_multilingual_v2", Description: "ElevenLabs multilingual v2"}, - {ModelKey: "eleven_turbo_v2_5", Description: "ElevenLabs Turbo v2.5"}, - {ModelKey: "eleven_flash_v2_5", Description: "ElevenLabs Flash v2.5"}, - {ModelKey: "eleven_multilingual_v1", Description: "ElevenLabs multilingual v1"}, - } - case "fireworks": - return []catalogSeedModel{ - {ModelKey: "accounts/fireworks/models/llama-v3p1-8b-instruct", Description: "Fireworks Llama 3.1 8B Instruct"}, - {ModelKey: "accounts/fireworks/models/llama-v3p1-70b-instruct", Description: "Fireworks Llama 3.1 70B Instruct"}, - {ModelKey: "accounts/fireworks/models/llama-v3p3-70b-instruct", Description: "Fireworks Llama 3.3 70B Instruct"}, - {ModelKey: "accounts/fireworks/models/mixtral-8x7b-instruct", Description: "Fireworks Mixtral 8x7B Instruct"}, - } - case "gemini": + // disabled: uncommon provider + // case api.ProviderKeyElevenlabs: + // return []catalogSeedModel{ + // {ModelKey: "eleven_multilingual_v2", Description: "ElevenLabs multilingual v2"}, + // {ModelKey: "eleven_turbo_v2_5", Description: "ElevenLabs Turbo v2.5"}, + // {ModelKey: "eleven_flash_v2_5", Description: "ElevenLabs Flash v2.5"}, + // {ModelKey: "eleven_multilingual_v1", Description: "ElevenLabs multilingual v1"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyFireworks: + // return []catalogSeedModel{ + // {ModelKey: "accounts/fireworks/models/llama-v3p1-8b-instruct", Description: "Fireworks Llama 3.1 8B Instruct"}, + // {ModelKey: "accounts/fireworks/models/llama-v3p1-70b-instruct", Description: "Fireworks Llama 3.1 70B Instruct"}, + // {ModelKey: "accounts/fireworks/models/llama-v3p3-70b-instruct", Description: "Fireworks Llama 3.3 70B Instruct"}, + // {ModelKey: "accounts/fireworks/models/mixtral-8x7b-instruct", Description: "Fireworks Mixtral 8x7B Instruct"}, + // } + case api.ProviderKeyGemini: return []catalogSeedModel{ {ModelKey: "gemini-2.0-flash", Description: "Google Gemini 2.0 Flash"}, {ModelKey: "gemini-2.0-flash-lite", Description: "Google Gemini 2.0 Flash-Lite"}, @@ -96,7 +109,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "gemini-1.5-flash-8b", Description: "Google Gemini 1.5 Flash 8B"}, {ModelKey: "gemini-embedding-001", Description: "Google Gemini Embedding 001"}, } - case "groq": + case api.ProviderKeyGroq: return []catalogSeedModel{ {ModelKey: "llama-3.3-70b-versatile", Description: "Groq Llama 3.3 70B Versatile"}, {ModelKey: "llama-3.1-8b-instant", Description: "Groq Llama 3.1 8B Instant"}, @@ -104,14 +117,14 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "mixtral-8x7b-32768", Description: "Groq Mixtral 8x7B"}, {ModelKey: "gemma2-9b-it", Description: "Groq Gemma2 9B IT"}, } - case "huggingface": + case api.ProviderKeyHuggingface: return []catalogSeedModel{ {ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "HF Llama 3.1 8B Instruct"}, {ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "HF Llama 3.1 70B Instruct"}, {ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "HF Mistral 7B Instruct"}, {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "HF Qwen2.5 72B Instruct"}, } - case "mistral": + case api.ProviderKeyMistral: return []catalogSeedModel{ {ModelKey: "mistral-large-latest", Description: "Mistral Large (latest)"}, {ModelKey: "mistral-small-latest", Description: "Mistral Small (latest)"}, @@ -122,13 +135,14 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "mixtral-8x22b", Description: "Mistral Mixtral 8x22B"}, {ModelKey: "mixtral-8x7b", Description: "Mistral Mixtral 8x7B"}, } - case "nebius": - return []catalogSeedModel{ - {ModelKey: "deepseek-ai/DeepSeek-V3", Description: "Nebius DeepSeek V3"}, - {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "Nebius Qwen2.5 72B Instruct"}, - {ModelKey: "meta-llama/Llama-3.3-70B-Instruct", Description: "Nebius Llama 3.3 70B Instruct"}, - } - case "ollama": + // disabled: uncommon provider + // case api.ProviderKeyNebius: + // return []catalogSeedModel{ + // {ModelKey: "deepseek-ai/DeepSeek-V3", Description: "Nebius DeepSeek V3"}, + // {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "Nebius Qwen2.5 72B Instruct"}, + // {ModelKey: "meta-llama/Llama-3.3-70B-Instruct", Description: "Nebius Llama 3.3 70B Instruct"}, + // } + case api.ProviderKeyOllama: return []catalogSeedModel{ {ModelKey: "llama3.2", Description: "Ollama Llama 3.2"}, {ModelKey: "llama3.1", Description: "Ollama Llama 3.1"}, @@ -137,7 +151,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "codellama", Description: "Ollama Code Llama"}, {ModelKey: "phi3", Description: "Ollama Phi 3"}, } - case "vllm": + case api.ProviderKeyVLLM: return []catalogSeedModel{ {ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "vLLM Llama 3.1 8B Instruct"}, {ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "vLLM Llama 3.1 70B Instruct"}, @@ -145,7 +159,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "vLLM Qwen2.5 72B Instruct"}, {ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "vLLM Mistral 7B Instruct"}, } - case "openai": + case api.ProviderKeyOpenAI: return []catalogSeedModel{ {ModelKey: "gpt-5-nano", Description: "OpenAI GPT-5 nano"}, {ModelKey: "gpt-5-mini", Description: "OpenAI GPT-5 mini"}, @@ -176,7 +190,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "text-embedding-3-large", Description: "OpenAI text-embedding-3-large"}, {ModelKey: "text-embedding-ada-002", Description: "OpenAI text-embedding-ada-002"}, } - case "openrouter": + case api.ProviderKeyOpenrouter: return []catalogSeedModel{ {ModelKey: "openai/gpt-4o", Description: "OpenRouter OpenAI GPT-4o"}, {ModelKey: "openai/gpt-4o-mini", Description: "OpenRouter OpenAI GPT-4o mini"}, @@ -186,46 +200,53 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel { {ModelKey: "meta-llama/llama-3.3-70b-instruct", Description: "OpenRouter Llama 3.3 70B Instruct"}, {ModelKey: "mistralai/mistral-large", Description: "OpenRouter Mistral Large"}, } - case "perplexity": - return []catalogSeedModel{ - {ModelKey: "sonar", Description: "Perplexity Sonar"}, - {ModelKey: "sonar-pro", Description: "Perplexity Sonar Pro"}, - {ModelKey: "sonar-reasoning", Description: "Perplexity Sonar Reasoning"}, - {ModelKey: "llama-3.1-sonar-small-128k-online", Description: "Perplexity Llama 3.1 Sonar Small online"}, - {ModelKey: "llama-3.1-sonar-large-128k-online", Description: "Perplexity Llama 3.1 Sonar Large online"}, - } - case "replicate": - return []catalogSeedModel{ - {ModelKey: "meta/meta-llama-3-8b-instruct", Description: "Replicate Meta Llama 3 8B Instruct"}, - {ModelKey: "meta/meta-llama-3-70b-instruct", Description: "Replicate Meta Llama 3 70B Instruct"}, - {ModelKey: "mistralai/mixtral-8x7b-instruct-v0.1", Description: "Replicate Mixtral 8x7B Instruct"}, - } - case "runway": - return []catalogSeedModel{ - {ModelKey: "gen3a_turbo", Description: "Runway Gen-3 Alpha Turbo"}, - {ModelKey: "gen3a", Description: "Runway Gen-3 Alpha"}, - {ModelKey: "gen4_aleph", Description: "Runway Gen-4 Aleph"}, - } - case "vertex": - return []catalogSeedModel{ - {ModelKey: "gemini-2.0-flash", Description: "Vertex AI Gemini 2.0 Flash"}, - {ModelKey: "gemini-1.5-pro", Description: "Vertex AI Gemini 1.5 Pro"}, - {ModelKey: "gemini-1.5-flash", Description: "Vertex AI Gemini 1.5 Flash"}, - {ModelKey: "publishers/google/models/gemini-1.5-pro", Description: "Vertex publisher path Gemini 1.5 Pro"}, - } - case "xai": - return []catalogSeedModel{ - {ModelKey: "grok-3", Description: "xAI Grok 3"}, - {ModelKey: "grok-3-mini", Description: "xAI Grok 3 mini"}, - {ModelKey: "grok-2-latest", Description: "xAI Grok 2 latest"}, - {ModelKey: "grok-2-1212", Description: "xAI Grok 2 1212"}, - {ModelKey: "grok-beta", Description: "xAI Grok beta"}, - } - case "aliyun": - return aliyunQwenSeedModels() - case "baidu": - return baiduErnieSeedModels() - case "xiaomi": + // disabled: uncommon provider + // case api.ProviderKeyPerplexity: + // return []catalogSeedModel{ + // {ModelKey: "sonar", Description: "Perplexity Sonar"}, + // {ModelKey: "sonar-pro", Description: "Perplexity Sonar Pro"}, + // {ModelKey: "sonar-reasoning", Description: "Perplexity Sonar Reasoning"}, + // {ModelKey: "llama-3.1-sonar-small-128k-online", Description: "Perplexity Llama 3.1 Sonar Small online"}, + // {ModelKey: "llama-3.1-sonar-large-128k-online", Description: "Perplexity Llama 3.1 Sonar Large online"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyReplicate: + // return []catalogSeedModel{ + // {ModelKey: "meta/meta-llama-3-8b-instruct", Description: "Replicate Meta Llama 3 8B Instruct"}, + // {ModelKey: "meta/meta-llama-3-70b-instruct", Description: "Replicate Meta Llama 3 70B Instruct"}, + // {ModelKey: "mistralai/mixtral-8x7b-instruct-v0.1", Description: "Replicate Mixtral 8x7B Instruct"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyRunway: + // return []catalogSeedModel{ + // {ModelKey: "gen3a_turbo", Description: "Runway Gen-3 Alpha Turbo"}, + // {ModelKey: "gen3a", Description: "Runway Gen-3 Alpha"}, + // {ModelKey: "gen4_aleph", Description: "Runway Gen-4 Aleph"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyVertex: + // return []catalogSeedModel{ + // {ModelKey: "gemini-2.0-flash", Description: "Vertex AI Gemini 2.0 Flash"}, + // {ModelKey: "gemini-1.5-pro", Description: "Vertex AI Gemini 1.5 Pro"}, + // {ModelKey: "gemini-1.5-flash", Description: "Vertex AI Gemini 1.5 Flash"}, + // {ModelKey: "publishers/google/models/gemini-1.5-pro", Description: "Vertex publisher path Gemini 1.5 Pro"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyXai: + // return []catalogSeedModel{ + // {ModelKey: "grok-3", Description: "xAI Grok 3"}, + // {ModelKey: "grok-3-mini", Description: "xAI Grok 3 mini"}, + // {ModelKey: "grok-2-latest", Description: "xAI Grok 2 latest"}, + // {ModelKey: "grok-2-1212", Description: "xAI Grok 2 1212"}, + // {ModelKey: "grok-beta", Description: "xAI Grok beta"}, + // } + // disabled: uncommon provider + // case api.ProviderKeyAliyun: + // return aliyunQwenSeedModels() + // disabled: uncommon provider + // case api.ProviderKeyBaidu: + // return baiduErnieSeedModels() + case api.ProviderKeyXiaomi: return xiaomiMimoSeedModels() default: return nil diff --git a/pkg/aiproxy/models/catalog_seed_test.go b/pkg/aiproxy/models/catalog_seed_test.go new file mode 100644 index 0000000000..faf173f6a6 --- /dev/null +++ b/pkg/aiproxy/models/catalog_seed_test.go @@ -0,0 +1,34 @@ +package models + +import ( + "testing" + + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) + +func TestCatalogSeedModelsForPublicProviders(t *testing.T) { + publicKeys := []string{ + api.ProviderKeyOpenAI, + api.ProviderKeyDeepseek, + api.ProviderKeyAnthropic, + api.ProviderKeyGemini, + } + for _, pk := range publicKeys { + if !api.HasDefaultPublicBaseURL(pk) { + t.Fatalf("%q should have default public base URL", pk) + } + entries := catalogSeedModelsForProvider(pk) + if len(entries) == 0 { + t.Fatalf("expected catalog models for public provider %q", pk) + } + } +} + +func TestCatalogSeedModelsSkippedForSelfHostedProviders(t *testing.T) { + selfHosted := []string{api.ProviderKeyOllama, api.ProviderKeyVLLM, api.ProviderKeySGLang} + for _, pk := range selfHosted { + if api.HasDefaultPublicBaseURL(pk) { + t.Fatalf("%q should not be treated as public SaaS provider", pk) + } + } +} diff --git a/pkg/aiproxy/models/chat_upstream.go b/pkg/aiproxy/models/chat_upstream.go index db2c947a7b..1ea631ec13 100644 --- a/pkg/aiproxy/models/chat_upstream.go +++ b/pkg/aiproxy/models/chat_upstream.go @@ -36,6 +36,7 @@ type ChatUpstream struct { ProviderKey string AiProviderId string AiKeyId string + APIMode string // VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row. VirtualKeyId string @@ -212,7 +213,7 @@ func resolveCatalogModelFromRouting( // 1. ai_virtual_key (auth + project scope) // 2. ai_routing in that project (model_key exact match first, then model_pattern / optional proxy-node scope, priority) // 3. ai_routing_model -> ai_provider + ai_model -// 4. ai_key rows for that provider matching the catalog model_key (weight), else provider.config api_key +// 4. ai_key rows for that provider matching the catalog model_key (weight) func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, virtualKey string, body *jsonutils.JSONDict) (*ChatUpstream, error) { vk, err := loadEnabledVirtualKey(virtualKey) if err != nil { @@ -246,7 +247,7 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, if prov.Config == nil { return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty") } - baseURL := prov.Config.ResolvedBaseURL() + baseURL := prov.Config.EffectiveBaseURL(prov.ProviderKey) if baseURL == "" { return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config must include base_url") } @@ -256,13 +257,15 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model.model_key is empty") } + apiMode := prov.Config.ResolvedAPIMode() + // Keys are scoped to ai_provider; routing on each ai_key matches the resolved catalog model_key. keyRes, err := resolveUpstreamAPIKey(prov, upstreamModel) if err != nil { return nil, err } if keyRes == nil || keyRes.Secret == "" { - return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no api_key for ai_provider and catalog model") + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "add an enabled ai_key with secret for this provider") } up := &ChatUpstream{ @@ -273,6 +276,7 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, AiProviderId: prov.Id, AiKeyId: keyRes.AiKeyId, VirtualKeyId: vk.Id, + APIMode: apiMode, } if vk.Limits != nil { up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest diff --git a/pkg/aiproxy/models/provider_connectivity.go b/pkg/aiproxy/models/provider_connectivity.go new file mode 100644 index 0000000000..a1c703f677 --- /dev/null +++ b/pkg/aiproxy/models/provider_connectivity.go @@ -0,0 +1,219 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "net/http" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/aiproxy/ft" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/upstream" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +const ( + providerCreateConnectivityTimeout = 15 * time.Second + providerTestConnectivityTimeout = 60 * time.Second +) + +func normalizeProviderModelKeys(modelKeys []string) ([]string, error) { + seen := make(map[string]struct{}, len(modelKeys)) + out := make([]string, 0, len(modelKeys)) + for _, key := range modelKeys { + mk, err := validateAiModelKey(key) + if err != nil { + return nil, err + } + if _, ok := seen[mk]; ok { + continue + } + seen[mk] = struct{}{} + out = append(out, mk) + } + return out, nil +} + +func catalogModelKeysForConnectivity(providerKey string) []string { + entries := catalogSeedModelsForProvider(providerKey) + if len(entries) == 0 { + return []string{placeholderCatalogModelKey} + } + keys := make([]string, len(entries)) + for i := range entries { + keys[i] = entries[i].ModelKey + } + return keys +} + +func probeModelForConnectivity(providerKey string) string { + entries := catalogSeedModelsForProvider(providerKey) + if len(entries) > 0 && strings.TrimSpace(entries[0].ModelKey) != "" { + return entries[0].ModelKey + } + if model := strings.TrimSpace(ft.DefaultModelForProvider(providerKey)); model != "" { + return model + } + return placeholderCatalogModelKey +} + +func shouldFallbackToChatFromListModels(uerr *upstream.Error) bool { + if uerr == nil { + return true + } + switch uerr.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return false + default: + return true + } +} + +func probeChatConnectivity(ctx context.Context, providerKey, secret string, cfg *api.SAiProviderConfig) error { + if cfg == nil { + cfg = &api.SAiProviderConfig{} + } + effectiveURL := cfg.EffectiveBaseURL(providerKey) + if effectiveURL == "" { + return errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required (no default for this provider_key)") + } + apiMode := cfg.ResolvedAPIMode() + probeModel := probeModelForConnectivity(providerKey) + + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("ping")) + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString(probeModel)) + body.Set("max_tokens", jsonutils.NewInt(1)) + body.Set("messages", jsonutils.NewArray(userMsg)) + + prov := providers.ChatProviderForUpstream(providerKey, apiMode) + httpReq, err := prov.BuildUpstreamRequest(providers.ChatContextFromUpstream( + providerKey, effectiveURL, secret, probeModel, apiMode, + ), body, false) + if err != nil { + return httperrors.NewInputParameterError("failed to build chat probe request: %s", err.Error()) + } + _, uerr := upstream.ChatCompletion(ctx, providers.ToUpstreamRequest(httpReq, secret)) + if uerr != nil { + return connectivityErrorFromUpstream(uerr) + } + return nil +} + +func listProviderModels(ctx context.Context, providerKey, secret string, cfg *api.SAiProviderConfig, timeout time.Duration) ([]string, bool, error) { + pk, err := validateAiCatalogIdentifier("provider_key", providerKey, maxAiProviderKeyLen) + if err != nil { + return nil, false, err + } + secret = strings.TrimSpace(secret) + if secret == "" { + return nil, false, errors.Wrap(httperrors.ErrInputParameter, "secret is required for connectivity test") + } + cfg = normalizeAiProviderConfig(cfg) + if err := validateAiProviderConfig(cfg, pk); err != nil { + return nil, false, err + } + if cfg == nil { + cfg = &api.SAiProviderConfig{} + } + effectiveURL := cfg.EffectiveBaseURL(pk) + if effectiveURL == "" { + return nil, false, errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required (no default for this provider_key)") + } + if timeout <= 0 { + timeout = providerCreateConnectivityTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + resp, uerr := upstream.ListModels(ctx, effectiveURL, secret) + if uerr == nil { + modelKeys, err := upstream.ParseModelsListBody(resp.Body) + if err == nil && len(modelKeys) > 0 { + return modelKeys, false, nil + } + } else if !shouldFallbackToChatFromListModels(uerr) { + return nil, false, connectivityErrorFromUpstream(uerr) + } + + if err := probeChatConnectivity(ctx, pk, secret, cfg); err != nil { + return nil, false, err + } + return catalogModelKeysForConnectivity(pk), true, nil +} + +func probeProviderConnectivity(ctx context.Context, providerKey, secret string, cfg *api.SAiProviderConfig) error { + _, _, err := listProviderModels(ctx, providerKey, secret, cfg, providerCreateConnectivityTimeout) + return err +} + +func connectivityErrorFromUpstream(uerr *upstream.Error) error { + if uerr == nil { + return nil + } + msg := strings.TrimSpace(uerr.Message) + if msg == "" { + msg = uerr.Error() + } + switch uerr.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return httperrors.NewInputParameterError("invalid API key: %s", msg) + case http.StatusNotFound: + return httperrors.NewInputParameterError("API URL not found (check config.base_url): %s", msg) + default: + return httperrors.NewInputParameterError("upstream connectivity test failed: %s", msg) + } +} + +func providerUpstreamModels(modelKeys []string) []api.AiProviderUpstreamModel { + out := make([]api.AiProviderUpstreamModel, len(modelKeys)) + for i, mk := range modelKeys { + out[i] = api.AiProviderUpstreamModel{ModelKey: mk} + } + return out +} + +// PerformTestConnectivity probes upstream list-models without persisting an ai_provider row. +func (manager *SAiProviderManager) PerformTestConnectivity( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input api.AiProviderTestConnectivityInput, +) (api.AiProviderTestConnectivityOutput, error) { + out := api.AiProviderTestConnectivityOutput{} + modelKeys, fromCatalog, err := listProviderModels(ctx, input.ProviderKey, input.Secret, input.Config, providerTestConnectivityTimeout) + if err != nil { + return out, err + } + out.Ok = true + if fromCatalog { + out.Message = "connectivity test passed (catalog models)" + out.ModelsSource = api.AiProviderModelsSourceCatalog + } else { + out.Message = "connectivity test passed" + out.ModelsSource = api.AiProviderModelsSourceUpstream + } + out.Models = providerUpstreamModels(modelKeys) + return out, nil +} diff --git a/pkg/aiproxy/providerapi/types.go b/pkg/aiproxy/providerapi/types.go index 037c1e9cea..3018081e2c 100644 --- a/pkg/aiproxy/providerapi/types.go +++ b/pkg/aiproxy/providerapi/types.go @@ -25,6 +25,7 @@ type ChatContext struct { BaseURL string APIKey string UpstreamModel string + APIMode string } // HTTPRequest is the wire-format call sent to an upstream provider. @@ -82,3 +83,24 @@ type CompletionsProvider interface { NormalizeCompletionsResponse(body []byte) ([]byte, error) OpenAICompletionsStreamPassthrough() bool } + +// AnthropicStreamChunk is one Anthropic Messages API SSE event. +type AnthropicStreamChunk struct { + Event string + Data []byte +} + +// AnthropicStreamState carries per-stream metadata for Anthropic Messages streaming. +type AnthropicStreamState struct { + RequestModel string +} + +// MessagesAdapter converts Anthropic Messages API requests to upstream HTTP calls and +// normalizes responses back to Anthropic format. +type MessagesAdapter interface { + BuildUpstreamRequest(ctx *ChatContext, body *jsonutils.JSONDict, stream bool) (*HTTPRequest, error) + NormalizeResponse(prov Provider, body []byte) ([]byte, error) + AnthropicStreamPassthrough() bool + NewStreamState(requestModel string) interface{} + ConvertStreamPayload(state interface{}, payload []byte, endOfStream bool) ([]AnthropicStreamChunk, error) +} diff --git a/pkg/aiproxy/providers/aliyun/aliyun.go b/pkg/aiproxy/providers/aliyun/aliyun.go index 01b845f583..ea0a7939d7 100644 --- a/pkg/aiproxy/providers/aliyun/aliyun.go +++ b/pkg/aiproxy/providers/aliyun/aliyun.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) func patchEnableThinkingFalse(body *jsonutils.JSONDict, stream bool) { @@ -33,5 +34,5 @@ func patchEnableThinkingFalse(body *jsonutils.JSONDict, stream bool) { // New returns the Aliyun (DashScope compatible-mode) provider adapter. func New() providerapi.Provider { - return openai.NewCompat("aliyun", patchEnableThinkingFalse) + return openai.NewCompat(api.ProviderKeyAliyun, patchEnableThinkingFalse) } diff --git a/pkg/aiproxy/providers/anthropic/anthropic.go b/pkg/aiproxy/providers/anthropic/anthropic.go index 78656fd406..8c99659f30 100644 --- a/pkg/aiproxy/providers/anthropic/anthropic.go +++ b/pkg/aiproxy/providers/anthropic/anthropic.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) const apiVersion = "2023-06-01" @@ -36,7 +37,7 @@ func New() providerapi.Provider { } func (p *provider) Key() string { - return "anthropic" + return api.ProviderKeyAnthropic } func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { diff --git a/pkg/aiproxy/providers/anthropic/openai_bridge.go b/pkg/aiproxy/providers/anthropic/openai_bridge.go new file mode 100644 index 0000000000..ddba8da7c9 --- /dev/null +++ b/pkg/aiproxy/providers/anthropic/openai_bridge.go @@ -0,0 +1,22 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package anthropic + +import "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + +// OpenAINativeBridge returns a Provider that converts OpenAI chat/completions to Anthropic Messages upstream. +func OpenAINativeBridge() providerapi.Provider { + return New() +} diff --git a/pkg/aiproxy/providers/anthropic/openai_bridge_test.go b/pkg/aiproxy/providers/anthropic/openai_bridge_test.go new file mode 100644 index 0000000000..cab6e93ce3 --- /dev/null +++ b/pkg/aiproxy/providers/anthropic/openai_bridge_test.go @@ -0,0 +1,34 @@ +package anthropic + +import ( + "testing" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +func TestOpenAINativeBridgeBuildDeepseekAnthropicURL(t *testing.T) { + p := OpenAINativeBridge() + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("deepseek-chat")) + user := jsonutils.NewDict() + user.Set("role", jsonutils.NewString("user")) + user.Set("content", jsonutils.NewString("hello")) + body.Set("messages", jsonutils.NewArray(user)) + + req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{ + BaseURL: "https://api.deepseek.com/anthropic", + APIKey: "ds-key", + UpstreamModel: "deepseek-chat", + }, body, false) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.deepseek.com/anthropic/v1/messages" { + t.Fatalf("url: %s", req.URL) + } + if req.Headers["x-api-key"] != "ds-key" { + t.Fatal("missing x-api-key header") + } +} diff --git a/pkg/aiproxy/providers/azure/azure.go b/pkg/aiproxy/providers/azure/azure.go index 6f7ac0a21d..6732607356 100644 --- a/pkg/aiproxy/providers/azure/azure.go +++ b/pkg/aiproxy/providers/azure/azure.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) type provider struct{} @@ -33,7 +34,7 @@ func New() providerapi.Provider { } func (p *provider) Key() string { - return "azure" + return api.ProviderKeyAzure } func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { diff --git a/pkg/aiproxy/providers/baidu/baidu.go b/pkg/aiproxy/providers/baidu/baidu.go index 3c97f71447..65970e3ddc 100644 --- a/pkg/aiproxy/providers/baidu/baidu.go +++ b/pkg/aiproxy/providers/baidu/baidu.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) type provider struct { @@ -29,11 +30,11 @@ type provider struct { // New returns the Baidu Wenxin / Qianfan provider adapter. func New() providerapi.Provider { - return &provider{v2: openai.NewCompat("baidu")} + return &provider{v2: openai.NewCompat(api.ProviderKeyBaidu)} } func (p *provider) Key() string { - return "baidu" + return api.ProviderKeyBaidu } func (p *provider) useV2(ctx *providerapi.ChatContext) bool { diff --git a/pkg/aiproxy/providers/chat_provider.go b/pkg/aiproxy/providers/chat_provider.go new file mode 100644 index 0000000000..e1fcb0bbb6 --- /dev/null +++ b/pkg/aiproxy/providers/chat_provider.go @@ -0,0 +1,39 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providers + +import ( + "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" + apapi "yunion.io/x/onecloud/pkg/apis/aiproxy" +) + +// ChatProviderForUpstream returns the provider adapter for chat/completions upstream calls. +func ChatProviderForUpstream(providerKey, apiMode string) Provider { + if apiMode == apapi.ProviderAPIModeAnthropic && apapi.SupportsDualAPIMode(providerKey) { + return anthropic.OpenAINativeBridge() + } + return Get(providerKey) +} + +// ChatContextFromUpstream builds a provider ChatContext from resolved upstream fields. +func ChatContextFromUpstream(providerKey, baseURL, apiKey, upstreamModel, apiMode string) *ChatContext { + return &ChatContext{ + ProviderKey: providerKey, + BaseURL: baseURL, + APIKey: apiKey, + UpstreamModel: upstreamModel, + APIMode: apiMode, + } +} diff --git a/pkg/aiproxy/providers/cohere/cohere.go b/pkg/aiproxy/providers/cohere/cohere.go index e62518ea29..16f13b034a 100644 --- a/pkg/aiproxy/providers/cohere/cohere.go +++ b/pkg/aiproxy/providers/cohere/cohere.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) type provider struct { @@ -32,7 +33,7 @@ type provider struct { // New returns the Cohere provider adapter (OpenAI-compatible chat, native embeddings). func New() providerapi.Provider { - return &provider{Compat: openai.NewCompat("cohere")} + return &provider{Compat: openai.NewCompat(api.ProviderKeyCohere)} } func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) { diff --git a/pkg/aiproxy/providers/embeddings_test.go b/pkg/aiproxy/providers/embeddings_test.go deleted file mode 100644 index adacb9ba6d..0000000000 --- a/pkg/aiproxy/providers/embeddings_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package providers - -import ( - "encoding/json" - "testing" - - "yunion.io/x/jsonutils" -) - -func TestOpenAIEmbeddingsCompatBuild(t *testing.T) { - body := jsonutils.NewDict() - body.Add(jsonutils.NewString("text-embedding-3-small"), "model") - body.Add(jsonutils.NewString("hello"), "input") - - p := GetEmbeddings("openai") - req, err := p.BuildEmbeddingsRequest(&ChatContext{ - BaseURL: "https://api.openai.com", - APIKey: "sk-test", - UpstreamModel: "text-embedding-3-small", - }, body) - if err != nil { - t.Fatal(err) - } - if req.URL != "https://api.openai.com/v1/embeddings" { - t.Fatalf("unexpected url: %s", req.URL) - } -} - -func TestGeminiEmbeddingsBuildSingle(t *testing.T) { - body := jsonutils.NewDict() - body.Add(jsonutils.NewString("text-embedding-004"), "model") - body.Add(jsonutils.NewString("hello world"), "input") - - p := GetEmbeddings("gemini") - req, err := p.BuildEmbeddingsRequest(&ChatContext{ - BaseURL: "https://generativelanguage.googleapis.com/v1beta", - APIKey: "key", - UpstreamModel: "text-embedding-004", - }, body) - if err != nil { - t.Fatal(err) - } - if req.URL != "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent" { - t.Fatalf("unexpected url: %s", req.URL) - } -} - -func TestGeminiEmbeddingsNormalize(t *testing.T) { - p := GetEmbeddings("gemini") - out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embedding":{"values":[0.1,0.2]}}`)) - if err != nil { - t.Fatal(err) - } - var resp map[string]interface{} - if err := json.Unmarshal(out, &resp); err != nil { - t.Fatal(err) - } - if resp["object"] != "list" { - t.Fatalf("unexpected object: %#v", resp["object"]) - } -} - -func TestCohereEmbeddingsBuild(t *testing.T) { - body := jsonutils.NewDict() - body.Add(jsonutils.NewString("embed-english-v3.0"), "model") - body.Add(jsonutils.NewArray(jsonutils.NewString("a"), jsonutils.NewString("b")), "input") - - p := GetEmbeddings("cohere") - req, err := p.BuildEmbeddingsRequest(&ChatContext{ - BaseURL: "https://api.cohere.ai", - APIKey: "key", - UpstreamModel: "embed-english-v3.0", - }, body) - if err != nil { - t.Fatal(err) - } - if req.URL != "https://api.cohere.ai/v2/embed" { - t.Fatalf("unexpected url: %s", req.URL) - } -} - -func TestCohereEmbeddingsNormalize(t *testing.T) { - p := GetEmbeddings("cohere") - out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embeddings":{"float":[[0.1],[0.2]]}}`)) - if err != nil { - t.Fatal(err) - } - var resp struct { - Data []struct { - Index int `json:"index"` - } `json:"data"` - } - if err := json.Unmarshal(out, &resp); err != nil { - t.Fatal(err) - } - if len(resp.Data) != 2 { - t.Fatalf("expected 2 embeddings, got %d", len(resp.Data)) - } -} - -func TestAnthropicEmbeddingsUnsupported(t *testing.T) { - p := GetEmbeddings("anthropic") - _, err := p.BuildEmbeddingsRequest(&ChatContext{ProviderKey: "anthropic"}, jsonutils.NewDict()) - if err == nil { - t.Fatal("expected error for anthropic embeddings") - } -} diff --git a/pkg/aiproxy/providers/gemini/gemini.go b/pkg/aiproxy/providers/gemini/gemini.go index 06acdc6041..eed68ebe83 100644 --- a/pkg/aiproxy/providers/gemini/gemini.go +++ b/pkg/aiproxy/providers/gemini/gemini.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) type provider struct{} @@ -36,7 +37,7 @@ func New() providerapi.Provider { } func (p *provider) Key() string { - return "gemini" + return api.ProviderKeyGemini } func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { diff --git a/pkg/aiproxy/providers/images_test.go b/pkg/aiproxy/providers/images_test.go index df69c6f613..81cb3560d7 100644 --- a/pkg/aiproxy/providers/images_test.go +++ b/pkg/aiproxy/providers/images_test.go @@ -20,6 +20,8 @@ import ( "testing" "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" ) func TestOpenAIImagesCompatBuild(t *testing.T) { @@ -98,7 +100,10 @@ func TestAzureImagesBuild(t *testing.T) { body.Add(jsonutils.NewString("dall-e-3"), "model") body.Add(jsonutils.NewString("test"), "prompt") - p := GetImages("azure") + p, ok := azure.New().(ImagesProvider) + if !ok { + t.Fatal("azure provider should implement ImagesProvider") + } req, err := p.BuildImagesGenerationsRequest(&ChatContext{ BaseURL: "https://example.openai.azure.com", APIKey: "key", diff --git a/pkg/aiproxy/providers/messages/doc.go b/pkg/aiproxy/providers/messages/doc.go new file mode 100644 index 0000000000..1edc8357d4 --- /dev/null +++ b/pkg/aiproxy/providers/messages/doc.go @@ -0,0 +1 @@ +package messages // import "yunion.io/x/onecloud/pkg/aiproxy/providers/messages" diff --git a/pkg/aiproxy/providers/messages/messages_test.go b/pkg/aiproxy/providers/messages/messages_test.go new file mode 100644 index 0000000000..2a2553708d --- /dev/null +++ b/pkg/aiproxy/providers/messages/messages_test.go @@ -0,0 +1,164 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messages + +import ( + "encoding/json" + "testing" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" +) + +func TestPassthroughAdapterBuild(t *testing.T) { + adapter, err := GetAdapter("anthropic", "") + if err != nil { + t.Fatal(err) + } + if !adapter.AnthropicStreamPassthrough() { + t.Fatal("expected passthrough stream") + } + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("claude-sonnet-4-5")) + body.Set("max_tokens", jsonutils.NewInt(100)) + user := jsonutils.NewDict() + user.Set("role", jsonutils.NewString("user")) + user.Set("content", jsonutils.NewString("hi")) + body.Set("messages", jsonutils.NewArray(user)) + + req, err := adapter.BuildUpstreamRequest(testChatCtx("anthropic", "https://api.anthropic.com", "sk-ant", "claude-sonnet-4-5"), body, false) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.anthropic.com/v1/messages" { + t.Fatalf("url: %s", req.URL) + } + if req.Headers["x-api-key"] != "sk-ant" { + t.Fatal("missing x-api-key") + } + var wire map[string]interface{} + if err := json.Unmarshal(req.Body, &wire); err != nil { + t.Fatal(err) + } + if wire["model"] != "claude-sonnet-4-5" { + t.Fatalf("model override: %#v", wire["model"]) + } +} + +func TestTranslationAdapterBuildDeepSeek(t *testing.T) { + adapter, err := GetAdapter("openai", "") + if err != nil { + t.Fatal(err) + } + if adapter.AnthropicStreamPassthrough() { + t.Fatal("expected translated stream") + } + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("deepseek-chat")) + body.Set("max_tokens", jsonutils.NewInt(256)) + user := jsonutils.NewDict() + user.Set("role", jsonutils.NewString("user")) + user.Set("content", jsonutils.NewString("hello")) + body.Set("messages", jsonutils.NewArray(user)) + + req, err := adapter.BuildUpstreamRequest(testChatCtx("openai", "https://api.deepseek.com", "ds-key", "deepseek-chat"), body, true) + if err != nil { + t.Fatal(err) + } + if req.URL != "https://api.deepseek.com/v1/chat/completions" { + t.Fatalf("url: %s", req.URL) + } + var wire map[string]interface{} + if err := json.Unmarshal(req.Body, &wire); err != nil { + t.Fatal(err) + } + if wire["model"] != "deepseek-chat" { + t.Fatalf("model: %#v", wire["model"]) + } + if wire["stream"] != true { + t.Fatalf("stream: %#v", wire["stream"]) + } +} + +func TestGetAdapterDeepseekAnthropicPassthrough(t *testing.T) { + adapter, err := GetAdapter("deepseek", "anthropic") + if err != nil { + t.Fatal(err) + } + if !adapter.AnthropicStreamPassthrough() { + t.Fatal("expected passthrough stream for deepseek anthropic mode") + } +} + +func TestGetAdapterDeepseekOpenAITranslation(t *testing.T) { + adapter, err := GetAdapter("deepseek", "openai") + if err != nil { + t.Fatal(err) + } + if adapter.AnthropicStreamPassthrough() { + t.Fatal("expected translated stream for deepseek openai mode") + } +} + +func TestGetAdapterCustomAnthropicPassthrough(t *testing.T) { + adapter, err := GetAdapter("custom", "anthropic") + if err != nil { + t.Fatal(err) + } + if !adapter.AnthropicStreamPassthrough() { + t.Fatal("expected passthrough stream for custom anthropic mode") + } +} + +func TestGetAdapterBlocksGemini(t *testing.T) { + if _, err := GetAdapter("gemini", ""); err == nil { + t.Fatal("expected gemini to be unsupported") + } +} + +func TestGetAdapterVLLMTranslation(t *testing.T) { + adapter, err := GetAdapter("vllm", "") + if err != nil { + t.Fatal(err) + } + if adapter.AnthropicStreamPassthrough() { + t.Fatal("expected translated stream for vllm") + } + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("t-vllm")) + body.Set("max_tokens", jsonutils.NewInt(256)) + user := jsonutils.NewDict() + user.Set("role", jsonutils.NewString("user")) + user.Set("content", jsonutils.NewString("hello")) + body.Set("messages", jsonutils.NewArray(user)) + + req, err := adapter.BuildUpstreamRequest(testChatCtx("vllm", "http://127.0.0.1:8000/v1", "sk-test", "t-vllm"), body, true) + if err != nil { + t.Fatal(err) + } + if req.URL != "http://127.0.0.1:8000/v1/chat/completions" { + t.Fatalf("url: %s", req.URL) + } +} + +func testChatCtx(providerKey, baseURL, apiKey, model string) *providerapi.ChatContext { + return &providerapi.ChatContext{ + ProviderKey: providerKey, + BaseURL: baseURL, + APIKey: apiKey, + UpstreamModel: model, + } +} diff --git a/pkg/aiproxy/providers/messages/passthrough.go b/pkg/aiproxy/providers/messages/passthrough.go new file mode 100644 index 0000000000..949c0ed2a3 --- /dev/null +++ b/pkg/aiproxy/providers/messages/passthrough.go @@ -0,0 +1,74 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messages + +import ( + "fmt" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +const anthropicAPIVersion = "2023-06-01" + +type passthroughAdapter struct{} + +func (passthroughAdapter) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + dup := jsonutils.NewDict() + if body != nil { + dup = body.Copy() + } + dup.Set("model", jsonutils.NewString(ctx.UpstreamModel)) + if stream { + dup.Set("stream", jsonutils.JSONTrue) + } + base := strings.TrimSpace(ctx.BaseURL) + if base == "" { + base = "https://api.anthropic.com" + } + return &providerapi.HTTPRequest{ + Method: http.MethodPost, + URL: openai.JoinURL(base, "/v1/messages"), + Headers: map[string]string{ + "x-api-key": strings.TrimSpace(ctx.APIKey), + "anthropic-version": anthropicAPIVersion, + "Content-Type": "application/json", + }, + Body: []byte(dup.String()), + }, nil +} + +func (passthroughAdapter) NormalizeResponse(_ providerapi.Provider, body []byte) ([]byte, error) { + return body, nil +} + +func (passthroughAdapter) AnthropicStreamPassthrough() bool { + return true +} + +func (passthroughAdapter) NewStreamState(string) interface{} { + return nil +} + +func (passthroughAdapter) ConvertStreamPayload(_ interface{}, _ []byte, _ bool) ([]providerapi.AnthropicStreamChunk, error) { + return nil, nil +} diff --git a/pkg/aiproxy/providers/messages/registry.go b/pkg/aiproxy/providers/messages/registry.go new file mode 100644 index 0000000000..261de32944 --- /dev/null +++ b/pkg/aiproxy/providers/messages/registry.go @@ -0,0 +1,46 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messages + +import ( + "fmt" + "strings" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" +) + +var ( + passthrough = passthroughAdapter{} +) + +// GetAdapter returns the MessagesAdapter for a resolved catalog provider_key and api_mode. +func GetAdapter(providerKey, apiMode string) (providerapi.MessagesAdapter, error) { + key := strings.ToLower(strings.TrimSpace(providerKey)) + mode := strings.ToLower(strings.TrimSpace(apiMode)) + if mode == "" { + mode = api.ProviderAPIModeOpenAI + } + if key == api.ProviderKeyAnthropic { + return passthrough, nil + } + if api.IsNativeMessagesAdapterProvider(key) { + return nil, fmt.Errorf("provider %q does not support anthropic messages API", providerKey) + } + if mode == api.ProviderAPIModeAnthropic && api.SupportsDualAPIMode(key) { + return passthrough, nil + } + return translationAdapter{providerKey: providerKey}, nil +} diff --git a/pkg/aiproxy/providers/messages/translation.go b/pkg/aiproxy/providers/messages/translation.go new file mode 100644 index 0000000000..5babcd52fb --- /dev/null +++ b/pkg/aiproxy/providers/messages/translation.go @@ -0,0 +1,96 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package messages + +import ( + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providerapi" + "yunion.io/x/onecloud/pkg/aiproxy/providers" + "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" +) + +type translationStreamState struct { + conv *openai.AnthropicStreamConverter +} + +type translationAdapter struct { + providerKey string +} + +func (a translationAdapter) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) { + if ctx == nil { + return nil, fmt.Errorf("nil chat context") + } + openaiBody, err := openai.AnthropicToChatCompletions(body, ctx.UpstreamModel) + if err != nil { + return nil, err + } + if stream { + openaiBody.Set("stream", jsonutils.JSONTrue) + streamOpts := jsonutils.NewDict() + streamOpts.Set("include_usage", jsonutils.JSONTrue) + openaiBody.Set("stream_options", streamOpts) + } + prov := providers.Get(a.providerKey) + return prov.BuildUpstreamRequest(&providerapi.ChatContext{ + ProviderKey: ctx.ProviderKey, + BaseURL: ctx.BaseURL, + APIKey: ctx.APIKey, + UpstreamModel: ctx.UpstreamModel, + }, openaiBody, stream) +} + +func (a translationAdapter) NormalizeResponse(prov providerapi.Provider, body []byte) ([]byte, error) { + norm, err := prov.NormalizeResponse(body) + if err != nil { + return nil, err + } + if len(norm) > 0 { + body = norm + } + return openai.ChatCompletionToAnthropic(body) +} + +func (translationAdapter) AnthropicStreamPassthrough() bool { + return false +} + +func (translationAdapter) NewStreamState(requestModel string) interface{} { + return &translationStreamState{ + conv: openai.NewAnthropicStreamConverter(requestModel), + } +} + +func (translationAdapter) ConvertStreamPayload(state interface{}, payload []byte, endOfStream bool) ([]providerapi.AnthropicStreamChunk, error) { + st, ok := state.(*translationStreamState) + if !ok || st == nil || st.conv == nil { + return nil, nil + } + events, err := st.conv.Feed(payload, endOfStream) + if err != nil { + return nil, err + } + out := make([]providerapi.AnthropicStreamChunk, 0, len(events)) + for _, evt := range events { + out = append(out, providerapi.AnthropicStreamChunk{ + Event: evt.Event, + Data: evt.Data, + }) + } + return out, nil +} diff --git a/pkg/aiproxy/providers/openai/anthropic_compat.go b/pkg/aiproxy/providers/openai/anthropic_compat.go new file mode 100644 index 0000000000..4a09d86bf5 --- /dev/null +++ b/pkg/aiproxy/providers/openai/anthropic_compat.go @@ -0,0 +1,508 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "encoding/json" + "fmt" + "strings" + + "yunion.io/x/jsonutils" +) + +// AnthropicToChatCompletions converts an Anthropic Messages request body to OpenAI chat/completions shape. +func AnthropicToChatCompletions(body *jsonutils.JSONDict, upstreamModel string) (*jsonutils.JSONDict, error) { + if body == nil { + return nil, fmt.Errorf("nil request body") + } + out := jsonutils.NewDict() + model := strings.TrimSpace(upstreamModel) + if model == "" { + if m, err := body.GetString("model"); err == nil { + model = strings.TrimSpace(m) + } + } + if model == "" { + return nil, fmt.Errorf("missing model") + } + out.Set("model", jsonutils.NewString(model)) + + maxTokens, err := body.Int("max_tokens") + if err != nil || maxTokens <= 0 { + return nil, fmt.Errorf("max_tokens is required") + } + out.Set("max_tokens", jsonutils.NewInt(maxTokens)) + + if stream, _ := body.Bool("stream"); stream { + out.Set("stream", jsonutils.JSONTrue) + streamOpts := jsonutils.NewDict() + streamOpts.Set("include_usage", jsonutils.JSONTrue) + out.Set("stream_options", streamOpts) + } + if v, ok := FloatParam(body, "temperature"); ok { + out.Set("temperature", jsonutils.NewFloat64(v)) + } + if v, ok := FloatParam(body, "top_p"); ok { + out.Set("top_p", jsonutils.NewFloat64(v)) + } + if stops, err := body.Get("stop_sequences"); err == nil { + out.Set("stop", stops) + } + + msgs, err := anthropicMessagesToOpenAI(body) + if err != nil { + return nil, err + } + out.Set("messages", msgs) + + if tools, toolChoice, err := anthropicToolsToOpenAI(body); err != nil { + return nil, err + } else if tools != nil && tools.Length() > 0 { + out.Set("tools", tools) + if toolChoice != nil { + out.Set("tool_choice", toolChoice) + } + } + return out, nil +} + +func anthropicMessagesToOpenAI(body *jsonutils.JSONDict) (*jsonutils.JSONArray, error) { + var systemParts []string + if sysRaw, err := body.Get("system"); err == nil { + if sysText := anthropicSystemText(sysRaw); sysText != "" { + systemParts = append(systemParts, sysText) + } + } + rawMsgs, err := body.Get("messages") + if err != nil { + return nil, fmt.Errorf("missing messages") + } + var messages []json.RawMessage + if err := json.Unmarshal([]byte(rawMsgs.String()), &messages); err != nil { + return nil, fmt.Errorf("invalid messages: %w", err) + } + converted := jsonutils.NewArray() + for _, raw := range messages { + var msg struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + } + if err := json.Unmarshal(raw, &msg); err != nil { + return nil, fmt.Errorf("invalid message: %w", err) + } + role := strings.ToLower(strings.TrimSpace(msg.Role)) + switch role { + case "system": + sysText, err := anthropicMessageContentText(msg.Content) + if err != nil { + return nil, err + } + if sysText != "" { + systemParts = append(systemParts, sysText) + } + case "user": + parts, tools, err := parseAnthropicUserContent(msg.Content) + if err != nil { + return nil, err + } + for _, tr := range tools { + toolMsg := jsonutils.NewDict() + toolMsg.Set("role", jsonutils.NewString("tool")) + toolMsg.Set("tool_call_id", jsonutils.NewString(tr.ID)) + toolMsg.Set("content", jsonutils.NewString(tr.Content)) + converted.Add(toolMsg) + } + if parts != nil { + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", parts) + converted.Add(userMsg) + } + case "assistant": + assistant, err := parseAnthropicAssistantContent(msg.Content) + if err != nil { + return nil, err + } + if assistant == nil { + continue + } + converted.Add(assistant) + default: + return nil, fmt.Errorf("unsupported message role %q", role) + } + } + if converted.Size() == 0 { + return nil, fmt.Errorf("no convertible messages") + } + arr := jsonutils.NewArray() + if len(systemParts) > 0 { + sysMsg := jsonutils.NewDict() + sysMsg.Set("role", jsonutils.NewString("system")) + sysMsg.Set("content", jsonutils.NewString(strings.Join(systemParts, "\n\n"))) + arr.Add(sysMsg) + } + for i := 0; i < converted.Size(); i++ { + obj, err := converted.GetAt(i) + if err != nil { + return nil, err + } + arr.Add(obj) + } + return arr, nil +} + +func anthropicMessageContentText(raw json.RawMessage) (string, error) { + if len(raw) == 0 { + return "", nil + } + parsed, err := jsonutils.Parse(raw) + if err != nil { + return "", fmt.Errorf("invalid message content: %w", err) + } + return anthropicSystemText(parsed), nil +} + +type anthropicToolResult struct { + ID string + Content string +} + +func anthropicSystemText(raw jsonutils.JSONObject) string { + if raw == nil { + return "" + } + var s string + if err := json.Unmarshal([]byte(raw.String()), &s); err == nil { + return strings.TrimSpace(s) + } + var blocks []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal([]byte(raw.String()), &blocks); err == nil { + var b strings.Builder + for _, blk := range blocks { + if blk.Type == "text" && blk.Text != "" { + if b.Len() > 0 { + b.WriteString("\n") + } + b.WriteString(blk.Text) + } + } + return b.String() + } + return "" +} + +func parseAnthropicUserContent(raw json.RawMessage) (jsonutils.JSONObject, []anthropicToolResult, error) { + if len(raw) == 0 { + return nil, nil, nil + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + if strings.TrimSpace(s) == "" { + return nil, nil, nil + } + return jsonutils.NewString(s), nil, nil + } + var blocks []map[string]interface{} + if err := json.Unmarshal(raw, &blocks); err != nil { + return nil, nil, fmt.Errorf("invalid user content: %w", err) + } + var textParts []string + var tools []anthropicToolResult + for _, blk := range blocks { + typ, _ := blk["type"].(string) + switch typ { + case "text": + if t, _ := blk["text"].(string); t != "" { + textParts = append(textParts, t) + } + case "tool_result": + id, _ := blk["tool_use_id"].(string) + content := anthropicBlockContentText(blk["content"]) + tools = append(tools, anthropicToolResult{ID: id, Content: content}) + } + } + if len(textParts) == 0 { + return nil, tools, nil + } + if len(textParts) == 1 { + return jsonutils.NewString(textParts[0]), tools, nil + } + parts := jsonutils.NewArray() + for _, p := range textParts { + blk := jsonutils.NewDict() + blk.Set("type", jsonutils.NewString("text")) + blk.Set("text", jsonutils.NewString(p)) + parts.Add(blk) + } + return parts, tools, nil +} + +func anthropicBlockContentText(v interface{}) string { + switch c := v.(type) { + case string: + return c + case []interface{}: + var parts []string + for _, item := range c { + if m, ok := item.(map[string]interface{}); ok { + if t, _ := m["text"].(string); t != "" { + parts = append(parts, t) + } + } + } + return strings.Join(parts, "\n") + default: + return fmt.Sprint(v) + } +} + +func parseAnthropicAssistantContent(raw json.RawMessage) (*jsonutils.JSONDict, error) { + if len(raw) == 0 { + return nil, nil + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + if strings.TrimSpace(s) == "" { + return nil, nil + } + msg := jsonutils.NewDict() + msg.Set("role", jsonutils.NewString("assistant")) + msg.Set("content", jsonutils.NewString(s)) + return msg, nil + } + var blocks []AnthropicBlock + if err := json.Unmarshal(raw, &blocks); err != nil { + return nil, fmt.Errorf("invalid assistant content: %w", err) + } + assistant := AnthropicBlocksToAssistant(blocks) + msg := jsonutils.NewDict() + msg.Set("role", jsonutils.NewString("assistant")) + if assistant.Content != "" { + msg.Set("content", jsonutils.NewString(assistant.Content)) + } + if len(assistant.ToolCalls) > 0 { + calls := jsonutils.NewArray() + for _, tc := range assistant.ToolCalls { + call := jsonutils.NewDict() + call.Set("id", jsonutils.NewString(tc.ID)) + call.Set("type", jsonutils.NewString("function")) + fn := jsonutils.NewDict() + fn.Set("name", jsonutils.NewString(tc.Function.Name)) + fn.Set("arguments", jsonutils.NewString(tc.Function.Arguments)) + call.Set("function", fn) + calls.Add(call) + } + msg.Set("tool_calls", calls) + } + return msg, nil +} + +func anthropicToolsToOpenAI(body *jsonutils.JSONDict) (*jsonutils.JSONArray, jsonutils.JSONObject, error) { + rawTools, err := body.Get("tools") + if err != nil { + return nil, nil, nil + } + var toolsIn []struct { + Name string `json:"name"` + Description string `json:"description"` + InputSchema json.RawMessage `json:"input_schema"` + } + if err := json.Unmarshal([]byte(rawTools.String()), &toolsIn); err != nil { + return nil, nil, fmt.Errorf("invalid tools: %w", err) + } + if len(toolsIn) == 0 { + return nil, nil, nil + } + out := jsonutils.NewArray() + for _, t := range toolsIn { + name := strings.TrimSpace(t.Name) + if name == "" { + continue + } + tool := jsonutils.NewDict() + tool.Set("type", jsonutils.NewString("function")) + fn := jsonutils.NewDict() + fn.Set("name", jsonutils.NewString(name)) + if desc := strings.TrimSpace(t.Description); desc != "" { + fn.Set("description", jsonutils.NewString(desc)) + } + if len(t.InputSchema) > 0 && string(t.InputSchema) != "null" { + if params, err := jsonutils.Parse(t.InputSchema); err == nil { + fn.Set("parameters", params) + } + } + tool.Set("function", fn) + out.Add(tool) + } + var toolChoice jsonutils.JSONObject + if tcRaw, err := body.Get("tool_choice"); err == nil { + toolChoice = anthropicToolChoiceToOpenAI(tcRaw) + } + return out, toolChoice, nil +} + +func anthropicToolChoiceToOpenAI(raw jsonutils.JSONObject) jsonutils.JSONObject { + var obj map[string]interface{} + if err := json.Unmarshal([]byte(raw.String()), &obj); err != nil { + return nil + } + typ, _ := obj["type"].(string) + switch strings.ToLower(strings.TrimSpace(typ)) { + case "auto", "": + return jsonutils.NewString("auto") + case "none": + return jsonutils.NewString("none") + case "any": + return jsonutils.NewString("required") + case "tool": + name, _ := obj["name"].(string) + if strings.TrimSpace(name) == "" { + return jsonutils.NewString("required") + } + choice := jsonutils.NewDict() + choice.Set("type", jsonutils.NewString("function")) + fn := jsonutils.NewDict() + fn.Set("name", jsonutils.NewString(strings.TrimSpace(name))) + choice.Set("function", fn) + return choice + default: + return nil + } +} + +// ChatCompletionToAnthropic converts an OpenAI chat.completion JSON body to Anthropic Messages response. +func ChatCompletionToAnthropic(body []byte) ([]byte, error) { + var resp struct { + ID string `json:"id"` + Model string `json:"model"` + Choices []struct { + Message struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + ToolCalls []ToolCall `json:"tool_calls"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("invalid OpenAI response: %w", err) + } + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("empty OpenAI choices") + } + choice := resp.Choices[0] + blocks := make([]map[string]interface{}, 0, 1+len(choice.Message.ToolCalls)) + if text := MessageTextContent(choice.Message.Content); text != "" { + blocks = append(blocks, map[string]interface{}{ + "type": "text", + "text": text, + }) + } + for _, tc := range choice.Message.ToolCalls { + input := map[string]interface{}{} + if args := strings.TrimSpace(tc.Function.Arguments); args != "" { + _ = json.Unmarshal([]byte(args), &input) + } + id := strings.TrimSpace(tc.ID) + if id == "" { + id = "toolu_" + strings.TrimSpace(tc.Function.Name) + } + blocks = append(blocks, map[string]interface{}{ + "type": "tool_use", + "id": id, + "name": strings.TrimSpace(tc.Function.Name), + "input": input, + }) + } + stopReason := openAIFinishReasonToAnthropic(choice.FinishReason) + out := map[string]interface{}{ + "id": resp.ID, + "type": "message", + "role": "assistant", + "model": resp.Model, + "content": blocks, + "stop_reason": stopReason, + "usage": map[string]interface{}{ + "input_tokens": resp.Usage.PromptTokens, + "output_tokens": resp.Usage.CompletionTokens, + }, + } + return json.Marshal(out) +} + +func openAIFinishReasonToAnthropic(reason string) string { + switch strings.TrimSpace(reason) { + case "stop": + return "end_turn" + case "length": + return "max_tokens" + case "tool_calls", "function_call": + return "tool_use" + default: + if reason == "" { + return "end_turn" + } + return reason + } +} + +// OpenAIErrorToAnthropic converts an OpenAI-style error JSON body to Anthropic error format. +func OpenAIErrorToAnthropic(body []byte, statusCode int) []byte { + msg := "upstream request failed" + var wrap struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + } `json:"error"` + } + if json.Unmarshal(body, &wrap) == nil && wrap.Error.Message != "" { + msg = wrap.Error.Message + } + errType := "api_error" + if statusCode == 400 { + errType = "invalid_request_error" + } else if statusCode == 401 { + errType = "authentication_error" + } else if statusCode == 429 { + errType = "rate_limit_error" + } + out, _ := json.Marshal(map[string]interface{}{ + "type": "error", + "error": map[string]interface{}{ + "type": errType, + "message": msg, + }, + }) + return out +} + +// NewAnthropicErrorBody builds an Anthropic-style error response body. +func NewAnthropicErrorBody(errType, message string) []byte { + out, _ := json.Marshal(map[string]interface{}{ + "type": "error", + "error": map[string]interface{}{ + "type": errType, + "message": message, + }, + }) + return out +} diff --git a/pkg/aiproxy/providers/openai/anthropic_compat_test.go b/pkg/aiproxy/providers/openai/anthropic_compat_test.go new file mode 100644 index 0000000000..28e3627f3b --- /dev/null +++ b/pkg/aiproxy/providers/openai/anthropic_compat_test.go @@ -0,0 +1,347 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "encoding/json" + "testing" + + "yunion.io/x/jsonutils" +) + +func TestAnthropicToChatCompletionsSystemInMessages(t *testing.T) { + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("deepseek-chat")) + body.Set("max_tokens", jsonutils.NewInt(512)) + sysMsg := jsonutils.NewDict() + sysMsg.Set("role", jsonutils.NewString("system")) + sysMsg.Set("content", jsonutils.NewString("You are Claude Code.")) + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("hi")) + body.Set("messages", jsonutils.NewArray(sysMsg, userMsg)) + + out, err := AnthropicToChatCompletions(body, "deepseek-chat") + if err != nil { + t.Fatal(err) + } + msgs, err := out.Get("messages") + if err != nil { + t.Fatal(err) + } + arr, ok := msgs.(*jsonutils.JSONArray) + if !ok || arr.Length() != 2 { + t.Fatalf("expected system+user messages, got %#v", msgs) + } + role0, _ := arr.GetAt(0) + if got, _ := role0.(*jsonutils.JSONDict).GetString("role"); got != "system" { + t.Fatalf("first message role: got %q", got) + } +} + +func TestAnthropicToChatCompletionsBasic(t *testing.T) { + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("deepseek-chat")) + body.Set("max_tokens", jsonutils.NewInt(512)) + body.Set("system", jsonutils.NewString("You are helpful.")) + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("Hello")) + body.Set("messages", jsonutils.NewArray(userMsg)) + + out, err := AnthropicToChatCompletions(body, "deepseek-chat") + if err != nil { + t.Fatal(err) + } + if got, _ := out.GetString("model"); got != "deepseek-chat" { + t.Fatalf("model: got %q", got) + } + msgs, err := out.Get("messages") + if err != nil { + t.Fatal(err) + } + arr, ok := msgs.(*jsonutils.JSONArray) + if !ok || arr.Length() != 2 { + t.Fatalf("expected system+user messages, got %#v", msgs) + } +} + +func TestChatCompletionToAnthropicBasic(t *testing.T) { + raw := []byte(`{ + "id":"chatcmpl-1", + "model":"deepseek-chat", + "choices":[{"message":{"role":"assistant","content":"Hi there"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":5,"completion_tokens":3} + }`) + out, err := ChatCompletionToAnthropic(raw) + if err != nil { + t.Fatal(err) + } + var resp map[string]interface{} + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if resp["type"] != "message" { + t.Fatalf("unexpected type: %#v", resp["type"]) + } + if resp["stop_reason"] != "end_turn" { + t.Fatalf("unexpected stop_reason: %#v", resp["stop_reason"]) + } + content := resp["content"].([]interface{}) + if len(content) != 1 { + t.Fatalf("expected one content block, got %#v", content) + } +} + +func TestAnthropicStreamConverterMultipleTools(t *testing.T) { + conv := NewAnthropicStreamConverter("deepseek-chat") + textChunk, _ := json.Marshal(NewStreamChunk("deepseek-chat", "chatcmpl-1", 0, "Checking.", "")) + if _, err := conv.Feed(textChunk, false); err != nil { + t.Fatal(err) + } + + tool1Start, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{ + ID: "call_0", Type: "function", Function: ToolFunction{Name: "Bash"}, + }, "")) + events, err := conv.Feed(tool1Start, false) + if err != nil { + t.Fatal(err) + } + if got := blockIndexFromEvent(t, events[1]); got != 1 { + t.Fatalf("tool1 start index: got %d", got) + } + + tool1End, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{ + Function: ToolFunction{Arguments: `{"command":"ls"}`}, + }, "")) + if _, err := conv.Feed(tool1End, false); err != nil { + t.Fatal(err) + } + + tool2Start, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 1, ToolCall{ + ID: "call_1", Type: "function", Function: ToolFunction{Name: "Bash"}, + }, "")) + events, err = conv.Feed(tool2Start, false) + if err != nil { + t.Fatal(err) + } + if len(events) != 2 { + t.Fatalf("expected stop+start for tool2, got %v", eventNames(events)) + } + if events[0].Event != "content_block_stop" { + t.Fatalf("expected stop before tool2, got %v", eventNames(events)) + } + if got := blockIndexFromEvent(t, events[0]); got != 1 { + t.Fatalf("tool1 stop index: got %d want 1", got) + } + if got := blockIndexFromEvent(t, events[1]); got != 2 { + t.Fatalf("tool2 start index: got %d want 2", got) + } + + finishChunk, _ := json.Marshal(map[string]interface{}{ + "id": "chatcmpl-1", + "choices": []map[string]interface{}{ + {"delta": map[string]interface{}{}, "finish_reason": "tool_calls"}, + }, + "usage": map[string]interface{}{ + "prompt_tokens": 100, + "completion_tokens": 50, + }, + }) + events, err = conv.Feed(finishChunk, true) + if err != nil { + t.Fatal(err) + } + var sawStop2, sawMessageStop bool + for _, evt := range events { + switch evt.Event { + case "content_block_stop": + if blockIndexFromEvent(t, evt) == 2 { + sawStop2 = true + } + case "message_stop": + sawMessageStop = true + } + } + if !sawStop2 { + t.Fatalf("expected content_block_stop for tool2, got %v", eventNames(events)) + } + if !sawMessageStop { + t.Fatalf("expected message_stop, got %v", eventNames(events)) + } + if conv.outputTokens != 50 { + t.Fatalf("output tokens: got %d want 50", conv.outputTokens) + } +} + +func blockIndexFromEvent(t *testing.T, evt AnthropicStreamEvent) int { + t.Helper() + var wrap map[string]interface{} + if err := json.Unmarshal(evt.Data, &wrap); err != nil { + t.Fatal(err) + } + idx, ok := wrap["index"].(float64) + if !ok { + t.Fatalf("missing index in %s: %#v", evt.Event, wrap) + } + return int(idx) +} + +func TestAnthropicStreamConverterTextToTool(t *testing.T) { + conv := NewAnthropicStreamConverter("deepseek-chat") + textChunk, _ := json.Marshal(NewStreamChunk("deepseek-chat", "chatcmpl-1", 0, "Let me check.", "")) + events, err := conv.Feed(textChunk, false) + if err != nil { + t.Fatal(err) + } + if len(events) < 3 { + t.Fatalf("expected message_start + block start + delta, got %d events", len(events)) + } + + toolChunk, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{ + ID: "call_1", + Type: "function", + Function: ToolFunction{ + Name: "Glob", + }, + }, "")) + events, err = conv.Feed(toolChunk, false) + if err != nil { + t.Fatal(err) + } + if len(events) != 2 { + t.Fatalf("expected content_block_stop + content_block_start, got %d events: %v", len(events), eventNames(events)) + } + if events[0].Event != "content_block_stop" || events[1].Event != "content_block_start" { + t.Fatalf("unexpected events: %v", eventNames(events)) + } + var start map[string]interface{} + if err := json.Unmarshal(events[1].Data, &start); err != nil { + t.Fatal(err) + } + if int(start["index"].(float64)) != 1 { + t.Fatalf("tool block index: %#v", start["index"]) + } + + argsChunk, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{ + Function: ToolFunction{Arguments: `{"pattern":"**/*.go"}`}, + }, "")) + events, err = conv.Feed(argsChunk, false) + if err != nil { + t.Fatal(err) + } + if len(events) != 1 || events[0].Event != "content_block_delta" { + t.Fatalf("expected tool args delta, got %v", eventNames(events)) + } + var delta map[string]interface{} + if err := json.Unmarshal(events[0].Data, &delta); err != nil { + t.Fatal(err) + } + if int(delta["index"].(float64)) != 1 { + t.Fatalf("tool delta index: %#v", delta["index"]) + } +} + +func eventNames(events []AnthropicStreamEvent) []string { + out := make([]string, len(events)) + for i, evt := range events { + out[i] = evt.Event + } + return out +} + +func TestAnthropicStreamConverterText(t *testing.T) { + conv := NewAnthropicStreamConverter("deepseek-chat") + chunk, _ := json.Marshal(NewStreamChunk("deepseek-chat", "chatcmpl-1", 0, "Hello", "")) + events, err := conv.Feed(chunk, false) + if err != nil { + t.Fatal(err) + } + if len(events) == 0 { + t.Fatal("expected stream events") + } + if events[0].Event != "message_start" { + t.Fatalf("first event: %s", events[0].Event) + } + finishChunk, _ := json.Marshal(map[string]interface{}{ + "id": "chatcmpl-1", + "choices": []map[string]interface{}{ + {"delta": map[string]interface{}{}, "finish_reason": "stop"}, + }, + "usage": map[string]interface{}{ + "prompt_tokens": 3, + "completion_tokens": 2, + }, + }) + events, err = conv.Feed(finishChunk, true) + if err != nil { + t.Fatal(err) + } + foundStop := false + for _, evt := range events { + if evt.Event == "message_stop" { + foundStop = true + } + } + if !foundStop { + t.Fatal("expected message_stop event") + } +} + +func TestAnthropicToolRoundTrip(t *testing.T) { + body := jsonutils.NewDict() + body.Set("model", jsonutils.NewString("claude-3-5-sonnet")) + body.Set("max_tokens", jsonutils.NewInt(1024)) + userMsg := jsonutils.NewDict() + userMsg.Set("role", jsonutils.NewString("user")) + userMsg.Set("content", jsonutils.NewString("Weather?")) + body.Set("messages", jsonutils.NewArray(userMsg)) + tool := jsonutils.NewDict() + tool.Set("name", jsonutils.NewString("get_weather")) + tool.Set("description", jsonutils.NewString("Get weather")) + tool.Set("input_schema", jsonutils.NewDict()) + body.Set("tools", jsonutils.NewArray(tool)) + + openaiBody, err := AnthropicToChatCompletions(body, "claude-3-5-sonnet") + if err != nil { + t.Fatal(err) + } + toolsObj, err := openaiBody.Get("tools") + if err != nil { + t.Fatal(err) + } + toolsArr, ok := toolsObj.(*jsonutils.JSONArray) + if !ok || toolsArr.Length() != 1 { + t.Fatalf("expected tools in OpenAI body: %#v", openaiBody) + } + + raw := []byte(`{ + "id":"msg_1", + "model":"claude-3-5-sonnet", + "choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Boston\"}"}}]},"finish_reason":"tool_calls"}], + "usage":{"prompt_tokens":10,"completion_tokens":5} + }`) + out, err := ChatCompletionToAnthropic(raw) + if err != nil { + t.Fatal(err) + } + var resp map[string]interface{} + if err := json.Unmarshal(out, &resp); err != nil { + t.Fatal(err) + } + if resp["stop_reason"] != "tool_use" { + t.Fatalf("stop_reason: %#v", resp["stop_reason"]) + } +} diff --git a/pkg/aiproxy/providers/openai/anthropic_stream_compat.go b/pkg/aiproxy/providers/openai/anthropic_stream_compat.go new file mode 100644 index 0000000000..b1492f2f9c --- /dev/null +++ b/pkg/aiproxy/providers/openai/anthropic_stream_compat.go @@ -0,0 +1,340 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openai + +import ( + "encoding/json" + "strings" +) + +// AnthropicStreamEvent is one Anthropic Messages SSE event for upstream clients. +type AnthropicStreamEvent struct { + Event string + Data []byte +} + +// AnthropicStreamConverter converts OpenAI chat.completion.chunk SSE payloads to Anthropic SSE events. +type AnthropicStreamConverter struct { + requestModel string + messageStarted bool + hasOpenBlock bool + openBlockIndex int + closingEmitted bool + messageID string + model string + stopReason string + inputTokens int + outputTokens int + blockIndex int + activeTools map[int]*streamToolState +} + +type streamToolState struct { + blockIdx int + id string + name string +} + +// NewAnthropicStreamConverter creates stream conversion state for one Anthropic Messages response. +func NewAnthropicStreamConverter(requestModel string) *AnthropicStreamConverter { + return &AnthropicStreamConverter{ + requestModel: requestModel, + activeTools: make(map[int]*streamToolState), + openBlockIndex: -1, + } +} + +// Feed processes one OpenAI SSE data payload (without the "data:" prefix). +func (s *AnthropicStreamConverter) Feed(payload []byte, endOfStream bool) ([]AnthropicStreamEvent, error) { + if len(payload) == 0 { + if endOfStream && !s.closingEmitted { + return s.emitClosing() + } + return nil, nil + } + if string(payload) == "[DONE]" { + if !s.closingEmitted { + return s.emitClosing() + } + return nil, nil + } + var chunk struct { + ID string `json:"id"` + Model string `json:"model"` + Choices []struct { + Delta struct { + Content *string `json:"content"` + ToolCalls []ToolCall `json:"tool_calls"` + } `json:"delta"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(payload, &chunk); err != nil { + return nil, nil + } + if chunk.ID != "" && s.messageID == "" { + s.messageID = chunk.ID + } + if chunk.Model != "" && s.model == "" { + s.model = chunk.Model + } + if chunk.Usage != nil { + s.inputTokens = chunk.Usage.PromptTokens + s.outputTokens = chunk.Usage.CompletionTokens + } + var out []AnthropicStreamEvent + if chunk.Usage != nil && len(chunk.Choices) == 0 { + closing, err := s.emitClosing() + if err != nil { + return nil, err + } + return closing, nil + } + if len(chunk.Choices) == 0 { + if endOfStream && !s.closingEmitted { + return s.emitClosing() + } + return nil, nil + } + choice := chunk.Choices[0] + if !s.messageStarted { + events, err := s.emitMessageStart() + if err != nil { + return nil, err + } + out = append(out, events...) + } + if choice.Delta.Content != nil && *choice.Delta.Content != "" { + if !s.hasOpenBlock { + events, err := s.emitTextBlockStart() + if err != nil { + return nil, err + } + out = append(out, events...) + } + events, err := s.emitTextDelta(*choice.Delta.Content) + if err != nil { + return nil, err + } + out = append(out, events...) + } + for _, tc := range choice.Delta.ToolCalls { + events, err := s.handleToolDelta(tc) + if err != nil { + return nil, err + } + out = append(out, events...) + } + if choice.FinishReason != "" { + s.stopReason = openAIFinishReasonToAnthropic(choice.FinishReason) + } + if endOfStream && !s.closingEmitted { + closing, err := s.emitClosing() + if err != nil { + return nil, err + } + out = append(out, closing...) + } + return out, nil +} + +func (s *AnthropicStreamConverter) emitMessageStart() ([]AnthropicStreamEvent, error) { + s.messageStarted = true + model := s.model + if model == "" { + model = s.requestModel + } + data, err := json.Marshal(map[string]interface{}{ + "type": "message_start", + "message": map[string]interface{}{ + "id": s.messageID, + "type": "message", + "role": "assistant", + "model": model, + "content": []interface{}{}, + "stop_reason": nil, + "stop_sequence": nil, + "usage": map[string]interface{}{ + "input_tokens": 0, + "output_tokens": 0, + }, + }, + }) + if err != nil { + return nil, err + } + return []AnthropicStreamEvent{{Event: "message_start", Data: data}}, nil +} + +func (s *AnthropicStreamConverter) emitTextBlockStart() ([]AnthropicStreamEvent, error) { + s.hasOpenBlock = true + s.openBlockIndex = s.blockIndex + data, err := json.Marshal(map[string]interface{}{ + "type": "content_block_start", + "index": s.blockIndex, + "content_block": map[string]interface{}{"type": "text", "text": ""}, + }) + if err != nil { + return nil, err + } + return []AnthropicStreamEvent{{Event: "content_block_start", Data: data}}, nil +} + +func (s *AnthropicStreamConverter) emitTextDelta(text string) ([]AnthropicStreamEvent, error) { + data, err := json.Marshal(map[string]interface{}{ + "type": "content_block_delta", + "index": s.blockIndex, + "delta": map[string]interface{}{"type": "text_delta", "text": text}, + }) + if err != nil { + return nil, err + } + return []AnthropicStreamEvent{{Event: "content_block_delta", Data: data}}, nil +} + +func (s *AnthropicStreamConverter) handleToolDelta(tc ToolCall) ([]AnthropicStreamEvent, error) { + var out []AnthropicStreamEvent + idx := tc.Index + st, ok := s.activeTools[idx] + if !ok { + if s.hasOpenBlock { + events, err := s.closeOpenBlock() + if err != nil { + return nil, err + } + out = append(out, events...) + } + st = &streamToolState{blockIdx: s.blockIndex} + s.activeTools[idx] = st + id := strings.TrimSpace(tc.ID) + name := strings.TrimSpace(tc.Function.Name) + if id != "" { + st.id = id + } + if name != "" { + st.name = name + } + if st.id == "" { + st.id = "toolu_" + st.name + } + data, err := json.Marshal(map[string]interface{}{ + "type": "content_block_start", + "index": st.blockIdx, + "content_block": map[string]interface{}{ + "type": "tool_use", + "id": st.id, + "name": st.name, + "input": map[string]interface{}{}, + }, + }) + if err != nil { + return nil, err + } + s.hasOpenBlock = true + s.openBlockIndex = st.blockIdx + s.blockIndex++ + out = append(out, AnthropicStreamEvent{Event: "content_block_start", Data: data}) + } + if tc.Function.Name != "" { + st.name = tc.Function.Name + } + if tc.ID != "" { + st.id = tc.ID + } + if tc.Function.Arguments == "" { + return out, nil + } + data, err := json.Marshal(map[string]interface{}{ + "type": "content_block_delta", + "index": st.blockIdx, + "delta": map[string]interface{}{ + "type": "input_json_delta", + "partial_json": tc.Function.Arguments, + }, + }) + if err != nil { + return nil, err + } + out = append(out, AnthropicStreamEvent{Event: "content_block_delta", Data: data}) + return out, nil +} + +func (s *AnthropicStreamConverter) closeOpenBlock() ([]AnthropicStreamEvent, error) { + if !s.hasOpenBlock { + return nil, nil + } + idx := s.openBlockIndex + if idx < 0 { + idx = s.blockIndex + } + data, err := json.Marshal(map[string]interface{}{ + "type": "content_block_stop", + "index": idx, + }) + if err != nil { + return nil, err + } + s.hasOpenBlock = false + s.openBlockIndex = -1 + events := []AnthropicStreamEvent{{Event: "content_block_stop", Data: data}} + if s.blockIndex <= idx { + s.blockIndex = idx + 1 + } + return events, nil +} + +func (s *AnthropicStreamConverter) emitClosing() ([]AnthropicStreamEvent, error) { + if s.closingEmitted { + return nil, nil + } + s.closingEmitted = true + var out []AnthropicStreamEvent + if s.hasOpenBlock { + events, err := s.closeOpenBlock() + if err != nil { + return nil, err + } + out = append(out, events...) + } + stopReason := s.stopReason + if stopReason == "" { + stopReason = "end_turn" + } + deltaData, err := json.Marshal(map[string]interface{}{ + "type": "message_delta", + "delta": map[string]interface{}{ + "stop_reason": stopReason, + "stop_sequence": nil, + }, + "usage": map[string]interface{}{ + "output_tokens": s.outputTokens, + }, + }) + if err != nil { + return nil, err + } + out = append(out, AnthropicStreamEvent{Event: "message_delta", Data: deltaData}) + + stopData, err := json.Marshal(map[string]interface{}{"type": "message_stop"}) + if err != nil { + return nil, err + } + out = append(out, AnthropicStreamEvent{Event: "message_stop", Data: stopData}) + return out, nil +} diff --git a/pkg/aiproxy/providers/providers_test.go b/pkg/aiproxy/providers/providers_test.go index 6b6bfb5ae3..c607635836 100644 --- a/pkg/aiproxy/providers/providers_test.go +++ b/pkg/aiproxy/providers/providers_test.go @@ -19,6 +19,8 @@ import ( "testing" "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" ) func TestAliyunProviderEnableThinkingPatch(t *testing.T) { @@ -26,7 +28,7 @@ func TestAliyunProviderEnableThinkingPatch(t *testing.T) { body.Add(jsonutils.NewString("qwen-turbo"), "model") body.Add(jsonutils.NewArray(jsonutils.NewDict()), "messages") - p := Get("aliyun") + p := aliyun.New() req, err := p.BuildUpstreamRequest(&ChatContext{ ProviderKey: "aliyun", BaseURL: "https://dashscope.aliyuncs.com/compatible-mode", diff --git a/pkg/aiproxy/providers/registry.go b/pkg/aiproxy/providers/registry.go index 8bfd001d40..0927cac728 100644 --- a/pkg/aiproxy/providers/registry.go +++ b/pkg/aiproxy/providers/registry.go @@ -19,11 +19,12 @@ import ( "sync" "yunion.io/x/onecloud/pkg/aiproxy/providerapi" - "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" + // "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" // uncommon "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic" - "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" - "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" - "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" + // "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" // uncommon + // "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" // uncommon + // "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" // uncommon "yunion.io/x/onecloud/pkg/aiproxy/providers/gemini" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" "yunion.io/x/onecloud/pkg/aiproxy/providers/vllm" @@ -35,36 +36,20 @@ var ( defaultP providerapi.Provider ) -var openAICompatKeys = []string{ - "openai", - "groq", - "mistral", - "cerebras", - "perplexity", - "openrouter", - "fireworks", - "nebius", - "xai", - "parasail", - "sgl", - "huggingface", - "ollama", - "xiaomi", -} - func init() { defaultP = openai.NewCompat("") register(defaultP) - for _, key := range openAICompatKeys { + for _, key := range api.OpenAICompatProviderKeys { register(openai.NewCompat(key)) } - register(cohere.New()) - register(aliyun.New()) - register(baidu.New()) + // register(cohere.New()) // uncommon + // register(aliyun.New()) // uncommon + // register(baidu.New()) // uncommon register(anthropic.New()) register(gemini.New()) - register(azure.New()) + // register(azure.New()) // uncommon register(vllm.New()) + register(openai.NewCompat(api.ProviderKeyCustom)) } // Register adds or replaces a provider implementation for its Key(). diff --git a/pkg/aiproxy/providers/types.go b/pkg/aiproxy/providers/types.go index 588ccd4b5e..86b477262f 100644 --- a/pkg/aiproxy/providers/types.go +++ b/pkg/aiproxy/providers/types.go @@ -19,14 +19,16 @@ import ( ) type ( - ChatContext = api.ChatContext - HTTPRequest = api.HTTPRequest - StreamChunk = api.StreamChunk - StreamState = api.StreamState - Provider = api.Provider - EmbeddingsProvider = api.EmbeddingsProvider - ImagesProvider = api.ImagesProvider - CompletionsProvider = api.CompletionsProvider + ChatContext = api.ChatContext + HTTPRequest = api.HTTPRequest + StreamChunk = api.StreamChunk + StreamState = api.StreamState + Provider = api.Provider + EmbeddingsProvider = api.EmbeddingsProvider + ImagesProvider = api.ImagesProvider + CompletionsProvider = api.CompletionsProvider + MessagesAdapter = api.MessagesAdapter + AnthropicStreamChunk = api.AnthropicStreamChunk ) type ContextualStreamPassthrough = api.ContextualStreamPassthrough diff --git a/pkg/aiproxy/providers/vllm/vllm.go b/pkg/aiproxy/providers/vllm/vllm.go index 94c40bcaf2..9309b1a206 100644 --- a/pkg/aiproxy/providers/vllm/vllm.go +++ b/pkg/aiproxy/providers/vllm/vllm.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/onecloud/pkg/aiproxy/providerapi" "yunion.io/x/onecloud/pkg/aiproxy/providers/openai" + api "yunion.io/x/onecloud/pkg/apis/aiproxy" ) type provider struct { @@ -39,7 +40,7 @@ func patchVLLMRequest(body *jsonutils.JSONDict, stream bool) { func New() providerapi.Provider { patches := []openai.PatchFunc{patchVLLMRequest} return &provider{ - Compat: openai.NewCompat("vllm", patches...), + Compat: openai.NewCompat(api.ProviderKeyVLLM, patches...), completions: openai.NewCompletionsCompat(patches...), } } diff --git a/pkg/aiproxy/upstream/openai_compat.go b/pkg/aiproxy/upstream/openai_compat.go index 8d32739a41..ebbd2e6138 100644 --- a/pkg/aiproxy/upstream/openai_compat.go +++ b/pkg/aiproxy/upstream/openai_compat.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "net/http" + "sort" "strings" "sync" "time" @@ -84,6 +85,27 @@ func ChatCompletionsURL(baseURL string) string { return base + "/v1/chat/completions" } +// ModelsURL builds the OpenAI-compatible models list endpoint from a provider base URL. +func ModelsURL(baseURL string) string { + base := strings.TrimRight(strings.TrimSpace(baseURL), "/") + if strings.HasSuffix(base, "/v1") { + return base + "/models" + } + if hasAPIVersionPathSuffix(base) { + return base + "/models" + } + return base + "/v1/models" +} + +func hasAPIVersionPathSuffix(base string) bool { + idx := strings.LastIndex(base, "/") + if idx < 0 { + return false + } + seg := base[idx+1:] + return len(seg) >= 2 && seg[0] == 'v' && seg[1] >= '0' && seg[1] <= '9' +} + var ( httpClient *http.Client httpClientOnce sync.Once @@ -182,6 +204,136 @@ func ChatCompletion(ctx context.Context, req *Request) (*Response, *Error) { return &Response{StatusCode: resp.StatusCode, Body: body}, nil } +// ListModels performs a GET on the upstream models list endpoint. +func ListModels(ctx context.Context, baseURL, apiKey string) (*Response, *Error) { + req := &Request{ + BaseURL: strings.TrimSpace(baseURL), + URL: ModelsURL(baseURL), + APIKey: strings.TrimSpace(apiKey), + } + httpReq, err := newUpstreamGETRequest(ctx, req) + if err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + resp, err := sharedHTTPClient().Do(httpReq) + if err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + body, err := readResponseBody(resp, 4<<20) + if err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, errorFromResponse(resp, body) + } + if err := validateModelsListBody(body); err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + if _, err := ParseModelsListBody(body); err != nil { + return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()} + } + return &Response{StatusCode: resp.StatusCode, Body: body}, nil +} + +// ParseModelsListBody extracts upstream model ids from a list-models JSON body. +func ParseModelsListBody(body []byte) ([]string, error) { + if err := validateModelsListBody(body); err != nil { + return nil, err + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("invalid models response JSON") + } + keys := make([]string, 0) + if dataRaw, ok := raw["data"]; ok { + var items []struct { + ID string `json:"id"` + } + if err := json.Unmarshal(dataRaw, &items); err != nil { + return nil, fmt.Errorf("invalid models data array") + } + for _, item := range items { + if id := strings.TrimSpace(item.ID); id != "" { + keys = append(keys, id) + } + } + } + if modelsRaw, ok := raw["models"]; ok { + var items []struct { + Name string `json:"name"` + } + if err := json.Unmarshal(modelsRaw, &items); err != nil { + return nil, fmt.Errorf("invalid models array") + } + for _, item := range items { + name := strings.TrimSpace(item.Name) + name = strings.TrimPrefix(name, "models/") + if name != "" { + keys = append(keys, name) + } + } + } + if len(keys) == 0 { + return []string{}, nil + } + seen := make(map[string]struct{}, len(keys)) + uniq := make([]string, 0, len(keys)) + for _, key := range keys { + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + uniq = append(uniq, key) + } + sort.Strings(uniq) + return uniq, nil +} + +func newUpstreamGETRequest(ctx context.Context, req *Request) (*http.Request, error) { + if req == nil { + return nil, fmt.Errorf("nil upstream request") + } + url := strings.TrimSpace(req.URL) + if url == "" { + url = ModelsURL(req.BaseURL) + } + apiKey := strings.TrimSpace(req.APIKey) + if url == "" { + return nil, fmt.Errorf("empty upstream URL") + } + if apiKey == "" && len(req.Headers) == 0 { + return nil, fmt.Errorf("empty API key") + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + for k, v := range req.Headers { + httpReq.Header.Set(k, v) + } + if apiKey != "" && httpReq.Header.Get("Authorization") == "" && httpReq.Header.Get("x-api-key") == "" && httpReq.Header.Get("api-key") == "" && httpReq.Header.Get("x-goog-api-key") == "" { + httpReq.Header.Set("Authorization", "Bearer "+apiKey) + } + return httpReq, nil +} + +func validateModelsListBody(body []byte) error { + if len(body) == 0 { + return fmt.Errorf("empty models response") + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return fmt.Errorf("invalid models response JSON") + } + if _, ok := raw["data"]; ok { + return nil + } + if _, ok := raw["models"]; ok { + return nil + } + return fmt.Errorf("models response missing data or models field") +} + // ChatCompletionStream opens a streaming chat completions request and returns SSE data chunks. func ChatCompletionStream(ctx context.Context, req *Request) (<-chan StreamChunk, *Error) { _, resp, uerr := openChatCompletionStream(ctx, req) diff --git a/pkg/aiproxy/upstream/openai_compat_models_test.go b/pkg/aiproxy/upstream/openai_compat_models_test.go new file mode 100644 index 0000000000..694a21e7ae --- /dev/null +++ b/pkg/aiproxy/upstream/openai_compat_models_test.go @@ -0,0 +1,66 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upstream + +import "testing" + +func TestModelsURL(t *testing.T) { + cases := []struct { + base string + want string + }{ + {"https://api.openai.com", "https://api.openai.com/v1/models"}, + {"https://api.openai.com/v1", "https://api.openai.com/v1/models"}, + {"https://generativelanguage.googleapis.com/v1beta", "https://generativelanguage.googleapis.com/v1beta/models"}, + {"https://api.deepseek.com/anthropic", "https://api.deepseek.com/anthropic/v1/models"}, + } + for _, tc := range cases { + if got := ModelsURL(tc.base); got != tc.want { + t.Fatalf("ModelsURL(%q) = %q, want %q", tc.base, got, tc.want) + } + } +} + +func TestValidateModelsListBody(t *testing.T) { + if err := validateModelsListBody([]byte(`{"object":"list","data":[]}`)); err != nil { + t.Fatalf("expected valid data field: %v", err) + } + if err := validateModelsListBody([]byte(`{"models":[]}`)); err != nil { + t.Fatalf("expected valid models field: %v", err) + } + if err := validateModelsListBody([]byte(`{"object":"list"}`)); err == nil { + t.Fatal("expected error for missing data/models") + } +} + +func TestParseModelsListBodyOpenAI(t *testing.T) { + keys, err := ParseModelsListBody([]byte(`{"object":"list","data":[{"id":"gpt-4o-mini"},{"id":"gpt-4o"}]}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(keys) != 2 || keys[0] != "gpt-4o" || keys[1] != "gpt-4o-mini" { + t.Fatalf("keys: %#v", keys) + } +} + +func TestParseModelsListBodyGemini(t *testing.T) { + keys, err := ParseModelsListBody([]byte(`{"models":[{"name":"models/gemini-2.0-flash"},{"name":"models/gemini-pro"}]}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(keys) != 2 || keys[0] != "gemini-2.0-flash" || keys[1] != "gemini-pro" { + t.Fatalf("keys: %#v", keys) + } +} diff --git a/pkg/apis/aiproxy/ai_provider.go b/pkg/apis/aiproxy/ai_provider.go index 5bf24b9e92..af2fd10036 100644 --- a/pkg/apis/aiproxy/ai_provider.go +++ b/pkg/apis/aiproxy/ai_provider.go @@ -16,6 +16,7 @@ package aiproxy import ( "encoding/json" + "errors" "strings" "yunion.io/x/onecloud/pkg/apis" @@ -24,7 +25,29 @@ import ( // SAiProviderConfig holds JSON-serialized provider connectivity settings for an ai_provider row. type SAiProviderConfig struct { BaseURL string `json:"base_url,omitempty"` - APIKey string `json:"api_key,omitempty"` + APIMode string `json:"api_mode,omitempty"` +} + +// UnmarshalJSON rejects legacy config.api_key and decodes supported fields only. +func (c *SAiProviderConfig) UnmarshalJSON(data []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + if msg, ok := raw["api_key"]; ok { + var key string + _ = json.Unmarshal(msg, &key) + if strings.TrimSpace(key) != "" { + return errors.New("config.api_key is not supported, use secret and ai_keys") + } + } + type cfgAlias SAiProviderConfig + var alias cfgAlias + if err := json.Unmarshal(data, &alias); err != nil { + return err + } + *c = SAiProviderConfig(alias) + return nil } // ResolvedBaseURL returns config.base_url. @@ -35,12 +58,39 @@ func (c *SAiProviderConfig) ResolvedBaseURL() string { return strings.TrimSpace(c.BaseURL) } -// ResolvedAPIKey returns config.api_key. -func (c *SAiProviderConfig) ResolvedAPIKey() string { +// ResolvedAPIMode returns config.api_mode (default openai). +func (c *SAiProviderConfig) ResolvedAPIMode() string { if c == nil { + return ProviderAPIModeOpenAI + } + mode := strings.ToLower(strings.TrimSpace(c.APIMode)) + if mode == "" { + return ProviderAPIModeOpenAI + } + return mode +} + +// EffectiveBaseURL returns the upstream base URL adjusted for api_mode and provider_key. +func (c *SAiProviderConfig) EffectiveBaseURL(providerKey string) string { + base := c.ResolvedBaseURL() + if base == "" { + base = DefaultPublicBaseURL(providerKey) + } + if base == "" { return "" } - return strings.TrimSpace(c.APIKey) + if c.ResolvedAPIMode() != ProviderAPIModeAnthropic { + return base + } + pk := strings.ToLower(strings.TrimSpace(providerKey)) + if pk != ProviderKeyDeepseek { + return base + } + base = strings.TrimRight(base, "/") + if strings.HasSuffix(strings.ToLower(base), "/anthropic") { + return base + } + return base + "/anthropic" } // String implements gotypes.ISerializable for sqlchemy JSON/compound columns. @@ -60,7 +110,7 @@ func (c *SAiProviderConfig) IsZero() bool { if c == nil { return true } - return c.ResolvedBaseURL() == "" && c.ResolvedAPIKey() == "" + return c.ResolvedBaseURL() == "" && strings.TrimSpace(c.APIMode) == "" } type AiProviderListInput struct { @@ -76,6 +126,8 @@ type AiProviderCreateInput struct { ProviderKey string `json:"provider_key"` Config *SAiProviderConfig `json:"config"` + Secret string `json:"secret"` + ModelKeys []string `json:"model_keys"` LlmDeploymentId string `json:"llm_deployment_id"` LlmId string `json:"llm_id"` } @@ -98,3 +150,25 @@ type AiProviderDetails struct { LlmDeploymentId string `json:"llm_deployment_id"` LlmId string `json:"llm_id"` } + +type AiProviderTestConnectivityInput struct { + ProviderKey string `json:"provider_key"` + Secret string `json:"secret"` + Config *SAiProviderConfig `json:"config"` +} + +const ( + AiProviderModelsSourceUpstream = "upstream" + AiProviderModelsSourceCatalog = "catalog" +) + +type AiProviderUpstreamModel struct { + ModelKey string `json:"model_key"` +} + +type AiProviderTestConnectivityOutput struct { + Ok bool `json:"ok"` + Message string `json:"message"` + ModelsSource string `json:"models_source"` + Models []AiProviderUpstreamModel `json:"models"` +} diff --git a/pkg/apis/aiproxy/ai_provider_config_test.go b/pkg/apis/aiproxy/ai_provider_config_test.go new file mode 100644 index 0000000000..5c4bc6cf60 --- /dev/null +++ b/pkg/apis/aiproxy/ai_provider_config_test.go @@ -0,0 +1,97 @@ +package aiproxy + +import "testing" + +func TestResolvedAPIModeDefault(t *testing.T) { + cfg := &SAiProviderConfig{} + if got := cfg.ResolvedAPIMode(); got != ProviderAPIModeOpenAI { + t.Fatalf("ResolvedAPIMode() = %q, want %q", got, ProviderAPIModeOpenAI) + } +} + +func TestEffectiveBaseURLDeepseekAnthropic(t *testing.T) { + cfg := &SAiProviderConfig{ + BaseURL: "https://api.deepseek.com", + APIMode: ProviderAPIModeAnthropic, + } + got := cfg.EffectiveBaseURL(ProviderKeyDeepseek) + want := "https://api.deepseek.com/anthropic" + if got != want { + t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want) + } +} + +func TestEffectiveBaseURLDeepseekOpenAI(t *testing.T) { + cfg := &SAiProviderConfig{ + BaseURL: "https://api.deepseek.com/anthropic", + APIMode: ProviderAPIModeOpenAI, + } + got := cfg.EffectiveBaseURL(ProviderKeyDeepseek) + want := "https://api.deepseek.com/anthropic" + if got != want { + t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want) + } +} + +func TestSupportsDualAPIMode(t *testing.T) { + if !SupportsDualAPIMode(ProviderKeyDeepseek) { + t.Fatal("deepseek should support dual api mode") + } + if !SupportsDualAPIMode(ProviderKeyCustom) { + t.Fatal("custom should support dual api mode") + } + if SupportsDualAPIMode(ProviderKeyOpenAI) { + t.Fatal("openai should not support dual api mode") + } +} + +func TestIsCustomProviderKey(t *testing.T) { + if !IsCustomProviderKey(ProviderKeyCustom) { + t.Fatal("custom key should match") + } + if IsCustomProviderKey(ProviderKeyOpenAI) { + t.Fatal("openai should not be custom") + } +} + +func TestEffectiveBaseURLFallbackOpenAI(t *testing.T) { + cfg := &SAiProviderConfig{} + got := cfg.EffectiveBaseURL(ProviderKeyOpenAI) + want := "https://api.openai.com" + if got != want { + t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want) + } +} + +func TestEffectiveBaseURLFallbackDeepseekAnthropic(t *testing.T) { + cfg := &SAiProviderConfig{APIMode: ProviderAPIModeAnthropic} + got := cfg.EffectiveBaseURL(ProviderKeyDeepseek) + want := "https://api.deepseek.com/anthropic" + if got != want { + t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want) + } +} + +func TestEffectiveBaseURLCustomNoAnthropicSuffix(t *testing.T) { + cfg := &SAiProviderConfig{ + BaseURL: "https://llm.example.com/v1", + APIMode: ProviderAPIModeAnthropic, + } + got := cfg.EffectiveBaseURL(ProviderKeyCustom) + want := "https://llm.example.com/v1" + if got != want { + t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want) + } +} + +func TestHasDefaultPublicBaseURL(t *testing.T) { + if !HasDefaultPublicBaseURL(ProviderKeyOpenAI) { + t.Fatal("openai should have default base url") + } + if HasDefaultPublicBaseURL(ProviderKeyAzure) { + t.Fatal("azure should not have default base url") + } + if HasDefaultPublicBaseURL(ProviderKeyCustom) { + t.Fatal("custom should not have default base url") + } +} diff --git a/pkg/apis/aiproxy/ai_provider_secret_test.go b/pkg/apis/aiproxy/ai_provider_secret_test.go new file mode 100644 index 0000000000..257cfc2dde --- /dev/null +++ b/pkg/apis/aiproxy/ai_provider_secret_test.go @@ -0,0 +1,37 @@ +package aiproxy + +import ( + "strings" + "testing" + + "yunion.io/x/jsonutils" +) + +func TestSAiProviderConfigRejectsAPIKey(t *testing.T) { + cfg := &SAiProviderConfig{} + obj, err := jsonutils.Parse([]byte(`{"base_url":"https://api.openai.com","api_key":"sk-test"}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + err = obj.Unmarshal(cfg) + if err == nil { + t.Fatal("expected error for config.api_key") + } + if !strings.Contains(err.Error(), "config.api_key is not supported") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestSAiProviderConfigAllowsWithoutAPIKey(t *testing.T) { + cfg := &SAiProviderConfig{} + obj, err := jsonutils.Parse([]byte(`{"base_url":"https://api.openai.com","api_mode":"openai"}`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := obj.Unmarshal(cfg); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.ResolvedBaseURL() != "https://api.openai.com" { + t.Fatalf("base_url = %q", cfg.ResolvedBaseURL()) + } +} diff --git a/pkg/apis/aiproxy/ai_routing_model.go b/pkg/apis/aiproxy/ai_routing_model.go index 752b4068cc..47d7bd6172 100644 --- a/pkg/apis/aiproxy/ai_routing_model.go +++ b/pkg/apis/aiproxy/ai_routing_model.go @@ -52,6 +52,11 @@ type AiRoutingModelUpdateInput struct { type AiRoutingModelDetails struct { apis.StandaloneResourceDetails + // Id and Name are set explicitly for nested routing_models on ai_routing details + // (list responses also merge the full model row, which includes these fields). + Id string `json:"id"` + Name string `json:"name"` + AiRoutingId string `json:"ai_routing_id"` AiProviderId string `json:"ai_provider_id"` AiModelId string `json:"ai_model_id"` diff --git a/pkg/apis/aiproxy/provider_defaults.go b/pkg/apis/aiproxy/provider_defaults.go new file mode 100644 index 0000000000..bc02f30c43 --- /dev/null +++ b/pkg/apis/aiproxy/provider_defaults.go @@ -0,0 +1,51 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import "strings" + +// DefaultPublicBaseURL returns a well-known public API base for catalog providers. +// Empty string means no default (user must set config.base_url). +func DefaultPublicBaseURL(providerKey string) string { + switch strings.ToLower(strings.TrimSpace(providerKey)) { + case ProviderKeyOpenAI: + return "https://api.openai.com" + case ProviderKeyAnthropic: + return "https://api.anthropic.com" + case ProviderKeyAzure, ProviderKeyBedrock, ProviderKeySGLang, ProviderKeyOllama, ProviderKeyVLLM, ProviderKeyAliyun, ProviderKeyBaidu: + return "" + case ProviderKeyDeepseek: + return "https://api.deepseek.com" + case ProviderKeyGemini: + return "https://generativelanguage.googleapis.com/v1beta" + case ProviderKeyGroq: + return "https://api.groq.com/openai" + case ProviderKeyMistral: + return "https://api.mistral.ai" + case ProviderKeyOpenrouter: + return "https://openrouter.ai/api" + case ProviderKeyHuggingface: + return "https://router.huggingface.co" + case ProviderKeyXiaomi: + return "https://api.xiaomimimo.com" + default: + return "" + } +} + +// HasDefaultPublicBaseURL reports whether provider_key has a built-in public base URL. +func HasDefaultPublicBaseURL(providerKey string) bool { + return DefaultPublicBaseURL(providerKey) != "" +} diff --git a/pkg/apis/aiproxy/provider_keys.go b/pkg/apis/aiproxy/provider_keys.go new file mode 100644 index 0000000000..c989363b6d --- /dev/null +++ b/pkg/apis/aiproxy/provider_keys.go @@ -0,0 +1,147 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aiproxy + +import "strings" + +// Built-in catalog provider_key values (seeded at InitDB and registered in providers). +const ( + ProviderKeyAliyun = "aliyun" + ProviderKeyAnthropic = "anthropic" + ProviderKeyAzure = "azure" + ProviderKeyBaidu = "baidu" + ProviderKeyBedrock = "bedrock" + ProviderKeyCerebras = "cerebras" + ProviderKeyCohere = "cohere" + ProviderKeyCustom = "custom" + ProviderKeyDeepseek = "deepseek" + ProviderKeyElevenlabs = "elevenlabs" + ProviderKeyFireworks = "fireworks" + ProviderKeyGemini = "gemini" + ProviderKeyGroq = "groq" + ProviderKeyHuggingface = "huggingface" + ProviderKeyMistral = "mistral" + ProviderKeyNebius = "nebius" + ProviderKeyOllama = "ollama" + ProviderKeyOpenAI = "openai" + ProviderKeyOpenrouter = "openrouter" + ProviderKeyParasail = "parasail" + ProviderKeyPerplexity = "perplexity" + ProviderKeyReplicate = "replicate" + ProviderKeyRunway = "runway" + ProviderKeySGLang = "sglang" + ProviderKeyVertex = "vertex" + ProviderKeyVLLM = "vllm" + ProviderKeyXai = "xai" + ProviderKeyXiaomi = "xiaomi" +) + +// StandardCatalogProviderKeys lists built-in provider_key values seeded at InitDB. +var StandardCatalogProviderKeys = []string{ + ProviderKeyAnthropic, + // ProviderKeyAzure, // uncommon + // ProviderKeyBedrock, // uncommon + // ProviderKeyCerebras, // uncommon + // ProviderKeyCohere, // uncommon + ProviderKeyDeepseek, + ProviderKeyGemini, + ProviderKeyGroq, + ProviderKeyMistral, + ProviderKeyOllama, + ProviderKeyOpenAI, + // ProviderKeyParasail, // uncommon + // ProviderKeyPerplexity, // uncommon + ProviderKeySGLang, + // ProviderKeyVertex, // uncommon + ProviderKeyOpenrouter, + // ProviderKeyElevenlabs, // uncommon + ProviderKeyHuggingface, + // ProviderKeyNebius, // uncommon + // ProviderKeyXai, // uncommon + // ProviderKeyReplicate, // uncommon + ProviderKeyVLLM, + // ProviderKeyRunway, // uncommon + // ProviderKeyFireworks, // uncommon + // ProviderKeyAliyun, // uncommon + // ProviderKeyBaidu, // uncommon + ProviderKeyXiaomi, +} + +// OpenAICompatProviderKeys are catalog keys routed through openai.NewCompat. +var OpenAICompatProviderKeys = []string{ + ProviderKeyOpenAI, + ProviderKeyGroq, + ProviderKeyMistral, + // ProviderKeyCerebras, // uncommon + ProviderKeyDeepseek, + // ProviderKeyPerplexity, // uncommon + ProviderKeyOpenrouter, + // ProviderKeyFireworks, // uncommon + // ProviderKeyNebius, // uncommon + // ProviderKeyXai, // uncommon + // ProviderKeyParasail, // uncommon + ProviderKeySGLang, + ProviderKeyHuggingface, + ProviderKeyOllama, + ProviderKeyXiaomi, +} + +var nativeMessagesAdapterProviderKeys = map[string]struct{}{ + ProviderKeyGemini: {}, + // ProviderKeyCohere: {}, // uncommon + // ProviderKeyBaidu: {}, // uncommon + // ProviderKeyAliyun: {}, // uncommon +} + +// IsNativeMessagesAdapterProvider reports whether provider_key uses a dedicated +// non-OpenAI-chat upstream API that cannot be reached via Anthropic-to-OpenAI translation. +func IsNativeMessagesAdapterProvider(providerKey string) bool { + key := strings.ToLower(strings.TrimSpace(providerKey)) + _, ok := nativeMessagesAdapterProviderKeys[key] + return ok +} + +const ( + ProviderAPIModeOpenAI = "openai" + ProviderAPIModeAnthropic = "anthropic" +) + +// DualAPIProviderKeys lists provider_key values that support openai and anthropic upstream APIs. +var DualAPIProviderKeys = map[string]struct{}{ + ProviderKeyCustom: {}, + ProviderKeyDeepseek: {}, +} + +// IsCustomProviderKey reports whether provider_key is the user-defined custom gateway type. +func IsCustomProviderKey(providerKey string) bool { + return strings.ToLower(strings.TrimSpace(providerKey)) == ProviderKeyCustom +} + +// SupportsDualAPIMode reports whether provider_key may use config.api_mode. +func SupportsDualAPIMode(providerKey string) bool { + key := strings.ToLower(strings.TrimSpace(providerKey)) + _, ok := DualAPIProviderKeys[key] + return ok +} + +// IsValidProviderAPIMode reports whether mode is a supported api_mode value. +func IsValidProviderAPIMode(mode string) bool { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "", ProviderAPIModeOpenAI, ProviderAPIModeAnthropic: + return true + default: + return false + } +} diff --git a/pkg/llm/models/llm_aiproxy_sync.go b/pkg/llm/models/llm_aiproxy_sync.go index 4388a3f986..2f1afc1bc3 100644 --- a/pkg/llm/models/llm_aiproxy_sync.go +++ b/pkg/llm/models/llm_aiproxy_sync.go @@ -21,8 +21,6 @@ import ( "yunion.io/x/pkg/util/printutils" ) -const aiproxyPlaceholderAPIKey = "unused" - func aiproxyAdminSession(ctx context.Context) *mcclient.ClientSession { return auth.GetAdminSession(ctx, options.Options.Region) } @@ -34,7 +32,7 @@ func mapLLMTypeToProviderKey(llmType string) (string, bool) { case string(api.LLM_CONTAINER_OLLAMA): return "ollama", true case string(api.LLM_CONTAINER_SGLANG): - return "sgl", true + return "sglang", true default: return "", false } @@ -300,7 +298,6 @@ func upsertAiProvider( } cfg := jsonutils.Marshal(&apapi.SAiProviderConfig{ BaseURL: baseURL, - APIKey: aiproxyPlaceholderAPIKey, }) params := jsonutils.NewDict() params.Set("provider_key", jsonutils.NewString(providerKey)) diff --git a/pkg/llm/models/llm_aiproxy_sync_test.go b/pkg/llm/models/llm_aiproxy_sync_test.go deleted file mode 100644 index 6aabf57cc6..0000000000 --- a/pkg/llm/models/llm_aiproxy_sync_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package models - -import ( - "testing" - - api "yunion.io/x/onecloud/pkg/apis/llm" -) - -func TestMapLLMTypeToProviderKey(t *testing.T) { - cases := []struct { - in string - key string - ok bool - }{ - {string(api.LLM_CONTAINER_VLLM), "vllm", true}, - {string(api.LLM_CONTAINER_OLLAMA), "ollama", true}, - {string(api.LLM_CONTAINER_SGLANG), "sgl", true}, - {"dify", "", false}, - } - for _, c := range cases { - key, ok := mapLLMTypeToProviderKey(c.in) - if ok != c.ok || key != c.key { - t.Fatalf("mapLLMTypeToProviderKey(%q) = (%q, %v), want (%q, %v)", c.in, key, ok, c.key, c.ok) - } - } -} - -func TestSlugModelKey(t *testing.T) { - if got := slugModelKey("Qwen/Qwen2.5-7B-Instruct"); got != "qwen-qwen2-5-7b-instruct" { - t.Fatalf("slugModelKey got %q", got) - } -} - -func TestDeploymentClientModelAlias(t *testing.T) { - dep := &SLLMDeployment{} - dep.Name = "my-qwen" - if got := deploymentClientModelAlias(dep, "Qwen3-0.6B"); got != "my-qwen-Qwen3-0.6B" { - t.Fatalf("deploymentClientModelAlias got %q", got) - } - depEmpty := &SLLMDeployment{} - depEmpty.Id = "dep-id-1" - if got := deploymentClientModelAlias(depEmpty, ""); got != "dep-id-1" { - t.Fatalf("deploymentClientModelAlias without model_key got %q", got) - } -} - -func TestDeploymentRoutingModelKey(t *testing.T) { - dep := &SLLMDeployment{} - dep.Name = "my-qwen" - if got := deploymentRoutingModelKey(dep, "Qwen3-0.6B"); got != "my-qwen-Qwen3-0.6B" { - t.Fatalf("deploymentRoutingModelKey got %q", got) - } -} - -func TestAiproxyResourceNames(t *testing.T) { - dep := &SLLMDeployment{} - dep.Name = "My-Qwen" - dep.Id = "dep-id-1" - if got := aiRoutingNameForDeployment(dep); got != "llm-dep-my-qwen" { - t.Fatalf("aiRoutingNameForDeployment got %q", got) - } - - depEmpty := &SLLMDeployment{} - depEmpty.Id = "dep-id-2" - if got := aiRoutingNameForDeployment(depEmpty); got != "llm-dep-dep-id-2" { - t.Fatalf("aiRoutingNameForDeployment empty name got %q", got) - } - - llm := &SLLM{} - llm.Name = "my-qwen-0" - llm.Id = "llm-id-1" - if got := aiProviderNameForLlm(llm); got != "llm-my-qwen-0" { - t.Fatalf("aiProviderNameForLlm got %q", got) - } - - llmEmpty := &SLLM{} - llmEmpty.Id = "llm-id-2" - if got := aiProviderNameForLlm(llmEmpty); got != "llm-llm-id-2" { - t.Fatalf("aiProviderNameForLlm empty name got %q", got) - } - - if got := aiModelNameForLlm(llm, "Qwen/Qwen3-0.6B"); got != "llm-my-qwen-0-qwen-qwen3-0-6b" { - t.Fatalf("aiModelNameForLlm got %q", got) - } -} - -func TestClearDeploymentAiproxyRegistrationState(t *testing.T) { - dep := &SLLMDeployment{} - dep.AutoRegisterAiproxy = true - dep.AiproxyRoutingId = "routing-1" - dep.AiproxyBindings = &api.AiproxyBindings{{LlmId: "llm-1"}} - dep.AiproxySyncStatus = api.AIPROXY_SYNC_STATUS_SYNCED - - clearDeploymentAiproxyRegistrationState(dep) - - if dep.AutoRegisterAiproxy { - t.Fatal("AutoRegisterAiproxy should be false") - } - if dep.AiproxyRoutingId != "" { - t.Fatalf("AiproxyRoutingId should be empty, got %q", dep.AiproxyRoutingId) - } - if dep.AiproxyBindings != nil { - t.Fatal("AiproxyBindings should be nil") - } - if dep.AiproxySyncStatus != api.AIPROXY_SYNC_STATUS_DISABLED { - t.Fatalf("AiproxySyncStatus should be disabled, got %q", dep.AiproxySyncStatus) - } -} - -func TestResolveAiproxySyncStatusAfterReconcile(t *testing.T) { - cases := []struct { - name string - result aiproxyBindingSyncResult - wantStat string - }{ - { - name: "fully synced", - result: aiproxyBindingSyncSynced, - wantStat: api.AIPROXY_SYNC_STATUS_SYNCED, - }, - { - name: "binding partial failure", - result: aiproxyBindingSyncPartial, - wantStat: api.AIPROXY_SYNC_STATUS_PARTIAL, - }, - { - name: "all bindings failed", - result: aiproxyBindingSyncFailed, - wantStat: api.AIPROXY_SYNC_STATUS_FAILED, - }, - { - name: "pending", - result: aiproxyBindingSyncPending, - wantStat: api.AIPROXY_SYNC_STATUS_PENDING, - }, - } - for _, c := range cases { - got := resolveAiproxySyncStatusAfterReconcile(c.result) - if got != c.wantStat { - t.Fatalf("%s: got %q want %q", c.name, got, c.wantStat) - } - } -} - -func TestAiproxySyncFailureReason(t *testing.T) { - dep := &SLLMDeployment{} - dep.AiproxyBindings = &api.AiproxyBindings{ - {LlmId: "llm-1", SyncStatus: api.AIPROXY_BINDING_SYNC_SYNCED}, - {LlmId: "llm-2", SyncStatus: api.AIPROXY_BINDING_SYNC_FAILED, LastError: "provider upsert failed"}, - } - got := AiproxySyncFailureReason(dep) - want := "llm llm-2: provider upsert failed" - if got != want { - t.Fatalf("AiproxySyncFailureReason() = %q, want %q", got, want) - } - - msg := aiproxySyncStatusMessage(dep, aiproxyBindingSyncFailed) - if msg != want { - t.Fatalf("aiproxySyncStatusMessage(failed) = %q, want %q", msg, want) - } -} - -func TestUpstreamModelKeyForBackend(t *testing.T) { - cases := []struct { - llmType string - modelName string - modelTag string - want string - }{ - {string(api.LLM_CONTAINER_VLLM), "Qwen/Qwen3-0.6B", "main", "Qwen3-0.6B"}, - {string(api.LLM_CONTAINER_SGLANG), "Qwen/Qwen2.5-7B-Instruct", "main", "Qwen2.5-7B-Instruct"}, - {string(api.LLM_CONTAINER_VLLM), "Qwen3-0.6B", "main", "Qwen3-0.6B"}, - {string(api.LLM_CONTAINER_OLLAMA), "qwen3", "8b", "qwen3:8b"}, - {string(api.LLM_CONTAINER_OLLAMA), "qwen3", "", "qwen3"}, - } - for _, c := range cases { - got := upstreamModelKeyForBackend(c.llmType, c.modelName, c.modelTag) - if got != c.want { - t.Fatalf("upstreamModelKeyForBackend(%q, %q, %q) = %q, want %q", - c.llmType, c.modelName, c.modelTag, got, c.want) - } - } -} diff --git a/pkg/mcclient/options/aiproxy/resources.go b/pkg/mcclient/options/aiproxy/resources.go index c3099fc024..8f28f2cee4 100644 --- a/pkg/mcclient/options/aiproxy/resources.go +++ b/pkg/mcclient/options/aiproxy/resources.go @@ -71,6 +71,21 @@ func (o *AiProviderCreateOptions) Params() (jsonutils.JSONObject, error) { return params, nil } +type AiProviderTestConnectivityOptions struct { + ProviderKey string `help:"provider key" json:"provider_key"` + Secret string `help:"API secret" json:"secret"` + Config string `help:"provider config as JSON object string" json:"-"` +} + +func (o *AiProviderTestConnectivityOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(o).(*jsonutils.JSONDict) + params.Remove("config") + if err := mergeJSONStringField(params, "config", o.Config); err != nil { + return nil, err + } + return params, nil +} + type AiProviderUpdateOptions struct { ID string `help:"ID or name" json:"-"` Name string `json:"name,omitempty"` diff --git a/scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh b/scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh deleted file mode 100755 index fe960b276c..0000000000 --- a/scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash -# Test climc ai-provider-create: create a custom ai_provider and verify with show/list. -# -# Usage: -# source /etc/yunion/rcadmin -# bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh -# -# Non-interactive: -# export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1 -# export AIPROXY_PROVIDER_FT_NAME=my-custom-provider -# export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-custom-key -# export AIPROXY_PROVIDER_FT_BASE_URL=https://api.example.com/v1 -# bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh -# -# Or pass full config JSON: -# export AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"https://api.example.com"}' - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/test/aiproxy/aiproxy-functional-test-common.sh -source "${SCRIPT_DIR}/aiproxy-functional-test-common.sh" - -CLIMC_OUTPUT_FORMAT="${CLIMC_OUTPUT_FORMAT:-json}" -export CLIMC_OUTPUT_FORMAT - -aiproxy_ft_need_cmds - -PROVIDER_RESOURCE_NAME="${AIPROXY_PROVIDER_FT_NAME:-}" -PROVIDER_KEY="${AIPROXY_PROVIDER_FT_PROVIDER_KEY:-}" -BASE_URL="${AIPROXY_PROVIDER_FT_BASE_URL:-}" -CONFIG_JSON="${AIPROXY_PROVIDER_FT_CONFIG:-}" -ENABLED_FLAG="${AIPROXY_PROVIDER_FT_ENABLED:-true}" -DELETE_IF_EXISTS="${AIPROXY_PROVIDER_FT_DELETE_EXISTING:-}" - -prompt_line() { - local prompt="$1" default="${2:-}" varname="$3" - local value - if [[ -n "$default" ]]; then - echo -n "${prompt} [${default}]: " >&2 - else - echo -n "${prompt}: " >&2 - fi - if [[ ! -t 0 ]]; then - value="$default" - else - read -r value &2 - read -r ans &2 - echo "将创建自定义 ai_provider（provider_key 不可与 catalog 重复）。" >&2 - echo - - if [[ -z "$PROVIDER_RESOURCE_NAME" ]]; then - prompt_line "资源名称 (climc 第一个参数 NAME)" "aiproxy-provider-ft-${suffix}" PROVIDER_RESOURCE_NAME - fi - if [[ -z "$PROVIDER_KEY" ]]; then - prompt_line "provider_key (唯一标识)" "${PROVIDER_RESOURCE_NAME}" PROVIDER_KEY - fi - if [[ -z "$CONFIG_JSON" && -z "$BASE_URL" ]]; then - prompt_line "config.base_url (OpenAI 兼容上游)" "https://api.openai.com" BASE_URL - fi - if prompt_yes_no "创建后启用 (--enabled)?" 1; then - ENABLED_FLAG=true - else - ENABLED_FLAG=false - fi -} - -delete_existing_provider() { - local name="$1" - if ! climc_json ai-provider-show "$name" >/dev/null 2>&1; then - return 0 - fi - if [[ "$DELETE_IF_EXISTS" != "1" ]]; then - die "ai_provider $name already exists; set AIPROXY_PROVIDER_FT_DELETE_EXISTING=1 to delete first" - fi - echo "deleting existing ai_provider $name" >&2 - climc ai-provider-delete "$name" -} - -create_provider() { - local config enabled_args=() - config="$(build_config_json)" - if [[ "$ENABLED_FLAG" == "true" ]]; then - enabled_args=(--enabled) - fi - climc ai-provider-create \ - "$PROVIDER_RESOURCE_NAME" \ - --provider-key "$PROVIDER_KEY" \ - --config "$config" \ - "${enabled_args[@]}" -} - -verify_provider() { - local row pk base enabled - row="$(climc_json ai-provider-show "$PROVIDER_RESOURCE_NAME")" - pk="$(echo "$row" | jq -r '.provider_key // empty')" - base="$(echo "$row" | jq -r '.config.base_url // empty')" - enabled="$(echo "$row" | jq -r '.enabled // false')" - [[ "$pk" == "$PROVIDER_KEY" ]] || die "provider_key mismatch: got $pk want $PROVIDER_KEY" - if [[ -n "$BASE_URL" ]]; then - [[ "$base" == "$BASE_URL" ]] || die "base_url mismatch: got $base want $BASE_URL" - fi - [[ "$enabled" == "true" ]] || [[ "$ENABLED_FLAG" != "true" ]] || die "expected enabled=true" - echo "$row" | jq '{id, name, provider_key, enabled, config}' -} - -collect_inputs -delete_existing_provider "$PROVIDER_RESOURCE_NAME" - -aiproxy_ft_step "create ai_provider" -create_provider - -aiproxy_ft_step "verify ai-provider-show" -verify_provider - -aiproxy_ft_step "verify ai-provider-list filter" -cnt="$(climc_json ai-provider-list --provider-key "$PROVIDER_KEY" \ - | jq --arg n "$PROVIDER_RESOURCE_NAME" '[.data[] | select(.name == $n)] | length')" -[[ "$cnt" -ge 1 ]] || die "ai-provider-list --provider-key did not return created row" - -echo -echo "OK: ai_provider create test passed." -echo " name: $PROVIDER_RESOURCE_NAME" -echo " provider_key: $PROVIDER_KEY" -echo "Cleanup:" -echo " climc ai-provider-delete $PROVIDER_RESOURCE_NAME" diff --git a/scripts/test/aiproxy/aiproxy-functional-test-common.sh b/scripts/test/aiproxy/aiproxy-functional-test-common.sh deleted file mode 100755 index 5bd3df0c9d..0000000000 --- a/scripts/test/aiproxy/aiproxy-functional-test-common.sh +++ /dev/null @@ -1,396 +0,0 @@ -# Shared helpers for aiproxy functional test scripts (source only, do not execute directly). - -die() { echo "ERROR: $*" >&2; exit 1; } - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || die "missing command: $1" -} - -aiproxy_ft_need_cmds() { - need_cmd climc - need_cmd curl - need_cmd jq -} - -climc_json() { - climc --output-format json "$@" -} - -aiproxy_ft_step() { echo; echo "==> $*"; } - -catalog_model_id() { - local provider="$1" model_key="$2" - echo "${provider}-${model_key}" -} - -default_model_for_provider() { - case "$1" in - aliyun) echo "qwen-turbo" ;; - xiaomi) echo "mimo-v2-flash" ;; - openai) echo "gpt-4o-mini" ;; - *) echo "" ;; - esac -} - -default_prompt_for_provider() { - case "$1" in - aliyun) echo "用一句话介绍通义千问" ;; - xiaomi) echo "用一句话介绍小米 MiMo" ;; - *) echo "用一句话介绍这个模型" ;; - esac -} - -resolve_aiproxy_url() { - if [[ -n "${AIPROXY_URL:-}" ]]; then - echo "${AIPROXY_URL%/}" - return - fi - local url - url="$(climc_json endpoint-list --service aiproxy --interface public --limit 1 \ - | jq -r '.data[0].url // empty')" - [[ -n "$url" ]] || die "cannot resolve aiproxy public URL; set AIPROXY_URL" - echo "${url%/}" -} - -list_catalog_provider_keys() { - climc_json ai-provider-list --limit 500 \ - | jq -r '.data[] | .provider_key // empty' | sed '/^$/d' | sort -u -} - -list_catalog_model_keys() { - local provider_key="$1" - climc_json ai-model-list --ai-provider-id "$provider_key" --limit 500 \ - | jq -r '.data[] | .model_key // empty' \ - | sed '/^$/d' | grep -vx 'default' | sort -u -} - -# Resolve API key from env (generic or provider-specific legacy names). -resolve_api_key_from_env() { - local provider_key="$1" - if [[ -n "${AIPROXY_FT_API_KEY:-}" ]]; then - echo "$AIPROXY_FT_API_KEY" - return - fi - case "$provider_key" in - aliyun) - [[ -n "${DASHSCOPE_API_KEY:-}" ]] && echo "$DASHSCOPE_API_KEY" && return - ;; - xiaomi) - [[ -n "${MIMO_API_KEY:-}" ]] && echo "$MIMO_API_KEY" && return - ;; - esac - return 1 -} - -prompt_api_key() { - local provider_key="$1" key - if key="$(resolve_api_key_from_env "$provider_key")"; then - echo "使用环境变量中的 API Key（未回显）" >&2 - echo "$key" - return - fi - if [[ ! -t 0 ]]; then - die "未设置 API Key：export AIPROXY_FT_API_KEY 或 ${provider_key} 对应的环境变量，或使用交互式终端" - fi - echo -n "请输入 ${provider_key} 的 API Key（不回显）: " >&2 - read -r -s key &2 - [[ -n "$key" ]] || die "API Key 不能为空" - echo "$key" -} - -# Populates global AIPROXY_FT_LINES[] (bash 3.2 compatible). -_load_lines_into_array() { - AIPROXY_FT_LINES=() - while IFS= read -r line; do - [[ -n "$line" ]] && AIPROXY_FT_LINES+=("$line") - done -} - -prompt_select_provider() { - local -a keys=() - local k i choice - _load_lines_into_array < <(list_catalog_provider_keys) - keys=("${AIPROXY_FT_LINES[@]}") - [[ ${#keys[@]} -gt 0 ]] || die "catalog 中无 ai_provider，请先执行 aiproxy master InitDB" - - if [[ -n "${AIPROXY_FT_PROVIDER:-}" ]]; then - for k in "${keys[@]}"; do - [[ "$k" == "${AIPROXY_FT_PROVIDER}" ]] && echo "${AIPROXY_FT_PROVIDER}" && return - done - die "ai_provider ${AIPROXY_FT_PROVIDER} 不在 catalog 中" - fi - - if [[ ! -t 0 ]]; then - die "请设置 AIPROXY_FT_PROVIDER 或在交互式终端运行" - fi - - echo "可用模型提供商 (catalog):" >&2 - for i in "${!keys[@]}"; do - printf ' [%d] %s\n' "$((i + 1))" "${keys[$i]}" >&2 - done - while true; do - echo -n "请选择序号 [1-${#keys[@]}] 或直接输入 provider_key: " >&2 - read -r choice = 1 && choice <= ${#keys[@]})); then - echo "${keys[$((choice - 1))]}" - return - fi - for k in "${keys[@]}"; do - [[ "$k" == "$choice" ]] && echo "$choice" && return - done - echo "无效选择，请重试。" >&2 - done -} - -prompt_select_model() { - local provider_key="$1" - local -a models=() - local m i choice default_m found - _load_lines_into_array < <(list_catalog_model_keys "$provider_key") - models=("${AIPROXY_FT_LINES[@]}") - [[ ${#models[@]} -gt 0 ]] || die "provider ${provider_key} 下无可用 model_key（catalog 未 seed？）" - - if [[ -n "${AIPROXY_FT_MODEL:-}" ]]; then - for m in "${models[@]}"; do - [[ "$m" == "${AIPROXY_FT_MODEL}" ]] && echo "${AIPROXY_FT_MODEL}" && return - done - die "model_key ${AIPROXY_FT_MODEL} 不在 provider ${provider_key} 的 catalog 中" - fi - - default_m="$(default_model_for_provider "$provider_key")" - found=0 - if [[ -n "$default_m" ]]; then - for m in "${models[@]}"; do - if [[ "$m" == "$default_m" ]]; then - found=1 - break - fi - done - fi - [[ "$found" -eq 1 ]] || default_m="${models[0]}" - - if [[ ! -t 0 ]]; then - echo "$default_m" - return - fi - - echo "提供商 ${provider_key} 的模型:" >&2 - for i in "${!models[@]}"; do - printf ' [%d] %s\n' "$((i + 1))" "${models[$i]}" >&2 - done - while true; do - echo -n "请选择序号 [1-${#models[@]}] 或输入 model_key [默认: ${default_m}]: " >&2 - read -r choice = 1 && choice <= ${#models[@]})); then - echo "${models[$((choice - 1))]}" - return - fi - for m in "${models[@]}"; do - [[ "$m" == "$choice" ]] && echo "$choice" && return - done - echo "无效选择，请重试。" >&2 - done -} - -prompt_run_stream() { - if [[ "${AIPROXY_FT_SKIP_STREAM:-}" == "1" ]]; then - return 1 - fi - if [[ "${AIPROXY_FT_SKIP_STREAM:-}" == "0" ]]; then - return 0 - fi - if [[ ! -t 0 ]]; then - return 0 - fi - local ans - echo -n "是否执行流式测试 (stream=true)? [Y/n]: " >&2 - read -r ans /dev/null 2>&1; then - echo "ai_key $key_name exists, syncing secret and ai_provider_id" - climc ai-key-update "$key_name" \ - --ai-provider-id "$provider_id" \ - --secret "$api_secret" \ - --weight 10 - else - climc ai-key-create \ - "$key_name" \ - --ai-provider-id "$provider_id" \ - --secret "$api_secret" \ - --weight 10 \ - --enabled - fi - ensure_ai_key_enabled "$key_name" -} - -verify_ai_key_for_provider() { - local provider_key="$1" - local provider_id count - provider_id="$(climc_json ai-provider-show "$provider_key" | jq -r '.id // empty')" - [[ -n "$provider_id" ]] || die "ai_provider $provider_key not found" - count="$(climc_json ai-key-list --ai-provider-id "$provider_id" | jq '[.data[] | select(.enabled == true)] | length')" - [[ "$count" -gt 0 ]] || die "no enabled ai_key bound to ai_provider_id=$provider_id" - echo "enabled ai_key rows for $provider_id: $count" -} - -ensure_virtual_key() { - local vk_name="$1" - if climc_json ai-virtual-key-show "$vk_name" >/dev/null 2>&1; then - echo "virtual key $vk_name already exists" - return - fi - climc ai-virtual-key-create "$vk_name" -} - -ensure_routing() { - local routing_name="$1" provider_key="$2" catalog_model_id="$3" - if climc_json ai-routing-show "$routing_name" >/dev/null 2>&1; then - echo "routing $routing_name already exists" - return - fi - climc ai-routing-create \ - "$routing_name" \ - --priority 10 \ - --models "[{\"ai_provider_id\":\"${provider_key}\",\"ai_model_id\":\"${catalog_model_id}\",\"priority\":1}]" -} - -verify_stream_chat() { - local base_url="$1" vk="$2" model="$3" out="$4" prompt="$5" - local http_code aggregated delta payload - - http_code="$(curl -k -sS -N -o "$out" -w '%{http_code}' \ - "${base_url%/}/openai/v1/chat/completions" \ - -H "Authorization: Bearer ${vk}" \ - -H "Content-Type: application/json" \ - -d "{\"model\":\"${model}\",\"stream\":true,\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rn --arg t "$prompt" '$t')}],\"max_tokens\":64}")" - - echo "HTTP $http_code (stream)" - [[ "$http_code" == "200" ]] || { - echo "--- stream body (first 40 lines) ---" >&2 - head -n 40 "$out" >&2 || true - die "stream chat failed with HTTP $http_code" - } - - aggregated="" - while IFS= read -r line || [[ -n "$line" ]]; do - [[ "$line" == data:* ]] || continue - payload="${line#data: }" - payload="${payload//$'\r'/}" - [[ -z "$payload" ]] && continue - [[ "$payload" == "[DONE]" ]] && continue - if echo "$payload" | jq -e '.error // .message' >/dev/null 2>&1; then - echo "upstream error chunk: $payload" >&2 - die "stream returned error event" - fi - delta="$(echo "$payload" | jq -r '.choices[0].delta.content // empty' 2>/dev/null || true)" - aggregated+="$delta" - done <"$out" - - [[ -n "$aggregated" ]] || { - echo "--- stream body ---" >&2 - cat "$out" >&2 - die "empty aggregated stream content (no choices[0].delta.content)" - } - - echo "stream content (${#aggregated} chars): ${aggregated:0:120}..." -} - -# aiproxy_ft_run executes the full functional test for one provider/model/api key. -aiproxy_ft_run() { - local provider_key="$1" chat_model="$2" api_secret="$3" chat_prompt="$4" - local run_stream="${5:-1}" - - local key_name vk_name routing_name catalog_mid - local chat_resp chat_stream_resp aiproxy_url vk http_code content - - key_name="${AIPROXY_FT_KEY_NAME:-aiproxy-ft-${provider_key}}" - vk_name="${AIPROXY_FT_VK_NAME:-aiproxy-ft-${provider_key}-vk}" - routing_name="${AIPROXY_FT_ROUTING_NAME:-aiproxy-ft-${provider_key}-routing}" - chat_resp="${AIPROXY_FT_CHAT_RESP:-/tmp/aiproxy-ft-${provider_key}-chat.json}" - chat_stream_resp="${AIPROXY_FT_STREAM_RESP:-/tmp/aiproxy-ft-${provider_key}-chat-stream.sse}" - catalog_mid="$(catalog_model_id "$provider_key" "$chat_model")" - - echo - echo "=== aiproxy 功能测试 ===" - echo "provider: ${provider_key} model: ${chat_model} catalog_id: ${catalog_mid}" - echo "ai_key: ${key_name} virtual_key: ${vk_name} routing: ${routing_name}" - echo - - aiproxy_ft_step "1. Keystone aiproxy public endpoint" - aiproxy_url="$(resolve_aiproxy_url)" - echo "AIPROXY_URL=$aiproxy_url" - - aiproxy_ft_step "2. Catalog ${provider_key} / ${chat_model}" - climc ai-provider-show "$provider_key" >/dev/null \ - || die "ai_provider $provider_key missing; run aiproxy master InitDB first" - climc ai-model-show "$catalog_mid" >/dev/null \ - || die "ai_model $catalog_mid not in catalog (re-run aiproxy master InitDB)" - - aiproxy_ft_step "3. ai_key" - ensure_ai_key "$provider_key" "$key_name" "$api_secret" - verify_ai_key_for_provider "$provider_key" - - aiproxy_ft_step "4. ai_virtual_key" - ensure_virtual_key "$vk_name" - vk="$(climc_json ai-virtual-key-show "$vk_name" | jq -r '.virtual_key')" - [[ -n "$vk" ]] || die "empty virtual_key from ai-virtual-key-show" - echo "virtual_key=${vk:0:12}..." - - aiproxy_ft_step "5. ai_routing" - ensure_routing "$routing_name" "$provider_key" "$catalog_mid" - - aiproxy_ft_step "6. POST /openai/v1/chat/completions" - http_code="$(curl -k -sS -o "$chat_resp" -w '%{http_code}' \ - "${aiproxy_url}/openai/v1/chat/completions" \ - -H "Authorization: Bearer ${vk}" \ - -H "Content-Type: application/json" \ - -d "{\"model\":\"${chat_model}\",\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rn --arg t "$chat_prompt" '$t')}],\"max_tokens\":128}")" - - echo "HTTP $http_code" - jq . <"$chat_resp" - [[ "$http_code" == "200" ]] || die "chat request failed with HTTP $http_code" - - content="$(jq -r '.choices[0].message.content // empty' <"$chat_resp")" - [[ -n "$content" ]] || die "empty choices[0].message.content" - - if [[ "$run_stream" == "1" ]]; then - aiproxy_ft_step "7. POST /openai/v1/chat/completions (stream=true)" - verify_stream_chat "$aiproxy_url" "$vk" "$chat_model" "$chat_stream_resp" "$chat_prompt" - fi - - echo - echo "OK: aiproxy functional test passed for ${provider_key}/${chat_model} (non-stream$([[ "$run_stream" == "1" ]] && echo ' + stream' || echo ''))." - echo "Cleanup (optional):" - echo " climc ai-routing-delete $routing_name" - echo " climc ai-virtual-key-delete $vk_name" - echo " climc ai-key-delete $key_name" -} diff --git a/scripts/test/aiproxy/aiproxy-functional-test-mimo.sh b/scripts/test/aiproxy/aiproxy-functional-test-mimo.sh deleted file mode 100755 index 91dbd16fba..0000000000 --- a/scripts/test/aiproxy/aiproxy-functional-test-mimo.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# Legacy entry: pre-select xiaomi provider, then run interactive / env-based test. -# Prefer: bash scripts/test/aiproxy/aiproxy-functional-test.sh - -set -euo pipefail - -export AIPROXY_FT_PROVIDER="${AIPROXY_FT_PROVIDER:-xiaomi}" -[[ -n "${MIMO_API_KEY:-}" && -z "${AIPROXY_FT_API_KEY:-}" ]] && export AIPROXY_FT_API_KEY="${MIMO_API_KEY}" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec bash "${SCRIPT_DIR}/aiproxy-functional-test.sh" "$@" diff --git a/scripts/test/aiproxy/aiproxy-functional-test-qwen.sh b/scripts/test/aiproxy/aiproxy-functional-test-qwen.sh deleted file mode 100755 index 7cb818fcd6..0000000000 --- a/scripts/test/aiproxy/aiproxy-functional-test-qwen.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -# Legacy entry: pre-select aliyun (通义千问) provider, then run interactive / env-based test. -# Prefer: bash scripts/test/aiproxy/aiproxy-functional-test.sh - -set -euo pipefail - -export AIPROXY_FT_PROVIDER="${AIPROXY_FT_PROVIDER:-aliyun}" -[[ -n "${DASHSCOPE_API_KEY:-}" && -z "${AIPROXY_FT_API_KEY:-}" ]] && export AIPROXY_FT_API_KEY="${DASHSCOPE_API_KEY}" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec bash "${SCRIPT_DIR}/aiproxy-functional-test.sh" "$@" diff --git a/scripts/test/aiproxy/aiproxy-functional-test.sh b/scripts/test/aiproxy/aiproxy-functional-test.sh deleted file mode 100755 index 6dc42411f8..0000000000 --- a/scripts/test/aiproxy/aiproxy-functional-test.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# aiproxy interactive functional test: select catalog provider/model, enter API key, run chat + stream. -# -# Usage: -# source /etc/yunion/rcadmin -# bash scripts/test/aiproxy/aiproxy-functional-test.sh -# -# Non-interactive (CI): -# export AIPROXY_FT_PROVIDER=aliyun -# export AIPROXY_FT_MODEL=qwen-turbo -# export AIPROXY_FT_API_KEY='...' -# export AIPROXY_FT_NONINTERACTIVE=1 -# bash scripts/test/aiproxy/aiproxy-functional-test.sh -# -# Legacy env (still supported): -# DASHSCOPE_API_KEY + AIPROXY_FT_PROVIDER=aliyun -# MIMO_API_KEY + AIPROXY_FT_PROVIDER=xiaomi - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/test/aiproxy/aiproxy-functional-test-common.sh -source "${SCRIPT_DIR}/aiproxy-functional-test-common.sh" - -CLIMC_OUTPUT_FORMAT="${CLIMC_OUTPUT_FORMAT:-json}" -export CLIMC_OUTPUT_FORMAT - -aiproxy_ft_need_cmds - -if [[ "${AIPROXY_FT_NONINTERACTIVE:-}" == "1" && -z "${AIPROXY_FT_PROVIDER:-}" ]]; then - die "AIPROXY_FT_NONINTERACTIVE=1 时需设置 AIPROXY_FT_PROVIDER" -fi - -PROVIDER_KEY="$(prompt_select_provider)" -CHAT_MODEL="$(prompt_select_model "$PROVIDER_KEY")" -API_SECRET="$(prompt_api_key "$PROVIDER_KEY")" -CHAT_PROMPT="${AIPROXY_FT_PROMPT:-$(default_prompt_for_provider "$PROVIDER_KEY")}" - -RUN_STREAM=1 -if prompt_run_stream; then - RUN_STREAM=1 -else - RUN_STREAM=0 -fi - -aiproxy_ft_run "$PROVIDER_KEY" "$CHAT_MODEL" "$API_SECRET" "$CHAT_PROMPT" "$RUN_STREAM"