mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-09-01 15:07:17 +08:00
feat(aiproxy): add Anthropic Messages API and migrate functional tests to Go (#25107)
Add /v1/messages handler with Anthropic-to-OpenAI translation, upstream failover, and probe endpoints. Replace shell-based functional test scripts with pkg/aiproxy/ft and climc test commands; consolidate documentation.
This commit is contained in:
Zexi Li
@@ -27,5 +27,6 @@ func init() {
|
||||
cmd.Show(new(apoptions.AiProviderShowOptions))
|
||||
cmd.Update(new(apoptions.AiProviderUpdateOptions))
|
||||
cmd.Delete(new(apoptions.AiProviderDeleteOptions))
|
||||
cmd.PerformClass("test-connectivity", new(apoptions.AiProviderTestConnectivityOptions))
|
||||
registerEnableDisable(cmd)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package aiproxy
|
||||
|
||||
import (
|
||||
"yunion.io/x/onecloud/cmd/climc/shell"
|
||||
)
|
||||
|
||||
var R = shell.R
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package aiproxy
|
||||
|
||||
import (
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/ft"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func init() {
|
||||
R(&ft.AnthropicOptions{}, "aiproxy-test-anthropic", "Run aiproxy Anthropic Messages API E2E test", func(s *mcclient.ClientSession, args *ft.AnthropicOptions) error {
|
||||
return ft.RunAnthropicTest(s, args)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package aiproxy
|
||||
|
||||
import (
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/ft"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func init() {
|
||||
R(&ft.ChatOptions{}, "aiproxy-test-chat", "Run aiproxy OpenAI chat E2E test", func(s *mcclient.ClientSession, args *ft.ChatOptions) error {
|
||||
return ft.RunChatTest(s, args)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package aiproxy
|
||||
|
||||
import (
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/ft"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func init() {
|
||||
R(&ft.ProviderCreateOptions{}, "aiproxy-test-provider-create", "Create custom ai_provider and verify", func(s *mcclient.ClientSession, args *ft.ProviderCreateOptions) error {
|
||||
return ft.RunProviderCreateTest(s, args)
|
||||
})
|
||||
}
|
||||
@@ -1,114 +0,0 @@
|
||||
# aiproxy 功能测试（climc + 小米 MiMo）
|
||||
|
||||
本文用 **climc** 配置 aiproxy 资源，并通过 **curl** 调用 `POST /v1/chat/completions` 验证 **xiaomi** catalog（`api.xiaomimimo.com`）。
|
||||
|
||||
> **安全**：请勿将 MiMo API Key 写入脚本或提交到 Git。使用环境变量 `MIMO_API_KEY`。若 Key 曾泄露，请到小米开放平台轮换。
|
||||
|
||||
通义千问（DashScope）测试见 [functional-test-climc.md](./functional-test-climc.md)。
|
||||
|
||||
## 前置条件
|
||||
|
||||
| 项 | 说明 |
|
||||
|----|------|
|
||||
| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` public endpoint |
|
||||
| 数据库 | 主节点已执行 `InitDB`，catalog 含 `xiaomi` 及 `mimo-*` 模型 |
|
||||
| 客户端 | 已 `source /etc/yunion/rcadmin`，`climc` 可用 |
|
||||
| 工具 | `jq` |
|
||||
| 网络 | aiproxy 节点能访问 `https://api.xiaomimimo.com` |
|
||||
|
||||
```bash
|
||||
source /etc/yunion/rcadmin
|
||||
export CLIMC_OUTPUT_FORMAT=json
|
||||
bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
# 交互菜单中选择 xiaomi，输入 MiMo API Key
|
||||
```
|
||||
|
||||
或快捷入口（默认选中 `xiaomi`）：
|
||||
|
||||
```bash
|
||||
export MIMO_API_KEY='你的 MiMo API Key'
|
||||
bash scripts/test/aiproxy/aiproxy-functional-test-mimo.sh
|
||||
```
|
||||
|
||||
预置模型：`export AIPROXY_FT_PROVIDER=xiaomi AIPROXY_FT_MODEL=mimo-v2.5-pro`
|
||||
|
||||
按 provider 自动命名的资源（可覆盖 `AIPROXY_FT_KEY_NAME` 等）：
|
||||
|
||||
| 变量 | 默认（xiaomi） |
|
||||
|------|----------------|
|
||||
| `AIPROXY_FT_KEY_NAME` | `aiproxy-ft-xiaomi` |
|
||||
| `AIPROXY_FT_VK_NAME` | `aiproxy-ft-xiaomi-vk` |
|
||||
| `AIPROXY_FT_ROUTING_NAME` | `aiproxy-ft-xiaomi-routing` |
|
||||
| `AIPROXY_FT_MODEL` | 交互默认 `mimo-v2-flash` |
|
||||
|
||||
## 手动步骤摘要
|
||||
|
||||
### Catalog
|
||||
|
||||
```bash
|
||||
climc ai-provider-show xiaomi
|
||||
climc ai-model-show xiaomi-mimo-v2-flash
|
||||
```
|
||||
|
||||
`config.base_url` 应为 `https://api.xiaomimimo.com`。
|
||||
|
||||
### ai_key
|
||||
|
||||
```bash
|
||||
climc ai-key-create mimo-ft \
|
||||
--ai-provider-id xiaomi \
|
||||
--secret "${MIMO_API_KEY}" \
|
||||
--weight 10 \
|
||||
--enabled
|
||||
```
|
||||
|
||||
### 路由与 chat
|
||||
|
||||
```bash
|
||||
climc ai-virtual-key-create aiproxy-mimo-ft-vk
|
||||
climc ai-routing-create aiproxy-mimo-ft-routing \
|
||||
--priority 10 \
|
||||
--models '[{"ai_provider_id":"xiaomi","ai_model_id":"mimo-v2-flash","priority":1}]'
|
||||
|
||||
AIPROXY_URL="$(climc endpoint-list --service aiproxy --interface public --limit 1 \
|
||||
--output-format json | jq -r '.data[0].url')"
|
||||
VK="$(climc ai-virtual-key-show aiproxy-mimo-ft-vk --output-format json | jq -r '.virtual_key')"
|
||||
|
||||
curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${VK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"mimo-v2-flash","messages":[{"role":"user","content":"hi"}],"max_tokens":64}' | jq .
|
||||
```
|
||||
|
||||
### 流式
|
||||
|
||||
`scripts/test/aiproxy/aiproxy-functional-test-mimo.sh` 在非流式通过后默认执行 step 7（`stream: true`）。跳过：`export AIPROXY_FT_SKIP_STREAM=1`。
|
||||
|
||||
```bash
|
||||
curl -k -sS -N -o /tmp/aiproxy-mimo-stream.sse \
|
||||
"${AIPROXY_URL%/}/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${VK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"mimo-v2-flash","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}'
|
||||
```
|
||||
|
||||
catalog 中其它模型：`mimo-v2.5-pro`、`mimo-v2-pro`、`mimo-v2.5`、`mimo-v2-omni`（id 形如 `xiaomi-mimo-v2.5-pro`）。
|
||||
|
||||
## 常见问题
|
||||
|
||||
**上游 401**
|
||||
检查 `MIMO_API_KEY` 是否有效；确认 `ai_key` 已 `--enabled` 且 `ai_provider_id=xiaomi`。
|
||||
|
||||
**`no ai_routing matched`**
|
||||
virtual key 与 routing 须在同一 climc 项目下创建。
|
||||
|
||||
**与 DashScope 脚本冲突**
|
||||
MiMo 脚本使用独立的 vk/routing/key 名称；勿与 `aiproxy-ft-vk` 混用同一 routing 的 model 列表。
|
||||
|
||||
## 清理
|
||||
|
||||
```bash
|
||||
climc ai-routing-delete aiproxy-mimo-ft-routing
|
||||
climc ai-virtual-key-delete aiproxy-mimo-ft-vk
|
||||
climc ai-key-delete mimo-ft
|
||||
```
|
||||
@@ -1,308 +0,0 @@
|
||||
# aiproxy 功能测试（climc + 通义千问 DashScope）
|
||||
|
||||
本文用 **climc** 配置 aiproxy 资源，并用 **curl** 调用 OpenAI 兼容接口 `POST /v1/chat/completions` 做端到端验证。
|
||||
|
||||
> **安全**：请勿把 DashScope API Key 写进脚本、文档或提交到 Git。在 shell 里用环境变量 `DASHSCOPE_API_KEY` 传入。若 Key 曾在聊天/工单中泄露，请到阿里云控制台轮换。
|
||||
|
||||
## 前置条件
|
||||
|
||||
| 项 | 说明 |
|
||||
|----|------|
|
||||
| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` 服务及 public endpoint |
|
||||
| 数据库 | 主节点已执行 `InitDB`，catalog 中已有 `aliyun` provider 及 `qwen-*` 模型（首次启动 master 会自动 seed） |
|
||||
| 客户端 | 已 `source /etc/yunion/rcadmin`（或等价 rc 文件），`climc` 能正常 list |
|
||||
| 工具 | `jq`（脚本与下文 curl 示例用于解析 JSON） |
|
||||
| 网络 | aiproxy 节点能访问 `https://dashscope.aliyuncs.com` |
|
||||
|
||||
### 一键脚本（交互式，推荐）
|
||||
|
||||
从 catalog 选择 **模型提供商** 与 **model_key**，终端输入 API Key（或使用环境变量跳过输入），自动完成非流式 + 流式 chat：
|
||||
|
||||
```bash
|
||||
source /etc/yunion/rcadmin
|
||||
export CLIMC_OUTPUT_FORMAT=json
|
||||
bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
```
|
||||
|
||||
也可预置后减少交互（仍会选择模型、是否流式，除非全部用环境变量）：
|
||||
|
||||
```bash
|
||||
export DASHSCOPE_API_KEY='你的 DashScope API Key' # 或 AIPROXY_FT_API_KEY
|
||||
export AIPROXY_FT_PROVIDER=aliyun
|
||||
export AIPROXY_FT_MODEL=qwen-turbo
|
||||
bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
```
|
||||
|
||||
通义快捷入口：`bash scripts/test/aiproxy/aiproxy-functional-test-qwen.sh`（默认 `aliyun`）。
|
||||
小米 MiMo 见 [functional-test-climc-mimo.md](./functional-test-climc-mimo.md)（`aiproxy-functional-test-mimo.sh`）。
|
||||
|
||||
非交互（CI）：
|
||||
|
||||
```bash
|
||||
export AIPROXY_FT_NONINTERACTIVE=1
|
||||
export AIPROXY_FT_PROVIDER=aliyun
|
||||
export AIPROXY_FT_MODEL=qwen-turbo
|
||||
export AIPROXY_FT_API_KEY='...'
|
||||
export AIPROXY_FT_SKIP_STREAM=1 # 可选，跳过流式
|
||||
bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
```
|
||||
|
||||
| 环境变量 | 说明 |
|
||||
|----------|------|
|
||||
| `AIPROXY_FT_PROVIDER` | `provider_key`（如 `aliyun`、`xiaomi`） |
|
||||
| `AIPROXY_FT_MODEL` | `model_key`（如 `qwen-turbo`） |
|
||||
| `AIPROXY_FT_API_KEY` | 上游 API Key（通用） |
|
||||
| `DASHSCOPE_API_KEY` / `MIMO_API_KEY` | 按提供商兼容的旧变量名 |
|
||||
| `AIPROXY_FT_SKIP_STREAM` | `1` 跳过流式；`0` 强制流式 |
|
||||
| `AIPROXY_URL` | 留空则从 endpoint-list 解析 |
|
||||
|
||||
## 测试流程概览
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
VK[ai_virtual_key] --> RT[ai_routing]
|
||||
RT --> RM[ai_routing_model]
|
||||
RM --> P[ai_provider aliyun]
|
||||
RM --> M[ai_model qwen-turbo]
|
||||
P --> K[ai_key secret]
|
||||
K --> DS[DashScope API]
|
||||
```
|
||||
|
||||
## 0a. ai_provider 创建测试脚本
|
||||
|
||||
自定义 provider（非 catalog seed）创建与校验：
|
||||
|
||||
```bash
|
||||
source /etc/yunion/rcadmin
|
||||
bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh
|
||||
```
|
||||
|
||||
交互输入：资源名、`provider_key`、`base_url`、是否 `--enabled`。非交互示例：
|
||||
|
||||
```bash
|
||||
export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1
|
||||
export AIPROXY_PROVIDER_FT_NAME=my-vllm
|
||||
export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-vllm
|
||||
export AIPROXY_PROVIDER_FT_BASE_URL=http://127.0.0.1:8000/v1
|
||||
bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh
|
||||
```
|
||||
|
||||
`provider_key` 须全局唯一；与 InitDB catalog（如 `aliyun`）重复会失败。完整 config 可用 `AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"..."}'`。
|
||||
|
||||
## 0. ai_proxy_node（多副本 / 路由绑定）
|
||||
|
||||
列出 aiproxy 实例节点（InitDB 后默认有 `primary`）：
|
||||
|
||||
```bash
|
||||
climc ai-proxy-node-list
|
||||
climc ai-proxy-node-show primary
|
||||
```
|
||||
|
||||
注册 standby 节点（与进程内 `register` 心跳相同，一般由 standby 自动调用；手工测试可用）：
|
||||
|
||||
```bash
|
||||
climc ai-proxy-node-register --address https://standby-host:30938 --hb-timeout 120
|
||||
```
|
||||
|
||||
手工创建/更新节点（需具备写权限策略）：
|
||||
|
||||
```bash
|
||||
climc ai-proxy-node-create standby-1 \
|
||||
--address https://standby-host:30938 \
|
||||
--access-address https://aiproxy-standby.example.com:443 \
|
||||
--hb-timeout 120 \
|
||||
--enabled
|
||||
|
||||
climc ai-proxy-node-update primary --address https://primary-host:30938 --access-address https://aiproxy.example.com:443
|
||||
climc ai-proxy-node-enable primary
|
||||
climc ai-proxy-node-disable <node-id>
|
||||
```
|
||||
|
||||
将 `ai_routing` 绑定到指定节点（chat 须走该节点 public endpoint）：
|
||||
|
||||
```bash
|
||||
climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary
|
||||
```
|
||||
|
||||
创建 `ai_routing` 时若省略 `--ai-proxy-node-id`，默认绑定 `primary` 节点；更新时显式传空值仍可清空绑定（任意 aiproxy 节点均可匹配）。
|
||||
|
||||
## 1. 检查 Keystone endpoint
|
||||
|
||||
```bash
|
||||
climc endpoint-list --service aiproxy --interface public
|
||||
```
|
||||
|
||||
应能看到当前 region 的 public URL（脚本会取第一条用于 curl）。
|
||||
|
||||
## 2. 检查 catalog（InitDB seed）
|
||||
|
||||
```bash
|
||||
climc ai-provider-list --provider-key aliyun
|
||||
climc ai-provider-show aliyun
|
||||
climc ai-model-list --ai-provider-id aliyun --model-key qwen-turbo
|
||||
```
|
||||
|
||||
确认 `provider_key=aliyun`，`config.base_url` 含 `https://dashscope.aliyuncs.com/compatible-mode`，且存在模型 `model_key=qwen-turbo`（catalog 固定 id / name：`aliyun-qwen-turbo`）。
|
||||
|
||||
## 3. 注册上游 API Key（ai_key）
|
||||
|
||||
将 DashScope Key 存为 `ai_key`，供 chat 时按 provider 加权选取：
|
||||
|
||||
```bash
|
||||
climc ai-key-create qwen-dashscope-ft \
|
||||
--ai-provider-id aliyun \
|
||||
--secret "${DASHSCOPE_API_KEY}" \
|
||||
--weight 10 \
|
||||
--enabled
|
||||
```
|
||||
|
||||
校验：
|
||||
|
||||
```bash
|
||||
climc ai-key-list --ai-provider-id aliyun
|
||||
climc ai-key-show qwen-dashscope-ft
|
||||
```
|
||||
|
||||
确认 `ai_provider_id` 为 `aliyun`，且 **`enabled=true`**（`ai_key` 默认 disabled，创建时需 `--enabled`；若已存在但被禁用，执行 `climc ai-key-enable qwen-dashscope-ft`）。若曾用错误参数创建过同名 key，可更新：
|
||||
|
||||
```bash
|
||||
climc ai-key-update qwen-dashscope-ft \
|
||||
--ai-provider-id aliyun \
|
||||
--secret "${DASHSCOPE_API_KEY}" \
|
||||
--weight 10
|
||||
climc ai-key-enable qwen-dashscope-ft
|
||||
```
|
||||
|
||||
（`secret` 在 API 中通常不回显，仅用于上游调用。）
|
||||
|
||||
## 4. 创建 Virtual Key（客户端鉴权）
|
||||
|
||||
```bash
|
||||
climc ai-virtual-key-create aiproxy-ft-vk
|
||||
```
|
||||
|
||||
记下返回的 `virtual_key`（形如 `sk-...`）。查看：
|
||||
|
||||
```bash
|
||||
climc ai-virtual-key-list
|
||||
climc ai-virtual-key-show aiproxy-ft-vk
|
||||
```
|
||||
|
||||
Virtual key 归属当前 climc 用户的 **项目**；后续 `ai_routing` 须在同一项目（或共享到该项目）下。
|
||||
|
||||
## 5. 创建项目路由（ai_routing + models）
|
||||
|
||||
将项目内请求 `model=qwen-turbo` 指到 catalog 的 aliyun/qwen-turbo。
|
||||
|
||||
**精确匹配**：`--model-key qwen-turbo` 与请求 body 中 `model` 完全一致时命中（优先于 `--model-pattern` 通配规则）。
|
||||
|
||||
`models` 里 `ai_model_id` 使用 catalog 固定 id（与 name 相同，如 `aliyun-qwen-turbo`），或在指定 `ai_provider_id` 时也可填 **model_key**（如 `qwen-turbo`）：
|
||||
|
||||
```bash
|
||||
climc ai-routing-create aiproxy-ft-routing \
|
||||
--priority 10 \
|
||||
--model-key qwen-turbo \
|
||||
--models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-turbo","priority":1}]'
|
||||
```
|
||||
|
||||
或手工指定 name：
|
||||
|
||||
```bash
|
||||
climc ai-routing-create aiproxy-ft-routing \
|
||||
--priority 10 \
|
||||
--models '[{"ai_provider_id":"aliyun","ai_model_id":"aliyun-qwen-turbo","priority":1}]'
|
||||
```
|
||||
|
||||
查看绑定模型：
|
||||
|
||||
```bash
|
||||
climc ai-routing-show aiproxy-ft-routing
|
||||
```
|
||||
|
||||
也可事后调整：
|
||||
|
||||
```bash
|
||||
climc ai-routing-set-models aiproxy-ft-routing \
|
||||
--models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-plus","priority":1}]'
|
||||
```
|
||||
|
||||
可选：将规则绑定到指定 aiproxy 实例（多副本时）：
|
||||
|
||||
```bash
|
||||
# 仅当需要固定到 primary 等节点时
|
||||
climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary
|
||||
```
|
||||
|
||||
## 6. Chat completions（curl）
|
||||
|
||||
climc 暂无 chat 子命令，用 public endpoint + virtual key 调用（`-k` 跳过 TLS 证书校验，适用于自签或内网 HTTPS）：
|
||||
|
||||
```bash
|
||||
AIPROXY_URL="${AIPROXY_URL:-$(climc endpoint-list --service aiproxy --interface public --limit 1 \
|
||||
--output-format json | jq -r '.data[0].url // empty')}"
|
||||
|
||||
VK="$(climc ai-virtual-key-show aiproxy-ft-vk --output-format json \
|
||||
| jq -r '.virtual_key')"
|
||||
|
||||
curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${VK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "qwen-turbo",
|
||||
"messages": [{"role": "user", "content": "用一句话介绍通义千问"}],
|
||||
"max_tokens": 128
|
||||
}' | jq .
|
||||
```
|
||||
|
||||
**期望**：HTTP 200，JSON 含 `choices[0].message.content` 及 `usage`。
|
||||
|
||||
## 6b. 流式 Chat（curl / 脚本 step 7）
|
||||
|
||||
一键脚本在步骤 6 非流式成功后，默认继续执行流式校验（聚合 `choices[0].delta.content`）。跳过流式：
|
||||
|
||||
```bash
|
||||
export AIPROXY_FT_SKIP_STREAM=1
|
||||
```
|
||||
|
||||
手动 curl（SSE，`data: [DONE]` 结束）：
|
||||
|
||||
```bash
|
||||
curl -k -sS -N -o /tmp/aiproxy-ft-stream.sse \
|
||||
"${AIPROXY_URL%/}/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${VK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"qwen-turbo","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}'
|
||||
```
|
||||
|
||||
期望：HTTP 200，响应体含 `data: {...}` 行且至少一条 `delta.content` 非空；最后为 `data: [DONE]`。
|
||||
|
||||
## 7. 负向用例（可选）
|
||||
|
||||
| 场景 | 操作 | 期望 |
|
||||
|------|------|------|
|
||||
| 错误 virtual key | `Authorization: Bearer sk-invalid` | 4xx，virtual key 无效 |
|
||||
| 无路由 | `climc ai-routing-disable aiproxy-ft-routing` 或删除后再 chat | 404，无匹配 routing |
|
||||
| 禁用 virtual key | `climc ai-virtual-key-disable aiproxy-ft-vk` | 4xx |
|
||||
| provider 限制 | create vk 时 `--limits '{"allowed_ai_provider_ids":["openai"]}'` | 4xx，provider 不允许 |
|
||||
|
||||
## 8. 清理（可选）
|
||||
|
||||
```bash
|
||||
climc ai-routing-delete aiproxy-ft-routing
|
||||
climc ai-virtual-key-delete aiproxy-ft-vk
|
||||
climc ai-key-delete qwen-dashscope-ft
|
||||
```
|
||||
|
||||
## 常见问题
|
||||
|
||||
**`no ai_routing matched for virtual key project`**
|
||||
Virtual key 的 `project_id` 与 routing 所在项目不一致，或 routing 未 `enabled`、未共享到该项目。用同一 `climc` 项目上下文创建两者。
|
||||
|
||||
**`no api_key for ai_provider`**
|
||||
未创建 `ai_key`，且 `ai_provider.config` 里也没有 `api_key`。按步骤 3 创建 `ai_key`。
|
||||
|
||||
**DashScope 401/403**
|
||||
检查 `DASHSCOPE_API_KEY` 是否有效、是否开通对应模型。
|
||||
|
||||
**多副本 `ai_routing` 绑定其它节点**
|
||||
若 routing 指定了 `ai_proxy_node_id`，须访问该节点的 public endpoint，或去掉绑定。
|
||||
@@ -0,0 +1,369 @@
|
||||
# aiproxy 功能测试（climc）
|
||||
|
||||
本文用 **climc** 配置 aiproxy 资源，并通过 **`climc aiproxy-test-*`** 子命令或 **curl** 做端到端验证。
|
||||
|
||||
| 子命令 | 数据面路径 | 用途 |
|
||||
|--------|------------|------|
|
||||
| `aiproxy-test-chat` | `POST /ai/openai/v1/chat/completions` | OpenAI 兼容 chat（非流式 + 流式） |
|
||||
| `aiproxy-test-anthropic` | `POST /ai/anthropic/v1/messages` | Anthropic Messages API |
|
||||
| `aiproxy-test-provider-create` | — | 创建自定义 `ai_provider` 并校验 |
|
||||
|
||||
> **安全**：请勿将上游 API Key 写入文档或提交到 Git。使用环境变量传入；若 Key 曾泄露，请到对应云平台控制台轮换。
|
||||
|
||||
## 前置条件
|
||||
|
||||
| 项 | 说明 |
|
||||
|----|------|
|
||||
| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` 服务及 public endpoint |
|
||||
| 数据库 | 主节点已执行 `InitDB`，catalog 已 seed 对应 provider / model |
|
||||
| 客户端 | 已 `source /etc/yunion/rcadmin`（或等价 rc 文件），`climc` 能正常 list |
|
||||
| 网络 | aiproxy 节点能访问目标上游（DashScope、MiMo、Anthropic 等） |
|
||||
|
||||
## 一键 E2E（交互式，推荐）
|
||||
|
||||
从 catalog 选择 **模型提供商** 与 **model_key**，终端输入 API Key（或使用环境变量跳过输入），自动完成 `ai_key` / `ai_virtual_key` / `ai_routing` 配置及 chat 校验：
|
||||
|
||||
```bash
|
||||
source /etc/yunion/rcadmin
|
||||
climc aiproxy-test-chat
|
||||
```
|
||||
|
||||
非交互（CI）：
|
||||
|
||||
```bash
|
||||
export AIPROXY_TEST_NONINTERACTIVE=1
|
||||
export AIPROXY_TEST_PROVIDER=aliyun
|
||||
export AIPROXY_TEST_MODEL=qwen-turbo
|
||||
export AIPROXY_TEST_API_KEY='...'
|
||||
export AIPROXY_TEST_SKIP_STREAM=1 # 可选，跳过流式
|
||||
climc aiproxy-test-chat
|
||||
```
|
||||
|
||||
### 环境变量
|
||||
|
||||
| 变量 | 说明 |
|
||||
|------|------|
|
||||
| `AIPROXY_TEST_PROVIDER` | `provider_key`（如 `aliyun`、`xiaomi`） |
|
||||
| `AIPROXY_TEST_MODEL` | `model_key`（如 `qwen-turbo`） |
|
||||
| `AIPROXY_TEST_API_KEY` | 上游 API Key（通用） |
|
||||
| `AIPROXY_FT_*` | 同上（兼容旧变量名） |
|
||||
| `DASHSCOPE_API_KEY` | 通义千问（`provider=aliyun`） |
|
||||
| `MIMO_API_KEY` | 小米 MiMo（`provider=xiaomi`） |
|
||||
| `ANTHROPIC_API_KEY` | Anthropic 直通 |
|
||||
| `DEEPSEEK_API_KEY` | DeepSeek（Anthropic 兼容场景） |
|
||||
| `AIPROXY_TEST_SKIP_STREAM` | `1` 跳过流式；`0` 强制流式 |
|
||||
| `AIPROXY_TEST_KEEP_RESOURCES` | `1` 测试结束后**保留**本次创建的资源（默认自动清理） |
|
||||
| `AIPROXY_URL` | 留空则从 `endpoint-list` 解析 |
|
||||
|
||||
`aiproxy-test-*` 会在测试过程中自动创建缺失的依赖（`ai_model`、`ai_key`、`ai_virtual_key`、`ai_routing` 等），**测试结束（成功或失败）后自动删除本次创建的资源**。临时修改的 `ai_provider.config.base_url` 会还原。仅删除本次新建项，测试前已存在的同名资源不会被删。
|
||||
|
||||
保留资源以便排查：`climc aiproxy-test-chat --keep-resources` 或 `export AIPROXY_TEST_KEEP_RESOURCES=1`。
|
||||
|
||||
`aiproxy-test-chat` 按 provider 自动命名资源（可用 `--key-name`、`--vk-name`、`--routing-name` 覆盖），默认形如 `aiproxy-test-{provider}`。
|
||||
|
||||
## 按模型提供商快速开始
|
||||
|
||||
### 通义千问（DashScope / aliyun）
|
||||
|
||||
catalog 需含 `aliyun` 及 `qwen-*` 模型；上游 `https://dashscope.aliyuncs.com/compatible-mode`。
|
||||
|
||||
```bash
|
||||
export DASHSCOPE_API_KEY='你的 DashScope API Key'
|
||||
climc aiproxy-test-chat --provider aliyun --model qwen-turbo --api-key "$DASHSCOPE_API_KEY"
|
||||
```
|
||||
|
||||
### 小米 MiMo（xiaomi）
|
||||
|
||||
catalog 需含 `xiaomi` 及 `mimo-*` 模型；上游 `https://api.xiaomimimo.com`。
|
||||
|
||||
```bash
|
||||
export MIMO_API_KEY='你的 MiMo API Key'
|
||||
climc aiproxy-test-chat --provider xiaomi --model mimo-v2-flash --api-key "$MIMO_API_KEY"
|
||||
```
|
||||
|
||||
其它 catalog 模型：`mimo-v2.5-pro`、`mimo-v2-pro`、`mimo-v2.5`、`mimo-v2-omni`（id 形如 `xiaomi-mimo-v2.5-pro`）。
|
||||
|
||||
```bash
|
||||
export AIPROXY_TEST_PROVIDER=xiaomi AIPROXY_TEST_MODEL=mimo-v2.5-pro
|
||||
climc aiproxy-test-chat --api-key "$MIMO_API_KEY"
|
||||
```
|
||||
|
||||
MiMo 与 DashScope 测试应使用独立的 vk/routing/key 名称，避免混用同一 routing 的 model 列表。
|
||||
|
||||
### Anthropic Messages API
|
||||
|
||||
数据面 **`POST /ai/anthropic/v1/messages`**，认证为 `Authorization: Bearer <virtual_key>`（**不是**上游 Anthropic/DeepSeek API Key）。
|
||||
|
||||
Claude Code / Anthropic SDK 在正式请求前会对 base URL 发 **`HEAD /ai/anthropic/`** 做连通性探测；aiproxy 已返回 `204`。对 **`HEAD /ai/anthropic/v1/messages`** 无 virtual key 时返回 `401`（表示路由存在、需鉴权）。
|
||||
|
||||
**Anthropic 直通**（catalog `provider_key=anthropic`）：
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_API_KEY='sk-ant-...'
|
||||
climc aiproxy-test-anthropic --provider anthropic --model claude-sonnet-4-5 --api-key "$ANTHROPIC_API_KEY"
|
||||
```
|
||||
|
||||
**OpenAI 兼容后端（DeepSeek，翻译模式）**：`config.api_mode=openai`（默认）；客户端仍用 Anthropic SDK；aiproxy 转换为 OpenAI `chat/completions` 转发。
|
||||
|
||||
| 资源 | 示例 |
|
||||
|------|------|
|
||||
| `ai_provider.provider_key` | `deepseek` 或 `openai` |
|
||||
| `ai_provider.config.base_url` | `https://api.deepseek.com` |
|
||||
| `ai_provider.config.api_mode` | `openai`（可省略） |
|
||||
| `ai_model.model_key` | `deepseek-chat` |
|
||||
|
||||
```bash
|
||||
export DEEPSEEK_API_KEY='...'
|
||||
climc aiproxy-test-anthropic --provider deepseek --model deepseek-chat \
|
||||
--api-key "$DEEPSEEK_API_KEY" --upstream-base-url https://api.deepseek.com
|
||||
```
|
||||
|
||||
**DeepSeek 原生 Anthropic 模式**：`provider_key=deepseek` 且 `config.api_mode=anthropic`；aiproxy 将 Anthropic SDK 请求直通 DeepSeek `https://api.deepseek.com/anthropic/v1/messages`（`base_url` 可仍填 `https://api.deepseek.com`，由 aiproxy 自动补 `/anthropic`）。
|
||||
|
||||
创建 provider 时在顶层 `secret` 写入上游密钥（PostCreate 自动创建关联 `ai_key`）；`config` 仅保留 `base_url` / `api_mode`：
|
||||
|
||||
```json
|
||||
{
|
||||
"generate_name": "my-deepseek",
|
||||
"provider_key": "deepseek",
|
||||
"secret": "<deepseek-api-key>",
|
||||
"config": {
|
||||
"base_url": "https://api.deepseek.com",
|
||||
"api_mode": "anthropic"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`config.api_key` 已不再支持；请在「供应商密钥」Tab 或独立 `ai_key` 资源中管理密钥。
|
||||
|
||||
OpenAI SDK 经 `/ai/openai/v1/chat/completions` 访问同一 provider 时，也会按 `api_mode=anthropic` 转为 Anthropic Messages 上游。
|
||||
|
||||
Anthropic SDK / Claude Code 配置（`base_url` 指向 aiproxy，**不要**加 `/v1`；`api_key` 为 **virtual_key**）：
|
||||
|
||||
```python
|
||||
import anthropic
|
||||
client = anthropic.Anthropic(
|
||||
base_url=f"{AIPROXY_URL}/ai/anthropic", # 正确：SDK 自行拼 /v1/messages
|
||||
api_key=VIRTUAL_KEY, # aiproxy virtual_key，不是上游 Key
|
||||
)
|
||||
client.messages.create(model="claude-sonnet-4-5", max_tokens=128, messages=[...])
|
||||
```
|
||||
|
||||
环境变量等价配置：
|
||||
|
||||
```bash
|
||||
export ANTHROPIC_BASE_URL="${AIPROXY_URL}/ai/anthropic" # 勿写成 .../ai/anthropic/v1
|
||||
export ANTHROPIC_API_KEY="${VIRTUAL_KEY}"
|
||||
```
|
||||
|
||||
| 配置项 | 正确 | 错误 |
|
||||
|--------|------|------|
|
||||
| `ANTHROPIC_BASE_URL` | `https://host/ai/anthropic` | `.../ai/anthropic/v1`（会变成 `/v1/v1/messages`） |
|
||||
| API Key | aiproxy **virtual_key** | 上游 Anthropic / DeepSeek key |
|
||||
|
||||
## 测试流程概览
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
VK[ai_virtual_key] --> RT[ai_routing]
|
||||
RT --> RM[ai_routing_model]
|
||||
RM --> P[ai_provider]
|
||||
RM --> M[ai_model]
|
||||
P --> K[ai_key secret]
|
||||
K --> UP[上游 API]
|
||||
```
|
||||
|
||||
## ai_provider 创建测试
|
||||
|
||||
### 自定义供应商（provider_key=custom）
|
||||
|
||||
用户自建网关，需填写完整 `base_url`、顶层 `secret` 与 `api_mode`（openai / anthropic）：
|
||||
|
||||
```json
|
||||
{
|
||||
"generate_name": "my-gateway",
|
||||
"provider_key": "custom",
|
||||
"secret": "sk-xxx",
|
||||
"config": {
|
||||
"base_url": "https://llm.example.com/v1",
|
||||
"api_mode": "openai"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Anthropic Messages 上游示例：
|
||||
|
||||
```json
|
||||
{
|
||||
"generate_name": "my-anthropic-gateway",
|
||||
"provider_key": "custom",
|
||||
"secret": "sk-ant-xxx",
|
||||
"config": {
|
||||
"base_url": "https://llm.example.com/anthropic",
|
||||
"api_mode": "anthropic"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
创建后不会自动注入 catalog 模型；须手动创建 `ai_model` 并配置路由。
|
||||
|
||||
### 自托管 provider（非 catalog seed）
|
||||
|
||||
```bash
|
||||
climc aiproxy-test-provider-create
|
||||
```
|
||||
|
||||
非交互示例：
|
||||
|
||||
```bash
|
||||
export AIPROXY_PROVIDER_TEST_NONINTERACTIVE=1
|
||||
climc aiproxy-test-provider-create \
|
||||
--name my-vllm --provider-key my-vllm \
|
||||
--base-url http://127.0.0.1:8000/v1 --enabled
|
||||
```
|
||||
|
||||
`provider_key` 须全局唯一；与 InitDB catalog 重复会失败。完整 config 可用 `--config '{"base_url":"..."}'` 或 `AIPROXY_PROVIDER_TEST_CONFIG`。
|
||||
|
||||
## ai_proxy_node（多副本 / 路由绑定）
|
||||
|
||||
```bash
|
||||
climc ai-proxy-node-list
|
||||
climc ai-proxy-node-show primary
|
||||
climc ai-proxy-node-register --address https://standby-host:30938 --hb-timeout 120
|
||||
```
|
||||
|
||||
将 `ai_routing` 绑定到指定节点（chat 须走该节点 public endpoint）：
|
||||
|
||||
```bash
|
||||
climc ai-routing-update aiproxy-test-routing --ai-proxy-node-id primary
|
||||
```
|
||||
|
||||
创建 `ai_routing` 时若省略 `--ai-proxy-node-id`，默认绑定 `primary` 节点。
|
||||
|
||||
## 手动步骤（以 aliyun / qwen-turbo 为例）
|
||||
|
||||
以下步骤与 `climc aiproxy-test-chat` 等价，便于理解各资源关系；其它 provider 替换 `aliyun`、`qwen-turbo` 及对应 API Key 即可。
|
||||
|
||||
### 1. 检查 Keystone endpoint
|
||||
|
||||
```bash
|
||||
climc endpoint-list --service aiproxy --interface public
|
||||
```
|
||||
|
||||
### 2. 检查 catalog
|
||||
|
||||
```bash
|
||||
climc ai-provider-show aliyun
|
||||
climc ai-model-show aliyun-qwen-turbo
|
||||
```
|
||||
|
||||
小米 MiMo：`climc ai-provider-show xiaomi`、`climc ai-model-show xiaomi-mimo-v2-flash`。
|
||||
|
||||
### 3. 注册上游 API Key（ai_key）
|
||||
|
||||
```bash
|
||||
climc ai-key-create qwen-dashscope-test \
|
||||
--ai-provider-id aliyun \
|
||||
--secret "${DASHSCOPE_API_KEY}" \
|
||||
--weight 10 \
|
||||
--enabled
|
||||
```
|
||||
|
||||
`ai_key` 默认 disabled，创建时需 `--enabled`。
|
||||
|
||||
### 4. 创建 Virtual Key
|
||||
|
||||
```bash
|
||||
climc ai-virtual-key-create aiproxy-test-vk
|
||||
climc ai-virtual-key-show aiproxy-test-vk
|
||||
```
|
||||
|
||||
Virtual key 归属当前 climc 用户的 **项目**；`ai_routing` 须在同一项目（或共享到该项目）下。
|
||||
|
||||
### 5. 创建项目路由
|
||||
|
||||
```bash
|
||||
climc ai-routing-create aiproxy-test-routing \
|
||||
--priority 10 \
|
||||
--model-key qwen-turbo \
|
||||
--models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-turbo","priority":1}]'
|
||||
```
|
||||
|
||||
### 6. Chat completions（curl）
|
||||
|
||||
```bash
|
||||
AIPROXY_URL="${AIPROXY_URL:-$(climc endpoint-list --service aiproxy --interface public --limit 1 \
|
||||
--output-format json | jq -r '.data[0].url // empty')}"
|
||||
|
||||
VK="$(climc ai-virtual-key-show aiproxy-test-vk --output-format json | jq -r '.virtual_key')"
|
||||
|
||||
curl -k -sS "${AIPROXY_URL%/}/ai/openai/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${VK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "qwen-turbo",
|
||||
"messages": [{"role": "user", "content": "用一句话介绍通义千问"}],
|
||||
"max_tokens": 128
|
||||
}' | jq .
|
||||
```
|
||||
|
||||
**期望**：HTTP 200，JSON 含 `choices[0].message.content` 及 `usage`。
|
||||
|
||||
### 6b. 流式 Chat
|
||||
|
||||
`climc aiproxy-test-chat` 默认在非流式成功后继续流式校验。跳过：`climc aiproxy-test-chat --skip-stream`。
|
||||
|
||||
```bash
|
||||
curl -k -sS -N "${AIPROXY_URL%/}/ai/openai/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${VK}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"qwen-turbo","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}'
|
||||
```
|
||||
|
||||
Anthropic 流式与非流式均走 `/ai/anthropic/v1/messages`，请求体设置 `"stream": true` 即可。
|
||||
|
||||
## 负向用例（可选）
|
||||
|
||||
| 场景 | 操作 | 期望 |
|
||||
|------|------|------|
|
||||
| 错误 virtual key | `Authorization: Bearer sk-invalid` | 4xx |
|
||||
| 无路由 | disable 或删除 routing 后再 chat | 404 |
|
||||
| 禁用 virtual key | `climc ai-virtual-key-disable aiproxy-test-vk` | 4xx |
|
||||
| provider 限制 | vk `--limits '{"allowed_ai_provider_ids":["openai"]}'` | 4xx |
|
||||
|
||||
## 清理
|
||||
|
||||
`aiproxy-test-*` 默认在结束时自动清理（见上文 `AIPROXY_TEST_KEEP_RESOURCES`）。手动清理示例（仅在使用 `--keep-resources` 或清理失败时需要）：
|
||||
|
||||
DashScope：
|
||||
|
||||
```bash
|
||||
climc ai-routing-delete aiproxy-test-aliyun-routing
|
||||
climc ai-virtual-key-delete aiproxy-test-aliyun-vk
|
||||
climc ai-key-delete aiproxy-test-aliyun
|
||||
```
|
||||
|
||||
MiMo 示例（若使用独立资源名）：
|
||||
|
||||
```bash
|
||||
climc ai-routing-delete aiproxy-test-xiaomi-routing
|
||||
climc ai-virtual-key-delete aiproxy-test-xiaomi-vk
|
||||
climc ai-key-delete aiproxy-test-xiaomi
|
||||
```
|
||||
|
||||
## 常见问题
|
||||
|
||||
**`no ai_routing matched for virtual key project`**
|
||||
Virtual key 与 routing 的项目不一致，或 routing 未 `enabled`、未共享到该项目。
|
||||
|
||||
**`add an enabled ai_key with secret for this provider`**
|
||||
未创建启用的 `ai_key`，或密钥为空。创建 provider 时使用顶层 `secret`，或在「供应商密钥」Tab 手动添加。
|
||||
|
||||
**DashScope / MiMo 401/403**
|
||||
检查对应环境变量中的 API Key 是否有效、模型是否已开通。
|
||||
|
||||
**多副本 `ai_routing` 绑定其它节点**
|
||||
若 routing 指定了 `ai_proxy_node_id`，须访问该节点的 public endpoint，或去掉绑定。
|
||||
|
||||
**MiMo 与 DashScope 资源冲突**
|
||||
各 provider 使用独立的 vk/routing/key 名称，勿共用同一 routing 的 model 列表。
|
||||
@@ -0,0 +1,242 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
|
||||
)
|
||||
|
||||
type AdminNames struct {
|
||||
KeyName string
|
||||
VkName string
|
||||
RoutingName string
|
||||
}
|
||||
|
||||
func DefaultAdminNames(providerKey string, suffix string) AdminNames {
|
||||
base := fmt.Sprintf("aiproxy-test-%s", providerKey)
|
||||
if suffix != "" {
|
||||
base = fmt.Sprintf("aiproxy-test-%s-%s", providerKey, suffix)
|
||||
}
|
||||
return AdminNames{
|
||||
KeyName: base,
|
||||
VkName: base + "-vk",
|
||||
RoutingName: base + "-routing",
|
||||
}
|
||||
}
|
||||
|
||||
func DefaultAnthropicAdminNames(providerKey string) AdminNames {
|
||||
if providerKey == "anthropic" {
|
||||
return AdminNames{
|
||||
KeyName: "aiproxy-test-anthropic",
|
||||
VkName: "aiproxy-test-anthropic-vk",
|
||||
RoutingName: "aiproxy-test-anthropic-routing",
|
||||
}
|
||||
}
|
||||
return AdminNames{
|
||||
KeyName: fmt.Sprintf("aiproxy-test-%s-anthropic", providerKey),
|
||||
VkName: "aiproxy-test-anthropic-vk",
|
||||
RoutingName: "aiproxy-test-anthropic-routing",
|
||||
}
|
||||
}
|
||||
|
||||
func ensureAiKey(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, keyName, apiSecret string) error {
|
||||
provider, err := apmodules.AiProviders.Get(session, providerKey, nil)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "ai_provider %s not found", providerKey)
|
||||
}
|
||||
providerID, _ := provider.GetString("id")
|
||||
if providerID == "" {
|
||||
return errors.Errorf("ai_provider %s has empty id", providerKey)
|
||||
}
|
||||
|
||||
if _, err := apmodules.AiKeys.Get(session, keyName, nil); err == nil {
|
||||
fmt.Printf("ai_key %s exists, syncing secret and ai_provider_id\n", keyName)
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("ai_provider_id", jsonutils.NewString(providerID))
|
||||
params.Set("secret", jsonutils.NewString(apiSecret))
|
||||
params.Set("weight", jsonutils.NewInt(10))
|
||||
if _, err := apmodules.AiKeys.Update(session, keyName, params); err != nil {
|
||||
return errors.Wrap(err, "ai-key-update")
|
||||
}
|
||||
} else {
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("name", jsonutils.NewString(keyName))
|
||||
params.Set("ai_provider_id", jsonutils.NewString(providerID))
|
||||
params.Set("secret", jsonutils.NewString(apiSecret))
|
||||
params.Set("weight", jsonutils.NewInt(10))
|
||||
params.Set("enabled", jsonutils.JSONTrue)
|
||||
if _, err := apmodules.AiKeys.Create(session, params); err != nil {
|
||||
return errors.Wrap(err, "ai-key-create")
|
||||
}
|
||||
if tracker != nil {
|
||||
tracker.createdAiKey = keyName
|
||||
}
|
||||
}
|
||||
return ensureAiKeyEnabled(session, keyName)
|
||||
}
|
||||
|
||||
func ensureAiKeyEnabled(session *mcclient.ClientSession, keyName string) error {
|
||||
obj, err := apmodules.AiKeys.Get(session, keyName, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enabled, _ := obj.Bool("enabled")
|
||||
if enabled {
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("ai_key %s is disabled, enabling\n", keyName)
|
||||
_, err = apmodules.AiKeys.PerformAction(session, keyName, "enable", nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func VerifyAiKeyForProvider(session *mcclient.ClientSession, providerKey string) error {
|
||||
provider, err := apmodules.AiProviders.Get(session, providerKey, nil)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "ai_provider %s not found", providerKey)
|
||||
}
|
||||
providerID, _ := provider.GetString("id")
|
||||
query := jsonutils.NewDict()
|
||||
query.Set("ai_provider_id", jsonutils.NewString(providerID))
|
||||
result, err := apmodules.AiKeys.List(session, query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
count := 0
|
||||
for _, item := range result.Data {
|
||||
ok, _ := item.Bool("enabled")
|
||||
if ok {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
return errors.Errorf("no enabled ai_key bound to ai_provider_id=%s", providerID)
|
||||
}
|
||||
fmt.Printf("enabled ai_key rows for %s: %d\n", providerID, count)
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureVirtualKey(session *mcclient.ClientSession, tracker *ResourceTracker, vkName string) (string, error) {
|
||||
if _, err := apmodules.AiVirtualKeys.Get(session, vkName, nil); err != nil {
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("name", jsonutils.NewString(vkName))
|
||||
if _, err := apmodules.AiVirtualKeys.Create(session, params); err != nil {
|
||||
return "", errors.Wrap(err, "ai-virtual-key-create")
|
||||
}
|
||||
if tracker != nil {
|
||||
tracker.createdVirtualKey = vkName
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("virtual key %s already exists\n", vkName)
|
||||
}
|
||||
obj, err := apmodules.AiVirtualKeys.Get(session, vkName, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
vk, _ := obj.GetString("virtual_key")
|
||||
if vk == "" {
|
||||
return "", errors.Error("empty virtual_key from ai-virtual-key-show")
|
||||
}
|
||||
return vk, nil
|
||||
}
|
||||
|
||||
func ensureRouting(session *mcclient.ClientSession, tracker *ResourceTracker, routingName, providerKey, routingModelRef string) error {
|
||||
if _, err := apmodules.AiRoutings.Get(session, routingName, nil); err == nil {
|
||||
fmt.Printf("routing %s already exists\n", routingName)
|
||||
return nil
|
||||
}
|
||||
models := jsonutils.NewArray()
|
||||
models.Add(jsonutils.Marshal(map[string]interface{}{
|
||||
"ai_provider_id": providerKey,
|
||||
"ai_model_id": routingModelRef,
|
||||
"priority": 1,
|
||||
}))
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("name", jsonutils.NewString(routingName))
|
||||
params.Set("priority", jsonutils.NewInt(10))
|
||||
params.Set("models", models)
|
||||
if _, err := apmodules.AiRoutings.Create(session, params); err != nil {
|
||||
return errors.Wrap(err, "ai-routing-create")
|
||||
}
|
||||
if tracker != nil {
|
||||
tracker.createdRouting = routingName
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetupAdminResources(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, modelKey, apiSecret string, names AdminNames) (virtualKey string, catalogModelID string, err error) {
|
||||
catalogModelID = CatalogModelID(providerKey, modelKey)
|
||||
routingModelRef, _, err := ensureAiModel(session, tracker, providerKey, modelKey)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = ensureAiKey(session, tracker, providerKey, names.KeyName, apiSecret); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = VerifyAiKeyForProvider(session, providerKey); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
virtualKey, err = ensureVirtualKey(session, tracker, names.VkName)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err = ensureRouting(session, tracker, names.RoutingName, providerKey, routingModelRef); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return virtualKey, catalogModelID, nil
|
||||
}
|
||||
|
||||
func EnsureAiProviderBaseURL(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, baseURL string) error {
|
||||
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
if baseURL == "" {
|
||||
return nil
|
||||
}
|
||||
obj, err := apmodules.AiProviders.Get(session, providerKey, nil)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "ai_provider %s not found", providerKey)
|
||||
}
|
||||
current, _ := obj.GetString("config", "base_url")
|
||||
current = strings.TrimRight(strings.TrimSpace(current), "/")
|
||||
if current == baseURL {
|
||||
fmt.Printf("ai_provider %s config.base_url=%s\n", providerKey, baseURL)
|
||||
return nil
|
||||
}
|
||||
if tracker != nil && tracker.providerConfigRestore == nil {
|
||||
snap, err := snapshotProviderConfig(session, providerKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tracker.providerConfigRestore = snap
|
||||
}
|
||||
configDict := jsonutils.NewDict()
|
||||
if obj.Contains("config") {
|
||||
cfg, _ := obj.Get("config")
|
||||
if cfgDict, ok := cfg.(*jsonutils.JSONDict); ok {
|
||||
configDict = cfgDict
|
||||
}
|
||||
}
|
||||
configDict.Set("base_url", jsonutils.NewString(baseURL))
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("config", configDict)
|
||||
fmt.Printf("updating ai_provider %s config.base_url -> %s\n", providerKey, baseURL)
|
||||
_, err = apmodules.AiProviders.Update(session, providerKey, params)
|
||||
return errors.Wrap(err, "ai-provider-update base_url")
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func RunAnthropicTest(session *mcclient.ClientSession, opts *AnthropicOptions) error {
|
||||
tracker := NewResourceTracker(envKeepResources(opts.KeepResources))
|
||||
defer tracker.Cleanup(session)
|
||||
|
||||
providerKey := strings.TrimSpace(opts.Provider)
|
||||
if providerKey == "" {
|
||||
providerKey = "anthropic"
|
||||
}
|
||||
|
||||
modelKey := strings.TrimSpace(opts.Model)
|
||||
if modelKey == "" {
|
||||
modelKey = resolveModelFromEnv()
|
||||
}
|
||||
if modelKey == "" {
|
||||
modelKey = DefaultModelForProvider(providerKey)
|
||||
}
|
||||
|
||||
apiSecret := strings.TrimSpace(opts.ApiKey)
|
||||
if apiSecret == "" {
|
||||
var err error
|
||||
apiSecret, err = promptApiKey(providerKey, "", envNonInteractive(opts.NonInteractive))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
prompt := strings.TrimSpace(opts.Prompt)
|
||||
if prompt == "" {
|
||||
prompt = resolvePromptFromEnv()
|
||||
}
|
||||
if prompt == "" {
|
||||
prompt = "Say hi in one short sentence."
|
||||
}
|
||||
|
||||
skipStream := envSkipStream(opts.SkipStream)
|
||||
names := DefaultAnthropicAdminNames(providerKey)
|
||||
if opts.KeyName != "" {
|
||||
names.KeyName = opts.KeyName
|
||||
}
|
||||
if opts.VkName != "" {
|
||||
names.VkName = opts.VkName
|
||||
}
|
||||
if opts.RoutingName != "" {
|
||||
names.RoutingName = opts.RoutingName
|
||||
}
|
||||
|
||||
catalogModelID := CatalogModelID(providerKey, modelKey)
|
||||
fmt.Println()
|
||||
fmt.Println("=== aiproxy Anthropic Messages 测试 ===")
|
||||
fmt.Printf("provider: %s model: %s catalog_id: %s\n", providerKey, modelKey, catalogModelID)
|
||||
fmt.Println()
|
||||
|
||||
Step("1. Resolve aiproxy URL")
|
||||
aiproxyURL, err := ResolveAiproxyURL(session, opts.AiproxyURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("AIPROXY_URL=%s\n", aiproxyURL)
|
||||
upstreamBase := strings.TrimSpace(opts.UpstreamBaseURL)
|
||||
if upstreamBase == "" {
|
||||
upstreamBase = envFirst("AIPROXY_TEST_BASE_URL", "AIPROXY_FT_BASE_URL")
|
||||
}
|
||||
if upstreamBase != "" {
|
||||
if err := EnsureAiProviderBaseURL(session, tracker, providerKey, upstreamBase); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
Step(fmt.Sprintf("2. Catalog %s / %s", providerKey, modelKey))
|
||||
if err := VerifyCatalog(session, providerKey, modelKey, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Step("3. ai_key / ai_virtual_key / ai_routing")
|
||||
vk, _, err := SetupAdminResources(session, tracker, providerKey, modelKey, apiSecret, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("virtual_key=%s...\n", previewText(vk, 12))
|
||||
|
||||
Step("4. POST /ai/anthropic/v1/messages")
|
||||
client := httpClientFromSession(session)
|
||||
payload := map[string]interface{}{
|
||||
"model": modelKey,
|
||||
"max_tokens": 128,
|
||||
"messages": []map[string]string{
|
||||
{"role": "user", "content": prompt},
|
||||
},
|
||||
}
|
||||
code, body, err := postJSON(client, anthropicMessagesURL(aiproxyURL), vk, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("HTTP %d\n", code)
|
||||
if err := printJSONBody(body); err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 200 {
|
||||
return errors.Errorf("anthropic messages request failed with HTTP %d", code)
|
||||
}
|
||||
content, err := extractAnthropicTextContent(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("text: %s\n", previewText(content, 120))
|
||||
|
||||
if !skipStream {
|
||||
Step("5. POST /ai/anthropic/v1/messages (stream=true)")
|
||||
streamPayload := map[string]interface{}{
|
||||
"model": modelKey,
|
||||
"stream": true,
|
||||
"max_tokens": 128,
|
||||
"messages": []map[string]string{
|
||||
{"role": "user", "content": prompt},
|
||||
},
|
||||
}
|
||||
streamCode, streamBody, err := postJSONStream(client, anthropicMessagesURL(aiproxyURL), vk, streamPayload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer streamBody.Close()
|
||||
fmt.Printf("HTTP %d (anthropic stream)\n", streamCode)
|
||||
aggregated, err := aggregateSSEStream(streamBody, parseAnthropicStreamDelta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("stream text: %s\n", previewText(aggregated, 120))
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Printf("OK: anthropic messages test passed for %s/%s.\n", providerKey, modelKey)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
|
||||
)
|
||||
|
||||
func CatalogModelID(providerKey, modelKey string) string {
|
||||
return fmt.Sprintf("%s-%s", providerKey, modelKey)
|
||||
}
|
||||
|
||||
func DefaultModelForProvider(providerKey string) string {
|
||||
switch providerKey {
|
||||
case api.ProviderKeyAliyun:
|
||||
return "qwen-turbo"
|
||||
case api.ProviderKeyXiaomi:
|
||||
return "mimo-v2-flash"
|
||||
case api.ProviderKeyDeepseek:
|
||||
return "deepseek-v4-flash"
|
||||
case api.ProviderKeyOpenAI:
|
||||
return "gpt-4o-mini"
|
||||
case api.ProviderKeyAnthropic:
|
||||
return "claude-sonnet-4-5"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func DefaultPromptForProvider(providerKey string) string {
|
||||
switch providerKey {
|
||||
case api.ProviderKeyAliyun:
|
||||
return "用一句话介绍通义千问"
|
||||
case api.ProviderKeyXiaomi:
|
||||
return "用一句话介绍小米 MiMo"
|
||||
default:
|
||||
return "用一句话介绍这个模型"
|
||||
}
|
||||
}
|
||||
|
||||
func ListCatalogProviderKeys(session *mcclient.ClientSession) ([]string, error) {
|
||||
query := jsonutils.NewDict()
|
||||
query.Set("limit", jsonutils.NewInt(500))
|
||||
result, err := apmodules.AiProviders.List(session, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(result.Data))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range result.Data {
|
||||
pk, _ := item.GetString("provider_key")
|
||||
pk = strings.TrimSpace(pk)
|
||||
if pk == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[pk]; ok {
|
||||
continue
|
||||
}
|
||||
seen[pk] = struct{}{}
|
||||
keys = append(keys, pk)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func ListCatalogModelKeys(session *mcclient.ClientSession, providerKey string) ([]string, error) {
|
||||
query := jsonutils.NewDict()
|
||||
query.Set("limit", jsonutils.NewInt(500))
|
||||
query.Set("ai_provider_id", jsonutils.NewString(providerKey))
|
||||
result, err := apmodules.AiModels.List(session, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
models := make([]string, 0, len(result.Data))
|
||||
seen := map[string]struct{}{}
|
||||
for _, item := range result.Data {
|
||||
mk, _ := item.GetString("model_key")
|
||||
mk = strings.TrimSpace(mk)
|
||||
if mk == "" || mk == "default" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[mk]; ok {
|
||||
continue
|
||||
}
|
||||
seen[mk] = struct{}{}
|
||||
models = append(models, mk)
|
||||
}
|
||||
sort.Strings(models)
|
||||
return models, nil
|
||||
}
|
||||
|
||||
func VerifyCatalog(session *mcclient.ClientSession, providerKey, modelKey string, warnMissingModel bool) error {
|
||||
if _, err := apmodules.AiProviders.Get(session, providerKey, nil); err != nil {
|
||||
return errors.Wrapf(err, "ai_provider %s missing; run aiproxy master InitDB first", providerKey)
|
||||
}
|
||||
if _, err := findAiModelByKey(session, providerKey, modelKey); err == nil {
|
||||
return nil
|
||||
}
|
||||
catalogID := CatalogModelID(providerKey, modelKey)
|
||||
if _, err := apmodules.AiModels.Get(session, catalogID, nil); err == nil {
|
||||
return nil
|
||||
}
|
||||
if warnMissingModel {
|
||||
fmt.Printf("WARN: ai_model %s not in catalog; will create for test if needed\n", catalogID)
|
||||
return nil
|
||||
}
|
||||
return errors.Errorf("ai_model %s not in catalog (re-run aiproxy master InitDB)", catalogID)
|
||||
}
|
||||
|
||||
func findAiModelByKey(session *mcclient.ClientSession, providerKey, modelKey string) (jsonutils.JSONObject, error) {
|
||||
query := jsonutils.NewDict()
|
||||
query.Set("limit", jsonutils.NewInt(10))
|
||||
query.Set("ai_provider_id", jsonutils.NewString(providerKey))
|
||||
query.Set("model_key", jsonutils.NewString(modelKey))
|
||||
result, err := apmodules.AiModels.List(session, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result.Data) == 0 {
|
||||
return nil, errors.Errorf("model_key %s not found under provider %s", modelKey, providerKey)
|
||||
}
|
||||
return result.Data[0], nil
|
||||
}
|
||||
|
||||
// EnsureAiModel guarantees an ai_model row exists for providerKey/modelKey.
|
||||
// Returns the model reference for ai_routing and the created resource name (if any).
|
||||
func ensureAiModel(session *mcclient.ClientSession, tracker *ResourceTracker, providerKey, modelKey string) (routingModelRef string, createdName string, err error) {
|
||||
catalogID := CatalogModelID(providerKey, modelKey)
|
||||
if _, err := apmodules.AiModels.Get(session, catalogID, nil); err == nil {
|
||||
return modelKey, "", nil
|
||||
}
|
||||
if _, err := findAiModelByKey(session, providerKey, modelKey); err == nil {
|
||||
return modelKey, "", nil
|
||||
}
|
||||
fmt.Printf("ai_model %s not in catalog, creating for test\n", catalogID)
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("name", jsonutils.NewString(catalogID))
|
||||
params.Set("ai_provider_id", jsonutils.NewString(providerKey))
|
||||
params.Set("model_key", jsonutils.NewString(modelKey))
|
||||
params.Set("enabled", jsonutils.JSONTrue)
|
||||
if _, err := apmodules.AiModels.Create(session, params); err != nil {
|
||||
return "", "", errors.Wrapf(err, "ai-model-create %s/%s", providerKey, modelKey)
|
||||
}
|
||||
if tracker != nil {
|
||||
tracker.createdAiModel = catalogID
|
||||
}
|
||||
return modelKey, catalogID, nil
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func RunChatTest(session *mcclient.ClientSession, opts *ChatOptions) error {
|
||||
tracker := NewResourceTracker(envKeepResources(opts.KeepResources))
|
||||
defer tracker.Cleanup(session)
|
||||
|
||||
nonInteractive := envNonInteractive(opts.NonInteractive)
|
||||
|
||||
providers, err := ListCatalogProviderKeys(session)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return errors.Error("catalog 中无 ai_provider，请先执行 aiproxy master InitDB")
|
||||
}
|
||||
|
||||
providerKey, err := promptSelectProvider(providers, opts.Provider, nonInteractive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
models, err := ListCatalogModelKeys(session, providerKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(models) == 0 {
|
||||
return errors.Errorf("provider %s 下无可用 model_key（catalog 未 seed？）", providerKey)
|
||||
}
|
||||
|
||||
modelKey, err := promptSelectModel(models, providerKey, opts.Model, nonInteractive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
apiSecret, err := promptApiKey(providerKey, opts.ApiKey, nonInteractive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
prompt := strings.TrimSpace(opts.Prompt)
|
||||
if prompt == "" {
|
||||
prompt = resolvePromptFromEnv()
|
||||
}
|
||||
if prompt == "" {
|
||||
prompt = DefaultPromptForProvider(providerKey)
|
||||
}
|
||||
|
||||
runStream := promptRunStream(opts.SkipStream, nonInteractive)
|
||||
|
||||
names := DefaultAdminNames(providerKey, "")
|
||||
if opts.KeyName != "" {
|
||||
names.KeyName = opts.KeyName
|
||||
}
|
||||
if opts.VkName != "" {
|
||||
names.VkName = opts.VkName
|
||||
}
|
||||
if opts.RoutingName != "" {
|
||||
names.RoutingName = opts.RoutingName
|
||||
}
|
||||
|
||||
catalogModelID := CatalogModelID(providerKey, modelKey)
|
||||
fmt.Println()
|
||||
fmt.Println("=== aiproxy OpenAI chat 测试 ===")
|
||||
fmt.Printf("provider: %s model: %s catalog_id: %s\n", providerKey, modelKey, catalogModelID)
|
||||
fmt.Printf("ai_key: %s virtual_key: %s routing: %s\n", names.KeyName, names.VkName, names.RoutingName)
|
||||
fmt.Println()
|
||||
|
||||
Step("1. Keystone aiproxy public endpoint")
|
||||
aiproxyURL, err := ResolveAiproxyURL(session, opts.AiproxyURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("AIPROXY_URL=%s\n", aiproxyURL)
|
||||
|
||||
Step(fmt.Sprintf("2. Catalog %s / %s", providerKey, modelKey))
|
||||
if err := VerifyCatalog(session, providerKey, modelKey, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Step("3. ai_key")
|
||||
vk, _, err := SetupAdminResources(session, tracker, providerKey, modelKey, apiSecret, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("virtual_key=%s...\n", previewText(vk, 12))
|
||||
|
||||
Step("4. POST /ai/openai/v1/chat/completions")
|
||||
client := httpClientFromSession(session)
|
||||
payload := map[string]interface{}{
|
||||
"model": modelKey,
|
||||
"messages": []map[string]string{
|
||||
{"role": "user", "content": prompt},
|
||||
},
|
||||
"max_tokens": 128,
|
||||
}
|
||||
code, body, err := postJSON(client, openAIChatURL(aiproxyURL), vk, payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("HTTP %d\n", code)
|
||||
if err := printJSONBody(body); err != nil {
|
||||
return err
|
||||
}
|
||||
if code != 200 {
|
||||
return errors.Errorf("chat request failed with HTTP %d", code)
|
||||
}
|
||||
content, err := extractOpenAIChatContent(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("content (%d chars): %s\n", len(content), previewText(content, 120))
|
||||
|
||||
if runStream {
|
||||
Step("5. POST /ai/openai/v1/chat/completions (stream=true)")
|
||||
streamPayload := map[string]interface{}{
|
||||
"model": modelKey,
|
||||
"stream": true,
|
||||
"messages": []map[string]string{
|
||||
{"role": "user", "content": prompt},
|
||||
},
|
||||
"max_tokens": 64,
|
||||
}
|
||||
streamCode, streamBody, err := postJSONStream(client, openAIChatURL(aiproxyURL), vk, streamPayload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer streamBody.Close()
|
||||
fmt.Printf("HTTP %d (stream)\n", streamCode)
|
||||
aggregated, err := aggregateSSEStream(streamBody, parseOpenAIStreamDelta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("stream content (%d chars): %s\n", len(aggregated), previewText(aggregated, 120))
|
||||
}
|
||||
|
||||
streamNote := ""
|
||||
if runStream {
|
||||
streamNote = " + stream"
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Printf("OK: aiproxy chat test passed for %s/%s (non-stream%s).\n", providerKey, modelKey, streamNote)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
|
||||
)
|
||||
|
||||
type providerConfigSnapshot struct {
|
||||
providerKey string
|
||||
config *jsonutils.JSONDict
|
||||
}
|
||||
|
||||
// ResourceTracker records resources created during a test run for automatic cleanup.
|
||||
type ResourceTracker struct {
|
||||
KeepResources bool
|
||||
|
||||
createdRouting string
|
||||
createdVirtualKey string
|
||||
createdAiKey string
|
||||
createdAiModel string
|
||||
createdProvider string
|
||||
|
||||
providerConfigRestore *providerConfigSnapshot
|
||||
}
|
||||
|
||||
func NewResourceTracker(keepResources bool) *ResourceTracker {
|
||||
return &ResourceTracker{KeepResources: keepResources}
|
||||
}
|
||||
|
||||
func envKeepResources(explicit bool) bool {
|
||||
if explicit {
|
||||
return true
|
||||
}
|
||||
return envTruthy("AIPROXY_TEST_KEEP_RESOURCES", "AIPROXY_FT_KEEP_RESOURCES")
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) hasCreated() bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
return t.createdRouting != "" ||
|
||||
t.createdVirtualKey != "" ||
|
||||
t.createdAiKey != "" ||
|
||||
t.createdAiModel != "" ||
|
||||
t.createdProvider != "" ||
|
||||
t.providerConfigRestore != nil
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) Cleanup(session *mcclient.ClientSession) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
if t.KeepResources {
|
||||
if t.hasCreated() {
|
||||
fmt.Println()
|
||||
fmt.Println("Keeping test resources (--keep-resources / AIPROXY_TEST_KEEP_RESOURCES=1)")
|
||||
t.printKeepHint()
|
||||
}
|
||||
return
|
||||
}
|
||||
if !t.hasCreated() {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
Step("cleanup test resources")
|
||||
|
||||
if t.createdRouting != "" {
|
||||
t.deleteRouting(session, t.createdRouting)
|
||||
}
|
||||
if t.createdVirtualKey != "" {
|
||||
t.deleteVirtualKey(session, t.createdVirtualKey)
|
||||
}
|
||||
if t.createdAiKey != "" {
|
||||
t.deleteAiKey(session, t.createdAiKey)
|
||||
}
|
||||
if t.createdAiModel != "" {
|
||||
t.deleteAiModel(session, t.createdAiModel)
|
||||
}
|
||||
if t.providerConfigRestore != nil {
|
||||
t.restoreProviderConfig(session)
|
||||
}
|
||||
if t.createdProvider != "" {
|
||||
t.deleteProvider(session, t.createdProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) printKeepHint() {
|
||||
if t.createdRouting != "" {
|
||||
fmt.Printf(" climc ai-routing-delete %s\n", t.createdRouting)
|
||||
}
|
||||
if t.createdVirtualKey != "" {
|
||||
fmt.Printf(" climc ai-virtual-key-delete %s\n", t.createdVirtualKey)
|
||||
}
|
||||
if t.createdAiKey != "" {
|
||||
fmt.Printf(" climc ai-key-delete %s\n", t.createdAiKey)
|
||||
}
|
||||
if t.createdAiModel != "" {
|
||||
fmt.Printf(" climc ai-model-delete %s\n", t.createdAiModel)
|
||||
}
|
||||
if t.createdProvider != "" {
|
||||
fmt.Printf(" climc ai-provider-delete %s\n", t.createdProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) deleteRouting(session *mcclient.ClientSession, name string) {
|
||||
if _, err := apmodules.AiRoutings.Delete(session, name, nil); err != nil {
|
||||
fmt.Printf("WARN: delete ai_routing %s: %v\n", name, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("deleted ai_routing %s\n", name)
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) deleteVirtualKey(session *mcclient.ClientSession, name string) {
|
||||
if _, err := apmodules.AiVirtualKeys.Delete(session, name, nil); err != nil {
|
||||
fmt.Printf("WARN: delete ai_virtual_key %s: %v\n", name, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("deleted ai_virtual_key %s\n", name)
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) deleteAiKey(session *mcclient.ClientSession, name string) {
|
||||
if _, err := apmodules.AiKeys.Delete(session, name, nil); err != nil {
|
||||
fmt.Printf("WARN: delete ai_key %s: %v\n", name, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("deleted ai_key %s\n", name)
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) deleteAiModel(session *mcclient.ClientSession, name string) {
|
||||
if _, err := apmodules.AiModels.Delete(session, name, nil); err != nil {
|
||||
fmt.Printf("WARN: delete ai_model %s: %v\n", name, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("deleted ai_model %s\n", name)
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) deleteProvider(session *mcclient.ClientSession, name string) {
|
||||
if _, err := apmodules.AiProviders.Delete(session, name, nil); err != nil {
|
||||
fmt.Printf("WARN: delete ai_provider %s: %v\n", name, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("deleted ai_provider %s\n", name)
|
||||
}
|
||||
|
||||
func (t *ResourceTracker) restoreProviderConfig(session *mcclient.ClientSession) {
|
||||
snap := t.providerConfigRestore
|
||||
if snap == nil {
|
||||
return
|
||||
}
|
||||
params := jsonutils.NewDict()
|
||||
if snap.config != nil {
|
||||
params.Set("config", snap.config)
|
||||
} else {
|
||||
params.Set("config", jsonutils.NewDict())
|
||||
}
|
||||
if _, err := apmodules.AiProviders.Update(session, snap.providerKey, params); err != nil {
|
||||
fmt.Printf("WARN: restore ai_provider %s config: %v\n", snap.providerKey, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("restored ai_provider %s config\n", snap.providerKey)
|
||||
}
|
||||
|
||||
func cloneJSONDict(obj jsonutils.JSONObject) *jsonutils.JSONDict {
|
||||
if obj == nil {
|
||||
return nil
|
||||
}
|
||||
if d, ok := obj.(*jsonutils.JSONDict); ok {
|
||||
out := jsonutils.NewDict()
|
||||
out.Update(d)
|
||||
return out
|
||||
}
|
||||
parsed, err := jsonutils.Parse([]byte(obj.String()))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if d, ok := parsed.(*jsonutils.JSONDict); ok {
|
||||
return d
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotProviderConfig(session *mcclient.ClientSession, providerKey string) (*providerConfigSnapshot, error) {
|
||||
obj, err := apmodules.AiProviders.Get(session, providerKey, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "ai_provider %s not found", providerKey)
|
||||
}
|
||||
snap := &providerConfigSnapshot{providerKey: providerKey}
|
||||
if obj.Contains("config") {
|
||||
cfg, _ := obj.Get("config")
|
||||
snap.config = cloneJSONDict(cfg)
|
||||
}
|
||||
return snap, nil
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package ft implements aiproxy end-to-end functional tests for climc subcommands.
|
||||
package ft // import "yunion.io/x/onecloud/pkg/aiproxy/ft"
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
modules "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
|
||||
)
|
||||
|
||||
func ResolveAiproxyURL(session *mcclient.ClientSession, override string) (string, error) {
|
||||
if override = strings.TrimRight(strings.TrimSpace(override), "/"); override != "" {
|
||||
return override, nil
|
||||
}
|
||||
if url := resolveAiproxyURLFromEnv(); url != "" {
|
||||
return url, nil
|
||||
}
|
||||
query := jsonutils.NewDict()
|
||||
query.Set("service", jsonutils.NewString("aiproxy"))
|
||||
query.Set("interface", jsonutils.NewString("public"))
|
||||
query.Set("limit", jsonutils.NewInt(1))
|
||||
result, err := modules.EndpointsV3.List(session, query)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "endpoint-list aiproxy public")
|
||||
}
|
||||
if len(result.Data) == 0 {
|
||||
return "", errors.Error("cannot resolve aiproxy public URL; set AIPROXY_URL")
|
||||
}
|
||||
url, _ := result.Data[0].GetString("url")
|
||||
url = strings.TrimRight(strings.TrimSpace(url), "/")
|
||||
if url == "" {
|
||||
return "", errors.Error("cannot resolve aiproxy public URL; set AIPROXY_URL")
|
||||
}
|
||||
return url, nil
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func envFirst(keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if v := strings.TrimSpace(os.Getenv(k)); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func envTruthy(keys ...string) bool {
|
||||
for _, k := range keys {
|
||||
switch strings.TrimSpace(os.Getenv(k)) {
|
||||
case "1", "true", "TRUE", "yes", "YES":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func envSkipStream(explicit bool) bool {
|
||||
if explicit {
|
||||
return true
|
||||
}
|
||||
return envTruthy("AIPROXY_TEST_SKIP_STREAM", "AIPROXY_FT_SKIP_STREAM")
|
||||
}
|
||||
|
||||
func envNonInteractive(explicit bool) bool {
|
||||
if explicit {
|
||||
return true
|
||||
}
|
||||
return envTruthy("AIPROXY_TEST_NONINTERACTIVE", "AIPROXY_FT_NONINTERACTIVE")
|
||||
}
|
||||
|
||||
func resolveProviderFromEnv() string {
|
||||
return envFirst("AIPROXY_TEST_PROVIDER", "AIPROXY_FT_PROVIDER")
|
||||
}
|
||||
|
||||
func resolveModelFromEnv() string {
|
||||
return envFirst("AIPROXY_TEST_MODEL", "AIPROXY_FT_MODEL")
|
||||
}
|
||||
|
||||
func resolvePromptFromEnv() string {
|
||||
return envFirst("AIPROXY_TEST_PROMPT", "AIPROXY_FT_PROMPT")
|
||||
}
|
||||
|
||||
func resolveApiKeyFromEnv(providerKey string) string {
|
||||
if v := envFirst("AIPROXY_TEST_API_KEY", "AIPROXY_FT_API_KEY"); v != "" {
|
||||
return v
|
||||
}
|
||||
switch providerKey {
|
||||
case "aliyun":
|
||||
return os.Getenv("DASHSCOPE_API_KEY")
|
||||
case "xiaomi":
|
||||
return os.Getenv("MIMO_API_KEY")
|
||||
case "anthropic":
|
||||
return os.Getenv("ANTHROPIC_API_KEY")
|
||||
case "openai":
|
||||
if v := os.Getenv("DEEPSEEK_API_KEY"); v != "" {
|
||||
return v
|
||||
}
|
||||
return os.Getenv("OPENAI_API_KEY")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func resolveAiproxyURLFromEnv() string {
|
||||
return strings.TrimRight(envFirst("AIPROXY_URL"), "/")
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCatalogModelID(t *testing.T) {
|
||||
if got := CatalogModelID("aliyun", "qwen-turbo"); got != "aliyun-qwen-turbo" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultModelForProvider(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"aliyun": "qwen-turbo",
|
||||
"xiaomi": "mimo-v2-flash",
|
||||
"anthropic": "claude-sonnet-4-5",
|
||||
"unknown": "",
|
||||
}
|
||||
for provider, want := range cases {
|
||||
if got := DefaultModelForProvider(provider); got != want {
|
||||
t.Fatalf("%s: got %q want %q", provider, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseOpenAIStreamDelta(t *testing.T) {
|
||||
payload := `{"choices":[{"delta":{"content":"hello"}}]}`
|
||||
delta, err := parseOpenAIStreamDelta(payload)
|
||||
if err != nil || delta != "hello" {
|
||||
t.Fatalf("delta=%q err=%v", delta, err)
|
||||
}
|
||||
_, err = parseOpenAIStreamDelta(`{"error":{"message":"fail"}}`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error event")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAnthropicStreamDelta(t *testing.T) {
|
||||
payload := `{"delta":{"text":"hi"}}`
|
||||
delta, err := parseAnthropicStreamDelta(payload)
|
||||
if err != nil || delta != "hi" {
|
||||
t.Fatalf("delta=%q err=%v", delta, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractOpenAIChatContent(t *testing.T) {
|
||||
body := []byte(`{"choices":[{"message":{"content":"answer"}}]}`)
|
||||
content, err := extractOpenAIChatContent(body)
|
||||
if err != nil || content != "answer" {
|
||||
t.Fatalf("content=%q err=%v", content, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAnthropicTextContent(t *testing.T) {
|
||||
body := []byte(`{"content":[{"type":"text","text":"hello"}]}`)
|
||||
content, err := extractAnthropicTextContent(body)
|
||||
if err != nil || content != "hello" {
|
||||
t.Fatalf("content=%q err=%v", content, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateSSEStreamOpenAI(t *testing.T) {
|
||||
input := strings.Join([]string{
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"Hel\"}}]}",
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}",
|
||||
"data: [DONE]",
|
||||
}, "\n")
|
||||
out, err := aggregateSSEStream(strings.NewReader(input), parseOpenAIStreamDelta)
|
||||
if err != nil || out != "Hello" {
|
||||
t.Fatalf("out=%q err=%v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAIChatURL(t *testing.T) {
|
||||
got := openAIChatURL("https://aiproxy.example.com/")
|
||||
if got != "https://aiproxy.example.com/ai/openai/v1/chat/completions" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicMessagesURL(t *testing.T) {
|
||||
got := anthropicMessagesURL("https://aiproxy.example.com")
|
||||
if got != "https://aiproxy.example.com/ai/anthropic/v1/messages" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceTrackerHasCreated(t *testing.T) {
|
||||
t.Parallel()
|
||||
tr := NewResourceTracker(false)
|
||||
if tr.hasCreated() {
|
||||
t.Fatal("expected empty tracker")
|
||||
}
|
||||
tr.createdAiKey = "k1"
|
||||
if !tr.hasCreated() {
|
||||
t.Fatal("expected created")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultAdminNames(t *testing.T) {
|
||||
names := DefaultAdminNames("aliyun", "")
|
||||
if names.KeyName != "aiproxy-test-aliyun" || names.VkName != "aiproxy-test-aliyun-vk" {
|
||||
t.Fatalf("%+v", names)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
func httpClientFromSession(session *mcclient.ClientSession) *http.Client {
|
||||
return session.GetClient().GetClient()
|
||||
}
|
||||
|
||||
func postJSON(client *http.Client, url, bearer string, payload interface{}) (int, []byte, error) {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+bearer)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return resp.StatusCode, nil, err
|
||||
}
|
||||
return resp.StatusCode, respBody, nil
|
||||
}
|
||||
|
||||
func postJSONStream(client *http.Client, url, bearer string, payload interface{}) (int, io.ReadCloser, error) {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+bearer)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "text/event-stream")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, nil, errors.Errorf("HTTP %d: %s", resp.StatusCode, truncateBody(b))
|
||||
}
|
||||
return resp.StatusCode, resp.Body, nil
|
||||
}
|
||||
|
||||
func openAIChatURL(baseURL string) string {
|
||||
return strings.TrimRight(baseURL, "/") + "/ai/openai/v1/chat/completions"
|
||||
}
|
||||
|
||||
func anthropicMessagesURL(baseURL string) string {
|
||||
return strings.TrimRight(baseURL, "/") + "/ai/anthropic/v1/messages"
|
||||
}
|
||||
|
||||
func Step(msg string) {
|
||||
fmt.Println()
|
||||
fmt.Println("==>", msg)
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/term"
|
||||
"yunion.io/x/pkg/errors"
|
||||
)
|
||||
|
||||
func isInteractive() bool {
|
||||
return term.IsTerminal(int(os.Stdin.Fd()))
|
||||
}
|
||||
|
||||
func promptSelectProvider(keys []string, providerOverride string, nonInteractive bool) (string, error) {
|
||||
if providerOverride = strings.TrimSpace(providerOverride); providerOverride != "" {
|
||||
for _, k := range keys {
|
||||
if k == providerOverride {
|
||||
return providerOverride, nil
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("ai_provider %s not in catalog", providerOverride)
|
||||
}
|
||||
providerOverride = resolveProviderFromEnv()
|
||||
if providerOverride != "" {
|
||||
for _, k := range keys {
|
||||
if k == providerOverride {
|
||||
return providerOverride, nil
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("ai_provider %s not in catalog", providerOverride)
|
||||
}
|
||||
if nonInteractive || !isInteractive() {
|
||||
return "", errors.Error("set --provider or AIPROXY_TEST_PROVIDER (or run in interactive terminal)")
|
||||
}
|
||||
fmt.Println("可用模型提供商 (catalog):")
|
||||
for i, k := range keys {
|
||||
fmt.Printf(" [%d] %s\n", i+1, k)
|
||||
}
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
for {
|
||||
fmt.Printf("请选择序号 [1-%d] 或直接输入 provider_key: ", len(keys))
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
choice := strings.TrimSpace(line)
|
||||
if choice == "" {
|
||||
continue
|
||||
}
|
||||
if n, err := strconv.Atoi(choice); err == nil && n >= 1 && n <= len(keys) {
|
||||
return keys[n-1], nil
|
||||
}
|
||||
for _, k := range keys {
|
||||
if k == choice {
|
||||
return choice, nil
|
||||
}
|
||||
}
|
||||
fmt.Println("无效选择，请重试。")
|
||||
}
|
||||
}
|
||||
|
||||
func promptSelectModel(models []string, providerKey, modelOverride string, nonInteractive bool) (string, error) {
|
||||
if modelOverride = strings.TrimSpace(modelOverride); modelOverride != "" {
|
||||
for _, m := range models {
|
||||
if m == modelOverride {
|
||||
return modelOverride, nil
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("model_key %s not in provider %s catalog", modelOverride, providerKey)
|
||||
}
|
||||
modelOverride = resolveModelFromEnv()
|
||||
if modelOverride != "" {
|
||||
for _, m := range models {
|
||||
if m == modelOverride {
|
||||
return modelOverride, nil
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("model_key %s not in provider %s catalog", modelOverride, providerKey)
|
||||
}
|
||||
|
||||
defaultM := DefaultModelForProvider(providerKey)
|
||||
found := false
|
||||
for _, m := range models {
|
||||
if m == defaultM {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
defaultM = models[0]
|
||||
}
|
||||
if nonInteractive || !isInteractive() {
|
||||
return defaultM, nil
|
||||
}
|
||||
|
||||
fmt.Printf("提供商 %s 的模型:\n", providerKey)
|
||||
for i, m := range models {
|
||||
fmt.Printf(" [%d] %s\n", i+1, m)
|
||||
}
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
for {
|
||||
fmt.Printf("请选择序号 [1-%d] 或输入 model_key [默认: %s]: ", len(models), defaultM)
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
choice := strings.TrimSpace(line)
|
||||
if choice == "" {
|
||||
return defaultM, nil
|
||||
}
|
||||
if n, err := strconv.Atoi(choice); err == nil && n >= 1 && n <= len(models) {
|
||||
return models[n-1], nil
|
||||
}
|
||||
for _, m := range models {
|
||||
if m == choice {
|
||||
return choice, nil
|
||||
}
|
||||
}
|
||||
fmt.Println("无效选择，请重试。")
|
||||
}
|
||||
}
|
||||
|
||||
func promptApiKey(providerKey, apiKeyOverride string, nonInteractive bool) (string, error) {
|
||||
if apiKeyOverride = strings.TrimSpace(apiKeyOverride); apiKeyOverride != "" {
|
||||
return apiKeyOverride, nil
|
||||
}
|
||||
if v := resolveApiKeyFromEnv(providerKey); v != "" {
|
||||
fmt.Println("使用环境变量中的 API Key（未回显）")
|
||||
return v, nil
|
||||
}
|
||||
if nonInteractive || !isInteractive() {
|
||||
return "", errors.Errorf("未设置 API Key：--api-key 或 AIPROXY_TEST_API_KEY 或 %s 对应的环境变量", providerKey)
|
||||
}
|
||||
fmt.Printf("请输入 %s 的 API Key（不回显）: ", providerKey)
|
||||
b, err := term.ReadPassword(int(os.Stdin.Fd()))
|
||||
fmt.Println()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := strings.TrimSpace(string(b))
|
||||
if key == "" {
|
||||
return "", errors.Error("API Key 不能为空")
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func promptRunStream(skipStream bool, nonInteractive bool) bool {
|
||||
if skipStream || envSkipStream(false) {
|
||||
return false
|
||||
}
|
||||
if v := os.Getenv("AIPROXY_TEST_SKIP_STREAM"); v == "0" {
|
||||
return true
|
||||
}
|
||||
if v := os.Getenv("AIPROXY_FT_SKIP_STREAM"); v == "0" {
|
||||
return true
|
||||
}
|
||||
if nonInteractive || !isInteractive() {
|
||||
return true
|
||||
}
|
||||
fmt.Print("是否执行流式测试 (stream=true)? [Y/n]: ")
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
switch strings.TrimSpace(strings.ToLower(line)) {
|
||||
case "n", "no":
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func promptLine(prompt, defaultVal string, nonInteractive bool) (string, error) {
|
||||
if defaultVal != "" {
|
||||
fmt.Printf("%s [%s]: ", prompt, defaultVal)
|
||||
} else {
|
||||
fmt.Printf("%s: ", prompt)
|
||||
}
|
||||
if nonInteractive || !isInteractive() {
|
||||
if defaultVal == "" {
|
||||
return "", errors.Errorf("empty input for %s in non-interactive mode", prompt)
|
||||
}
|
||||
return defaultVal, nil
|
||||
}
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
val := strings.TrimSpace(line)
|
||||
if val == "" {
|
||||
val = defaultVal
|
||||
}
|
||||
if val == "" {
|
||||
return "", errors.Errorf("empty input for %s", prompt)
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func promptYesNo(prompt string, defaultYes bool, nonInteractive bool) bool {
|
||||
if nonInteractive || !isInteractive() {
|
||||
return defaultYes
|
||||
}
|
||||
if defaultYes {
|
||||
fmt.Printf("%s [Y/n]: ", prompt)
|
||||
} else {
|
||||
fmt.Printf("%s [y/N]: ", prompt)
|
||||
}
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return defaultYes
|
||||
}
|
||||
switch strings.TrimSpace(strings.ToLower(line)) {
|
||||
case "n", "no":
|
||||
return false
|
||||
case "y", "yes":
|
||||
return true
|
||||
default:
|
||||
return defaultYes
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
type ChatOptions struct {
|
||||
Provider string `help:"provider_key from catalog"`
|
||||
Model string `help:"model_key from catalog"`
|
||||
ApiKey string `help:"upstream API key (or use env AIPROXY_TEST_API_KEY / provider-specific env)"`
|
||||
Prompt string `help:"user message content"`
|
||||
KeyName string `help:"ai_key resource name override"`
|
||||
VkName string `help:"ai_virtual_key resource name override"`
|
||||
RoutingName string `help:"ai_routing resource name override"`
|
||||
AiproxyURL string `help:"aiproxy public base URL (default: AIPROXY_URL or endpoint-list)"`
|
||||
SkipStream bool `help:"skip streaming test"`
|
||||
NonInteractive bool `help:"fail instead of prompting (also AIPROXY_TEST_NONINTERACTIVE=1)"`
|
||||
KeepResources bool `help:"keep created test resources after run (AIPROXY_TEST_KEEP_RESOURCES=1)"`
|
||||
}
|
||||
|
||||
type AnthropicOptions struct {
|
||||
Provider string `help:"provider_key (default anthropic; use openai for DeepSeek)"`
|
||||
Model string `help:"model_key"`
|
||||
ApiKey string `help:"upstream API key"`
|
||||
Prompt string `help:"user message content"`
|
||||
KeyName string `help:"ai_key resource name override"`
|
||||
VkName string `help:"ai_virtual_key resource name override"`
|
||||
RoutingName string `help:"ai_routing resource name override"`
|
||||
AiproxyURL string `help:"aiproxy public base URL"`
|
||||
UpstreamBaseURL string `help:"optional reminder: ensure ai_provider config.base_url is set"`
|
||||
SkipStream bool `help:"skip streaming test"`
|
||||
NonInteractive bool `help:"fail instead of prompting"`
|
||||
KeepResources bool `help:"keep created test resources after run (AIPROXY_TEST_KEEP_RESOURCES=1)"`
|
||||
}
|
||||
|
||||
type ProviderCreateOptions struct {
|
||||
Name string `help:"ai_provider resource name"`
|
||||
ProviderKey string `help:"provider_key (unique catalog identifier)"`
|
||||
BaseURL string `help:"config.base_url for OpenAI-compatible upstream"`
|
||||
Config string `help:"full provider config JSON (overrides --base-url)"`
|
||||
Enabled bool `help:"create with --enabled"`
|
||||
DeleteExisting bool `help:"delete existing resource with same name before create"`
|
||||
NonInteractive bool `help:"fail instead of prompting (AIPROXY_PROVIDER_TEST_NONINTERACTIVE=1)"`
|
||||
KeepResources bool `help:"keep created ai_provider after test (AIPROXY_TEST_KEEP_RESOURCES=1)"`
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
|
||||
)
|
||||
|
||||
func providerCreateNonInteractive(explicit bool) bool {
|
||||
if explicit {
|
||||
return true
|
||||
}
|
||||
return envTruthy("AIPROXY_PROVIDER_TEST_NONINTERACTIVE", "AIPROXY_PROVIDER_FT_NONINTERACTIVE")
|
||||
}
|
||||
|
||||
func buildProviderConfigJSON(configJSON, baseURL string) (jsonutils.JSONObject, error) {
|
||||
if strings.TrimSpace(configJSON) != "" {
|
||||
obj, err := jsonutils.ParseString(configJSON)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "parse config JSON")
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
baseURL = strings.TrimSpace(baseURL)
|
||||
if baseURL == "" {
|
||||
return nil, errors.Error("set --base-url or --config (or AIPROXY_PROVIDER_TEST_BASE_URL)")
|
||||
}
|
||||
return jsonutils.Marshal(map[string]string{"base_url": baseURL}), nil
|
||||
}
|
||||
|
||||
func collectProviderCreateInputs(opts *ProviderCreateOptions) error {
|
||||
nonInteractive := providerCreateNonInteractive(opts.NonInteractive)
|
||||
suffix := time.Now().Format("20060102150405")
|
||||
|
||||
if nonInteractive {
|
||||
if opts.Name == "" {
|
||||
opts.Name = fmt.Sprintf("aiproxy-provider-test-%s", suffix)
|
||||
}
|
||||
if opts.ProviderKey == "" {
|
||||
opts.ProviderKey = fmt.Sprintf("custom-test-%s", suffix)
|
||||
}
|
||||
if opts.Config == "" && opts.BaseURL == "" {
|
||||
opts.BaseURL = envFirst("AIPROXY_PROVIDER_TEST_BASE_URL", "AIPROXY_PROVIDER_FT_BASE_URL")
|
||||
}
|
||||
if opts.Config == "" {
|
||||
opts.Config = envFirst("AIPROXY_PROVIDER_TEST_CONFIG", "AIPROXY_PROVIDER_FT_CONFIG")
|
||||
}
|
||||
if !opts.Enabled {
|
||||
opts.Enabled = envTruthy("AIPROXY_PROVIDER_TEST_ENABLED") || !envTruthy("AIPROXY_PROVIDER_TEST_DISABLED")
|
||||
}
|
||||
if !opts.DeleteExisting {
|
||||
opts.DeleteExisting = envTruthy("AIPROXY_PROVIDER_TEST_DELETE_EXISTING", "AIPROXY_PROVIDER_FT_DELETE_EXISTING")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Println("=== ai_provider 创建测试 ===")
|
||||
fmt.Println("将创建自定义 ai_provider（provider_key 不可与 catalog 重复）。")
|
||||
fmt.Println()
|
||||
|
||||
var err error
|
||||
if opts.Name == "" {
|
||||
opts.Name, err = promptLine("资源名称 (climc 第一个参数 NAME)", fmt.Sprintf("aiproxy-provider-test-%s", suffix), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.ProviderKey == "" {
|
||||
opts.ProviderKey, err = promptLine("provider_key (唯一标识)", opts.Name, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.Config == "" && opts.BaseURL == "" {
|
||||
opts.BaseURL, err = promptLine("config.base_url (OpenAI 兼容上游)", "https://api.openai.com", false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !opts.Enabled && !envTruthy("AIPROXY_PROVIDER_TEST_DISABLED") {
|
||||
opts.Enabled = promptYesNo("创建后启用 (--enabled)?", true, false)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteExistingProvider(session *mcclient.ClientSession, name string, deleteIfExists bool) error {
|
||||
if _, err := apmodules.AiProviders.Get(session, name, nil); err != nil {
|
||||
return nil
|
||||
}
|
||||
if !deleteIfExists {
|
||||
return errors.Errorf("ai_provider %s already exists; use --delete-existing or AIPROXY_PROVIDER_TEST_DELETE_EXISTING=1", name)
|
||||
}
|
||||
fmt.Printf("deleting existing ai_provider %s\n", name)
|
||||
_, err := apmodules.AiProviders.Delete(session, name, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func RunProviderCreateTest(session *mcclient.ClientSession, opts *ProviderCreateOptions) error {
|
||||
tracker := NewResourceTracker(envKeepResources(opts.KeepResources))
|
||||
defer tracker.Cleanup(session)
|
||||
|
||||
if err := collectProviderCreateInputs(opts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configObj, err := buildProviderConfigJSON(opts.Config, opts.BaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := deleteExistingProvider(session, opts.Name, opts.DeleteExisting); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Step("create ai_provider")
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("name", jsonutils.NewString(opts.Name))
|
||||
params.Set("provider_key", jsonutils.NewString(opts.ProviderKey))
|
||||
params.Set("config", configObj)
|
||||
if opts.Enabled {
|
||||
params.Set("enabled", jsonutils.JSONTrue)
|
||||
}
|
||||
if _, err := apmodules.AiProviders.Create(session, params); err != nil {
|
||||
return errors.Wrap(err, "ai-provider-create")
|
||||
}
|
||||
tracker.createdProvider = opts.Name
|
||||
|
||||
Step("verify ai-provider-show")
|
||||
row, err := apmodules.AiProviders.Get(session, opts.Name, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pk, _ := row.GetString("provider_key")
|
||||
if pk != opts.ProviderKey {
|
||||
return errors.Errorf("provider_key mismatch: got %s want %s", pk, opts.ProviderKey)
|
||||
}
|
||||
if opts.BaseURL != "" {
|
||||
base, _ := row.GetString("config", "base_url")
|
||||
if base != opts.BaseURL {
|
||||
return errors.Errorf("base_url mismatch: got %s want %s", base, opts.BaseURL)
|
||||
}
|
||||
}
|
||||
enabled, _ := row.Bool("enabled")
|
||||
if opts.Enabled && !enabled {
|
||||
return errors.Error("expected enabled=true")
|
||||
}
|
||||
summary := jsonutils.NewDict()
|
||||
for _, k := range []string{"id", "name", "provider_key", "enabled"} {
|
||||
if row.Contains(k) {
|
||||
val, _ := row.Get(k)
|
||||
summary.Set(k, val)
|
||||
}
|
||||
}
|
||||
if row.Contains("config") {
|
||||
val, _ := row.Get("config")
|
||||
summary.Set("config", val)
|
||||
}
|
||||
fmt.Println(summary.PrettyString())
|
||||
|
||||
Step("verify ai-provider-list filter")
|
||||
query := jsonutils.NewDict()
|
||||
query.Set("provider_key", jsonutils.NewString(opts.ProviderKey))
|
||||
result, err := apmodules.AiProviders.List(session, query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
count := 0
|
||||
for _, item := range result.Data {
|
||||
name, _ := item.GetString("name")
|
||||
if name == opts.Name {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count < 1 {
|
||||
return errors.Error("ai-provider-list --provider-key did not return created row")
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("OK: ai_provider create test passed.")
|
||||
fmt.Printf(" name: %s\n", opts.Name)
|
||||
fmt.Printf(" provider_key: %s\n", opts.ProviderKey)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ft
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
)
|
||||
|
||||
func openAIChoiceDelta(obj jsonutils.JSONObject, field string) (string, error) {
|
||||
arr, err := obj.GetArray("choices")
|
||||
if err != nil || len(arr) == 0 {
|
||||
return "", err
|
||||
}
|
||||
if field == "delta" {
|
||||
return arr[0].GetString("delta", "content")
|
||||
}
|
||||
return arr[0].GetString("message", "content")
|
||||
}
|
||||
|
||||
func parseOpenAIStreamDelta(payload string) (string, error) {
|
||||
payload = strings.TrimSpace(payload)
|
||||
if payload == "" || payload == "[DONE]" {
|
||||
return "", nil
|
||||
}
|
||||
obj, err := jsonutils.ParseString(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if obj.Contains("error") {
|
||||
return "", errors.Errorf("stream error event: %s", payload)
|
||||
}
|
||||
return openAIChoiceDelta(obj, "delta")
|
||||
}
|
||||
|
||||
func parseAnthropicStreamDelta(payload string) (string, error) {
|
||||
payload = strings.TrimSpace(payload)
|
||||
if payload == "" || payload == "[DONE]" {
|
||||
return "", nil
|
||||
}
|
||||
obj, err := jsonutils.ParseString(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
delta, _ := obj.GetString("delta", "text")
|
||||
return delta, nil
|
||||
}
|
||||
|
||||
func aggregateSSEStream(r io.Reader, parseDelta func(string) (string, error)) (string, error) {
|
||||
scanner := bufio.NewScanner(r)
|
||||
buf := make([]byte, 0, 64*1024)
|
||||
scanner.Buffer(buf, 1024*1024)
|
||||
var aggregated strings.Builder
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if !strings.HasPrefix(line, "data:") {
|
||||
continue
|
||||
}
|
||||
payload := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
|
||||
delta, err := parseDelta(payload)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
aggregated.WriteString(delta)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out := aggregated.String()
|
||||
if out == "" {
|
||||
return "", errors.Error("empty aggregated stream content")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func extractOpenAIChatContent(body []byte) (string, error) {
|
||||
obj, err := jsonutils.Parse(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
content, err := openAIChoiceDelta(obj, "message")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "parse choices")
|
||||
}
|
||||
if content == "" {
|
||||
return "", errors.Errorf("empty choices[0].message.content: %s", truncateBody(body))
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func extractAnthropicTextContent(body []byte) (string, error) {
|
||||
obj, err := jsonutils.Parse(body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
arr, err := obj.GetArray("content")
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "parse content array")
|
||||
}
|
||||
for _, block := range arr {
|
||||
typ, _ := block.GetString("type")
|
||||
if typ == "text" {
|
||||
text, _ := block.GetString("text")
|
||||
if text != "" {
|
||||
return text, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errors.Errorf("empty anthropic text content block: %s", truncateBody(body))
|
||||
}
|
||||
|
||||
func truncateBody(body []byte) string {
|
||||
const max = 512
|
||||
if len(body) <= max {
|
||||
return string(body)
|
||||
}
|
||||
return string(body[:max]) + "..."
|
||||
}
|
||||
|
||||
func printJSONBody(body []byte) error {
|
||||
obj, err := jsonutils.Parse(body)
|
||||
if err != nil {
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Println(obj.PrettyString())
|
||||
return nil
|
||||
}
|
||||
|
||||
func previewText(s string, max int) string {
|
||||
if max <= 0 || len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + "..."
|
||||
}
|
||||
|
||||
func dumpStreamPreview(body []byte) {
|
||||
lines := bytes.Split(body, []byte("\n"))
|
||||
limit := 40
|
||||
if len(lines) < limit {
|
||||
limit = len(lines)
|
||||
}
|
||||
fmt.Println("--- stream body (first lines) ---")
|
||||
for i := 0; i < limit; i++ {
|
||||
fmt.Println(string(lines[i]))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const anthropicBasePrefix = "/ai/anthropic"
|
||||
|
||||
// anthropicBaseProbeHandler answers HEAD on the Anthropic base URL (e.g. /ai/anthropic or /ai/anthropic/).
|
||||
// appsrv SplitPath normalizes trailing slashes; one route covers both. Claude / Anthropic SDK probes
|
||||
// base URL connectivity before POST /v1/messages.
|
||||
func anthropicBaseProbeHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodHead {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// anthropicMessagesHeadHandler answers HEAD on /ai/anthropic/v1/messages for path-existence probes.
|
||||
// Returns 401 without virtual key (route exists, auth required); 204 when a key is present.
|
||||
func anthropicMessagesHeadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodHead {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
if extractVirtualKey(r) == "" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAnthropicBaseProbeHandler(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodHead, anthropicBasePrefix+"/", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
anthropicBaseProbeHandler(context.Background(), rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status=%d want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if rec.Body.Len() != 0 {
|
||||
t.Fatalf("expected empty body, got %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicMessagesHeadHandlerNoAuth(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodHead, anthropicCompatAPIPrefix+"/messages", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
anthropicMessagesHeadHandler(context.Background(), rec, req)
|
||||
if rec.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status=%d want %d", rec.Code, http.StatusUnauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicMessagesHeadHandlerWithAuth(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodHead, anthropicCompatAPIPrefix+"/messages", nil)
|
||||
req.Header.Set("Authorization", "Bearer sk-test-vk")
|
||||
rec := httptest.NewRecorder()
|
||||
anthropicMessagesHeadHandler(context.Background(), rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status=%d want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicMessagesHeadHandlerWithXAiVirtualKey(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodHead, anthropicCompatAPIPrefix+"/messages", nil)
|
||||
req.Header.Set(headerAiVirtualKey, "sk-test-vk")
|
||||
rec := httptest.NewRecorder()
|
||||
anthropicMessagesHeadHandler(context.Background(), rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status=%d want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
@@ -149,13 +149,9 @@ func chatCompletionsHandler(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
isStream, _ := dict.Bool("stream")
|
||||
prov := providers.Get(up.ProviderKey)
|
||||
if _, err := prov.BuildUpstreamRequest(&providers.ChatContext{
|
||||
ProviderKey: up.ProviderKey,
|
||||
BaseURL: up.BaseURL,
|
||||
APIKey: up.APIKey,
|
||||
UpstreamModel: up.UpstreamModel,
|
||||
}, dict, isStream); err != nil {
|
||||
prov := providers.ChatProviderForUpstream(up.ProviderKey, up.APIMode)
|
||||
chatCtx := providers.ChatContextFromUpstream(up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode)
|
||||
if _, err := prov.BuildUpstreamRequest(chatCtx, dict, isStream); err != nil {
|
||||
httperrors.InvalidInputError(ctx, w, "provider request: %v", err)
|
||||
return
|
||||
}
|
||||
@@ -250,37 +246,9 @@ func chatCompletionWithKeyFailover(
|
||||
stream bool,
|
||||
timeout time.Duration,
|
||||
) (*upstream.Response, *upstream.Error) {
|
||||
tried := make(map[string]bool)
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
var last *upstream.Error
|
||||
for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
|
||||
upReq, err := buildProviderUpstream(up, dict, stream)
|
||||
if err != nil {
|
||||
return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
|
||||
}
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
resp, uerr := upstream.ChatCompletion(reqCtx, upReq)
|
||||
cancel()
|
||||
if uerr == nil {
|
||||
models.RecordAiKeySuccess(up.AiKeyId)
|
||||
return resp, nil
|
||||
}
|
||||
last = uerr
|
||||
status := upstreamErrorStatusCode(uerr)
|
||||
models.RecordAiKeyFailure(up.AiKeyId, status)
|
||||
if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
|
||||
break
|
||||
}
|
||||
if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
|
||||
break
|
||||
}
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
}
|
||||
return nil, last
|
||||
return upstreamWithKeyFailover(ctx, up, timeout, func() (*upstream.Request, error) {
|
||||
return buildProviderUpstream(up, dict, stream)
|
||||
})
|
||||
}
|
||||
|
||||
func chatCompletionStreamWithKeyFailover(
|
||||
@@ -291,50 +259,18 @@ func chatCompletionStreamWithKeyFailover(
|
||||
prov providers.Provider,
|
||||
timeout time.Duration,
|
||||
) (<-chan upstream.StreamChunk, *upstream.Error) {
|
||||
tried := make(map[string]bool)
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
var last *upstream.Error
|
||||
for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
|
||||
upReq, err := buildProviderUpstream(up, dict, stream)
|
||||
if err != nil {
|
||||
return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
|
||||
}
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
ch, uerr := providerStreamChunks(reqCtx, up, upReq, prov)
|
||||
if uerr != nil {
|
||||
cancel()
|
||||
} else {
|
||||
ch = streamChunksWithCancel(ch, cancel)
|
||||
}
|
||||
if uerr == nil {
|
||||
return ch, nil
|
||||
}
|
||||
last = uerr
|
||||
status := upstreamErrorStatusCode(uerr)
|
||||
models.RecordAiKeyFailure(up.AiKeyId, status)
|
||||
if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
|
||||
break
|
||||
}
|
||||
if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
|
||||
break
|
||||
}
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
}
|
||||
return nil, last
|
||||
return upstreamStreamWithKeyFailover(ctx, up, timeout, func() (*upstream.Request, error) {
|
||||
return buildProviderUpstream(up, dict, stream)
|
||||
}, func(reqCtx context.Context, upReq *upstream.Request) (<-chan upstream.StreamChunk, *upstream.Error) {
|
||||
return providerStreamChunks(reqCtx, up, upReq, prov)
|
||||
})
|
||||
}
|
||||
|
||||
func buildProviderUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict, isStream bool) (*upstream.Request, error) {
|
||||
prov := providers.Get(up.ProviderKey)
|
||||
httpReq, err := prov.BuildUpstreamRequest(&providers.ChatContext{
|
||||
ProviderKey: up.ProviderKey,
|
||||
BaseURL: up.BaseURL,
|
||||
APIKey: up.APIKey,
|
||||
UpstreamModel: up.UpstreamModel,
|
||||
}, dict, isStream)
|
||||
prov := providers.ChatProviderForUpstream(up.ProviderKey, up.APIMode)
|
||||
httpReq, err := prov.BuildUpstreamRequest(providers.ChatContextFromUpstream(
|
||||
up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode,
|
||||
), dict, isStream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -347,12 +283,7 @@ func providerStreamChunks(
|
||||
upReq *upstream.Request,
|
||||
prov providers.Provider,
|
||||
) (<-chan upstream.StreamChunk, *upstream.Error) {
|
||||
chatCtx := &providers.ChatContext{
|
||||
ProviderKey: up.ProviderKey,
|
||||
BaseURL: up.BaseURL,
|
||||
APIKey: up.APIKey,
|
||||
UpstreamModel: up.UpstreamModel,
|
||||
}
|
||||
chatCtx := providers.ChatContextFromUpstream(up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode)
|
||||
if providers.OpenAIStreamPassthrough(prov, chatCtx) {
|
||||
return upstream.ChatCompletionStream(ctx, upReq)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
const (
|
||||
openaiCompatAPIPrefix = "/ai/openai/v1"
|
||||
anthropicCompatAPIPrefix = "/ai/anthropic/v1"
|
||||
openaiLongProcessTimeout = 2 * time.Hour
|
||||
openaiShortProcessTimeout = 5 * time.Minute
|
||||
)
|
||||
@@ -44,6 +45,10 @@ func InitHandlers(app *appsrv.Application, isSlave bool) {
|
||||
|
||||
app.AddHandler2("POST", openaiCompatAPIPrefix+"/chat/completions", chatCompletionsHandler, nil, "aiproxy_openai_v1_chat_completions", nil).
|
||||
SetProcessTimeout(openaiLongProcessTimeout)
|
||||
app.AddHandler2("POST", anthropicCompatAPIPrefix+"/messages", messagesHandler, nil, "aiproxy_anthropic_v1_messages", nil).
|
||||
SetProcessTimeout(openaiLongProcessTimeout)
|
||||
app.AddHandler2("HEAD", anthropicBasePrefix, anthropicBaseProbeHandler, nil, "aiproxy_anthropic_base_probe", nil)
|
||||
app.AddHandler2("HEAD", anthropicCompatAPIPrefix+"/messages", anthropicMessagesHeadHandler, nil, "aiproxy_anthropic_v1_messages_head", nil)
|
||||
app.AddHandler2("POST", openaiCompatAPIPrefix+"/completions", completionsHandler, nil, "aiproxy_openai_v1_completions", nil).
|
||||
SetProcessTimeout(openaiLongProcessTimeout)
|
||||
app.AddHandler2("POST", openaiCompatAPIPrefix+"/embeddings", embeddingsHandler, nil, "aiproxy_openai_v1_embeddings", nil).
|
||||
|
||||
@@ -0,0 +1,379 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/models"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/messages"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/upstream"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/mcclient/auth"
|
||||
)
|
||||
|
||||
// messagesHandler implements Anthropic-compatible POST /ai/anthropic/v1/messages.
|
||||
func messagesHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "only POST is supported")
|
||||
return
|
||||
}
|
||||
|
||||
defer r.Body.Close()
|
||||
raw, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "read body: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := jsonutils.Parse(raw)
|
||||
if err != nil {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "invalid JSON body: %v", err)
|
||||
return
|
||||
}
|
||||
dict, ok := body.(*jsonutils.JSONDict)
|
||||
if !ok {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "body must be a JSON object")
|
||||
return
|
||||
}
|
||||
|
||||
reqID := newMessagesReqID()
|
||||
|
||||
vk := extractVirtualKey(r)
|
||||
userCred := auth.AdminCredential()
|
||||
up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict)
|
||||
if err != nil {
|
||||
httperrors.GeneralServerError(ctx, w, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil {
|
||||
httperrors.GeneralServerError(ctx, w, err)
|
||||
return
|
||||
}
|
||||
var vkLim *api.SAiVirtualKeyLimits
|
||||
if up.MaxTokensPerRequest > 0 {
|
||||
vkLim = &api.SAiVirtualKeyLimits{
|
||||
MaxTokensPerRequest: up.MaxTokensPerRequest,
|
||||
}
|
||||
}
|
||||
if err := models.EnforceVirtualKeyMaxTokens(dict, vkLim); err != nil {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err)
|
||||
return
|
||||
}
|
||||
|
||||
adapter, err := messages.GetAdapter(up.ProviderKey, up.APIMode)
|
||||
if err != nil {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "%v", err)
|
||||
return
|
||||
}
|
||||
|
||||
isStream, _ := dict.Bool("stream")
|
||||
logMessagesClientRequest(reqID, r, dict, up, isStream)
|
||||
chatCtx := providers.ChatContextFromUpstream(up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode)
|
||||
if _, err := adapter.BuildUpstreamRequest(chatCtx, dict, isStream); err != nil {
|
||||
writeAnthropicError(ctx, w, http.StatusBadRequest, "invalid_request_error", "provider request: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
timeout := 120 * time.Second
|
||||
if isStream {
|
||||
timeout = 2 * time.Hour
|
||||
}
|
||||
|
||||
build := func() (*upstream.Request, error) {
|
||||
req, err := buildMessagesUpstream(up, adapter, dict, isStream)
|
||||
if err == nil {
|
||||
logMessagesUpstreamRequest(reqID, req)
|
||||
}
|
||||
return req, err
|
||||
}
|
||||
|
||||
if !isStream {
|
||||
prov := providers.Get(up.ProviderKey)
|
||||
resp, uerr := upstreamWithKeyFailover(ctx, up, timeout, build)
|
||||
if uerr != nil {
|
||||
logMessagesError(reqID, "upstream error status=%d body=%s", uerr.StatusCode, truncateLogBytes(uerr.Body, messagesDebugLogMax))
|
||||
writeMessagesUpstreamError(ctx, w, adapter, uerr)
|
||||
return
|
||||
}
|
||||
bodyOut := resp.Body
|
||||
logMessagesUpstreamResponse(reqID, bodyOut)
|
||||
if norm, nerr := adapter.NormalizeResponse(prov, bodyOut); nerr == nil && len(norm) > 0 {
|
||||
bodyOut = norm
|
||||
}
|
||||
logMessagesClientResponse(reqID, bodyOut)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(bodyOut)
|
||||
return
|
||||
}
|
||||
|
||||
if adapter.AnthropicStreamPassthrough() {
|
||||
ch, uerr := upstreamRawStreamWithKeyFailover(ctx, up, timeout, build, upstream.ChatCompletionStreamRaw)
|
||||
if uerr != nil {
|
||||
logMessagesError(reqID, "upstream stream error status=%d body=%s", uerr.StatusCode, truncateLogBytes(uerr.Body, messagesDebugLogMax))
|
||||
writeMessagesUpstreamError(ctx, w, adapter, uerr)
|
||||
return
|
||||
}
|
||||
writeAnthropicPassthroughStream(ctx, w, ch, up.AiKeyId, reqID)
|
||||
return
|
||||
}
|
||||
|
||||
prov := providers.Get(up.ProviderKey)
|
||||
ch, uerr := upstreamStreamWithKeyFailover(ctx, up, timeout, build, func(reqCtx context.Context, upReq *upstream.Request) (<-chan upstream.StreamChunk, *upstream.Error) {
|
||||
return messagesOpenAIStreamChunks(reqCtx, up, upReq, prov, reqID)
|
||||
})
|
||||
if uerr != nil {
|
||||
logMessagesError(reqID, "upstream stream error status=%d body=%s", uerr.StatusCode, truncateLogBytes(uerr.Body, messagesDebugLogMax))
|
||||
writeMessagesUpstreamError(ctx, w, adapter, uerr)
|
||||
return
|
||||
}
|
||||
writeAnthropicTranslatedStream(ctx, w, ch, adapter, up.UpstreamModel, up.AiKeyId, reqID)
|
||||
}
|
||||
|
||||
func buildMessagesUpstream(
|
||||
up *models.ChatUpstream,
|
||||
adapter providerapi.MessagesAdapter,
|
||||
dict *jsonutils.JSONDict,
|
||||
isStream bool,
|
||||
) (*upstream.Request, error) {
|
||||
httpReq, err := adapter.BuildUpstreamRequest(providers.ChatContextFromUpstream(
|
||||
up.ProviderKey, up.BaseURL, up.APIKey, up.UpstreamModel, up.APIMode,
|
||||
), dict, isStream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return providers.ToUpstreamRequest(httpReq, up.APIKey), nil
|
||||
}
|
||||
|
||||
func messagesOpenAIStreamChunks(
|
||||
ctx context.Context,
|
||||
up *models.ChatUpstream,
|
||||
upReq *upstream.Request,
|
||||
prov providers.Provider,
|
||||
reqID string,
|
||||
) (<-chan upstream.StreamChunk, *upstream.Error) {
|
||||
chatCtx := &providers.ChatContext{
|
||||
ProviderKey: up.ProviderKey,
|
||||
BaseURL: up.BaseURL,
|
||||
APIKey: up.APIKey,
|
||||
UpstreamModel: up.UpstreamModel,
|
||||
}
|
||||
if providers.OpenAIStreamPassthrough(prov, chatCtx) {
|
||||
ch, uerr := upstream.ChatCompletionStream(ctx, upReq)
|
||||
if uerr != nil {
|
||||
return nil, uerr
|
||||
}
|
||||
out := make(chan upstream.StreamChunk, 16)
|
||||
go func() {
|
||||
defer close(out)
|
||||
seq := 0
|
||||
for chunk := range ch {
|
||||
if len(chunk.Data) > 0 {
|
||||
seq++
|
||||
logMessagesUpstreamStreamChunk(reqID, seq, chunk.Data)
|
||||
}
|
||||
out <- chunk
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
rawCh, uerr := upstream.ChatCompletionStreamRaw(ctx, upReq)
|
||||
if uerr != nil {
|
||||
return nil, uerr
|
||||
}
|
||||
out := make(chan upstream.StreamChunk, 16)
|
||||
go func() {
|
||||
defer close(out)
|
||||
state := &providers.StreamState{Model: up.UpstreamModel}
|
||||
seq := 0
|
||||
for evt := range rawCh {
|
||||
seq++
|
||||
logMessagesUpstreamStreamChunk(reqID, seq, evt.Data)
|
||||
chunks, err := prov.ConvertStreamEvent(evt.Event, evt.Data, state)
|
||||
if err != nil {
|
||||
msg, _ := json.Marshal(map[string]interface{}{
|
||||
"error": map[string]interface{}{"message": err.Error()},
|
||||
})
|
||||
out <- upstream.StreamChunk{Data: msg}
|
||||
return
|
||||
}
|
||||
for _, c := range chunks {
|
||||
if len(c.Data) > 0 {
|
||||
out <- upstream.StreamChunk{Data: c.Data}
|
||||
}
|
||||
if c.Done {
|
||||
out <- upstream.StreamChunk{Done: true}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func writeAnthropicError(ctx context.Context, w http.ResponseWriter, status int, errType, format string, args ...interface{}) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write(openai.NewAnthropicErrorBody(errType, msg))
|
||||
}
|
||||
|
||||
func writeMessagesUpstreamError(ctx context.Context, w http.ResponseWriter, adapter providerapi.MessagesAdapter, uerr *upstream.Error) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
status := http.StatusBadGateway
|
||||
if uerr != nil && uerr.StatusCode > 0 {
|
||||
status = uerr.StatusCode
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if uerr != nil && len(uerr.Body) > 0 {
|
||||
if adapter.AnthropicStreamPassthrough() {
|
||||
_, _ = w.Write(uerr.Body)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(openai.OpenAIErrorToAnthropic(uerr.Body, status))
|
||||
return
|
||||
}
|
||||
msg := "upstream request failed"
|
||||
if uerr != nil {
|
||||
msg = uerr.Error()
|
||||
}
|
||||
_, _ = w.Write(openai.NewAnthropicErrorBody("api_error", msg))
|
||||
}
|
||||
|
||||
func writeAnthropicPassthroughStream(ctx context.Context, w http.ResponseWriter, ch <-chan upstream.RawSSEEvent, aiKeyId string, reqID string) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flushIf(w)
|
||||
|
||||
streamOK := true
|
||||
seq := 0
|
||||
for evt := range ch {
|
||||
seq++
|
||||
logMessagesClientStreamPassthrough(reqID, seq, evt.Event, evt.Data)
|
||||
if evt.Event != "" {
|
||||
_, _ = fmt.Fprintf(w, "event: %s\n", evt.Event)
|
||||
}
|
||||
if len(evt.Data) > 0 {
|
||||
_, _ = fmt.Fprintf(w, "data: %s\n\n", evt.Data)
|
||||
} else if evt.Event != "" {
|
||||
_, _ = fmt.Fprint(w, "\n")
|
||||
}
|
||||
flushIf(w)
|
||||
}
|
||||
if streamOK && aiKeyId != "" {
|
||||
models.RecordAiKeySuccess(aiKeyId)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAnthropicTranslatedStream(
|
||||
ctx context.Context,
|
||||
w http.ResponseWriter,
|
||||
ch <-chan upstream.StreamChunk,
|
||||
adapter providerapi.MessagesAdapter,
|
||||
requestModel string,
|
||||
aiKeyId string,
|
||||
reqID string,
|
||||
) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
flushIf(w)
|
||||
|
||||
state := adapter.NewStreamState(requestModel)
|
||||
streamOK := true
|
||||
outSeq := 0
|
||||
for chunk := range ch {
|
||||
if chunk.Done {
|
||||
events, err := adapter.ConvertStreamPayload(state, nil, true)
|
||||
if err != nil {
|
||||
logMessagesError(reqID, "stream convert end error: %v", err)
|
||||
streamOK = false
|
||||
break
|
||||
}
|
||||
outSeq++
|
||||
logMessagesClientStreamEvents(reqID, outSeq, events)
|
||||
writeAnthropicSSEEvents(w, events)
|
||||
break
|
||||
}
|
||||
if len(chunk.Data) == 0 {
|
||||
continue
|
||||
}
|
||||
if isAnthropicUpstreamErrorChunk(chunk.Data) {
|
||||
logMessagesError(reqID, "upstream stream error chunk: %s", truncateLogBytes(chunk.Data, messagesDebugLogMax))
|
||||
streamOK = false
|
||||
if aiKeyId != "" {
|
||||
models.RecordAiKeyFailure(aiKeyId, parseUpstreamErrorStatus(chunk.Data))
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, "event: error\ndata: %s\n\n", openai.OpenAIErrorToAnthropic(chunk.Data, http.StatusBadGateway))
|
||||
flushIf(w)
|
||||
break
|
||||
}
|
||||
events, err := adapter.ConvertStreamPayload(state, chunk.Data, false)
|
||||
if err != nil {
|
||||
logMessagesError(reqID, "stream convert error: %v upstream_chunk=%s", err, truncateLogBytes(chunk.Data, messagesDebugLogMax))
|
||||
streamOK = false
|
||||
break
|
||||
}
|
||||
outSeq++
|
||||
logMessagesClientStreamEvents(reqID, outSeq, events)
|
||||
writeAnthropicSSEEvents(w, events)
|
||||
}
|
||||
if streamOK && aiKeyId != "" {
|
||||
models.RecordAiKeySuccess(aiKeyId)
|
||||
}
|
||||
}
|
||||
|
||||
func writeAnthropicSSEEvents(w http.ResponseWriter, events []providerapi.AnthropicStreamChunk) {
|
||||
for _, evt := range events {
|
||||
if evt.Event != "" {
|
||||
_, _ = fmt.Fprintf(w, "event: %s\n", evt.Event)
|
||||
}
|
||||
if len(evt.Data) > 0 {
|
||||
_, _ = fmt.Fprintf(w, "data: %s\n\n", string(evt.Data))
|
||||
} else if evt.Event != "" {
|
||||
_, _ = fmt.Fprint(w, "\n")
|
||||
}
|
||||
flushIf(w)
|
||||
}
|
||||
}
|
||||
|
||||
func isAnthropicUpstreamErrorChunk(data []byte) bool {
|
||||
var wrap struct {
|
||||
Error interface{} `json:"error"`
|
||||
}
|
||||
return json.Unmarshal(data, &wrap) == nil && wrap.Error != nil
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/log"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/models"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/upstream"
|
||||
)
|
||||
|
||||
const messagesDebugLogMax = 4096
|
||||
|
||||
func newMessagesReqID() string {
|
||||
return fmt.Sprintf("%08x", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func logMessagesClientRequest(reqID string, r *http.Request, body *jsonutils.JSONDict, up *models.ChatUpstream, stream bool) {
|
||||
model, _ := body.GetString("model")
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] client request method=%s path=%s query=%s stream=%v provider=%s upstream_model=%s vk=%s body=%s",
|
||||
reqID,
|
||||
r.Method,
|
||||
r.URL.Path,
|
||||
r.URL.RawQuery,
|
||||
stream,
|
||||
up.ProviderKey,
|
||||
up.UpstreamModel,
|
||||
maskSecret(extractVirtualKey(r)),
|
||||
truncateLogBytes([]byte(body.String()), messagesDebugLogMax),
|
||||
)
|
||||
if model != "" && model != up.UpstreamModel {
|
||||
log.Debugf("aiproxy messages [%s] client model=%q routed upstream_model=%q", reqID, model, up.UpstreamModel)
|
||||
}
|
||||
}
|
||||
|
||||
func logMessagesUpstreamRequest(reqID string, req *upstream.Request) {
|
||||
if req == nil {
|
||||
return
|
||||
}
|
||||
url := strings.TrimSpace(req.URL)
|
||||
if url == "" {
|
||||
url = strings.TrimSpace(req.BaseURL)
|
||||
}
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] upstream request url=%s body=%s",
|
||||
reqID,
|
||||
url,
|
||||
truncateLogBytes(req.Body, messagesDebugLogMax),
|
||||
)
|
||||
}
|
||||
|
||||
func logMessagesUpstreamStreamChunk(reqID string, seq int, data []byte) {
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] upstream stream chunk #%d: %s",
|
||||
reqID,
|
||||
seq,
|
||||
truncateLogBytes(data, messagesDebugLogMax),
|
||||
)
|
||||
}
|
||||
|
||||
func logMessagesClientStreamEvents(reqID string, seq int, events []providerapi.AnthropicStreamChunk) {
|
||||
if len(events) == 0 {
|
||||
log.Debugf("aiproxy messages [%s] client stream out #%d: (no anthropic events)", reqID, seq)
|
||||
return
|
||||
}
|
||||
for i, evt := range events {
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] client stream out #%d.%d event=%s data=%s",
|
||||
reqID,
|
||||
seq,
|
||||
i,
|
||||
evt.Event,
|
||||
truncateLogBytes(evt.Data, messagesDebugLogMax),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func logMessagesUpstreamResponse(reqID string, body []byte) {
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] upstream response body=%s",
|
||||
reqID,
|
||||
truncateLogBytes(body, messagesDebugLogMax),
|
||||
)
|
||||
}
|
||||
|
||||
func logMessagesClientResponse(reqID string, body []byte) {
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] client response body=%s",
|
||||
reqID,
|
||||
truncateLogBytes(body, messagesDebugLogMax),
|
||||
)
|
||||
}
|
||||
|
||||
func logMessagesClientStreamPassthrough(reqID string, seq int, event string, data []byte) {
|
||||
log.Debugf(
|
||||
"aiproxy messages [%s] client stream passthrough #%d event=%s data=%s",
|
||||
reqID,
|
||||
seq,
|
||||
event,
|
||||
truncateLogBytes(data, messagesDebugLogMax),
|
||||
)
|
||||
}
|
||||
|
||||
func logMessagesError(reqID string, format string, args ...interface{}) {
|
||||
log.Debugf("aiproxy messages [%s] "+format, append([]interface{}{reqID}, args...)...)
|
||||
}
|
||||
|
||||
func truncateLogBytes(b []byte, max int) string {
|
||||
if len(b) == 0 {
|
||||
return ""
|
||||
}
|
||||
s := strings.TrimSpace(string(b))
|
||||
if max <= 0 || len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max] + fmt.Sprintf("...(truncated, total=%d)", len(s))
|
||||
}
|
||||
|
||||
func maskSecret(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if len(s) <= 8 {
|
||||
return "***"
|
||||
}
|
||||
return s[:4] + "..." + s[len(s)-4:]
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/models"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/upstream"
|
||||
)
|
||||
|
||||
type upstreamRequestBuilder func() (*upstream.Request, error)
|
||||
|
||||
func upstreamWithKeyFailover(
|
||||
ctx context.Context,
|
||||
up *models.ChatUpstream,
|
||||
timeout time.Duration,
|
||||
build upstreamRequestBuilder,
|
||||
) (*upstream.Response, *upstream.Error) {
|
||||
tried := make(map[string]bool)
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
var last *upstream.Error
|
||||
for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
|
||||
upReq, err := build()
|
||||
if err != nil {
|
||||
return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
|
||||
}
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
resp, uerr := upstream.ChatCompletion(reqCtx, upReq)
|
||||
cancel()
|
||||
if uerr == nil {
|
||||
models.RecordAiKeySuccess(up.AiKeyId)
|
||||
return resp, nil
|
||||
}
|
||||
last = uerr
|
||||
status := upstreamErrorStatusCode(uerr)
|
||||
models.RecordAiKeyFailure(up.AiKeyId, status)
|
||||
if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
|
||||
break
|
||||
}
|
||||
if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
|
||||
break
|
||||
}
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
type streamChunkProducer func(ctx context.Context, upReq *upstream.Request) (<-chan upstream.StreamChunk, *upstream.Error)
|
||||
|
||||
func upstreamStreamWithKeyFailover(
|
||||
ctx context.Context,
|
||||
up *models.ChatUpstream,
|
||||
timeout time.Duration,
|
||||
build upstreamRequestBuilder,
|
||||
produce streamChunkProducer,
|
||||
) (<-chan upstream.StreamChunk, *upstream.Error) {
|
||||
tried := make(map[string]bool)
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
var last *upstream.Error
|
||||
for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
|
||||
upReq, err := build()
|
||||
if err != nil {
|
||||
return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
|
||||
}
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
ch, uerr := produce(reqCtx, upReq)
|
||||
if uerr != nil {
|
||||
cancel()
|
||||
} else {
|
||||
ch = streamChunksWithCancel(ch, cancel)
|
||||
}
|
||||
if uerr == nil {
|
||||
return ch, nil
|
||||
}
|
||||
last = uerr
|
||||
status := upstreamErrorStatusCode(uerr)
|
||||
models.RecordAiKeyFailure(up.AiKeyId, status)
|
||||
if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
|
||||
break
|
||||
}
|
||||
if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
|
||||
break
|
||||
}
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
type rawSSEProducer func(ctx context.Context, upReq *upstream.Request) (<-chan upstream.RawSSEEvent, *upstream.Error)
|
||||
|
||||
func upstreamRawStreamWithKeyFailover(
|
||||
ctx context.Context,
|
||||
up *models.ChatUpstream,
|
||||
timeout time.Duration,
|
||||
build upstreamRequestBuilder,
|
||||
produce rawSSEProducer,
|
||||
) (<-chan upstream.RawSSEEvent, *upstream.Error) {
|
||||
tried := make(map[string]bool)
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
var last *upstream.Error
|
||||
for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
|
||||
upReq, err := build()
|
||||
if err != nil {
|
||||
return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
|
||||
}
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
ch, uerr := produce(reqCtx, upReq)
|
||||
if uerr != nil {
|
||||
cancel()
|
||||
} else {
|
||||
ch = rawSSEWithCancel(ch, cancel)
|
||||
}
|
||||
if uerr == nil {
|
||||
return ch, nil
|
||||
}
|
||||
last = uerr
|
||||
status := upstreamErrorStatusCode(uerr)
|
||||
models.RecordAiKeyFailure(up.AiKeyId, status)
|
||||
if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
|
||||
break
|
||||
}
|
||||
if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
|
||||
break
|
||||
}
|
||||
if up.AiKeyId != "" {
|
||||
tried[up.AiKeyId] = true
|
||||
}
|
||||
}
|
||||
return nil, last
|
||||
}
|
||||
|
||||
func rawSSEWithCancel(ch <-chan upstream.RawSSEEvent, cancel context.CancelFunc) <-chan upstream.RawSSEEvent {
|
||||
out := make(chan upstream.RawSSEEvent, 16)
|
||||
go func() {
|
||||
defer cancel()
|
||||
defer close(out)
|
||||
for evt := range ch {
|
||||
out <- evt
|
||||
}
|
||||
}()
|
||||
return out
|
||||
}
|
||||
@@ -122,7 +122,7 @@ type resolvedUpstreamAPIKey struct {
|
||||
// MaxAiKeyFailoverAttempts is how many alternate ai_key rows to try per chat request.
|
||||
const MaxAiKeyFailoverAttempts = 8
|
||||
|
||||
// resolveUpstreamAPIKey picks an ai_key (weighted + dynamic penalty) or provider.config api_key.
|
||||
// resolveUpstreamAPIKey picks an enabled ai_key for the provider (weighted + dynamic penalty).
|
||||
func resolveUpstreamAPIKey(prov *SAiProvider, modelKey string) (*resolvedUpstreamAPIKey, error) {
|
||||
return resolveUpstreamAPIKeyExcluding(prov, modelKey, nil)
|
||||
}
|
||||
@@ -175,14 +175,7 @@ func resolveUpstreamAPIKeyExcluding(prov *SAiProvider, modelKey string, exclude
|
||||
if hasSecretKey {
|
||||
return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "no available ai_key for catalog model %q (check weight, cooldown, allowed_model_keys)", modelKey)
|
||||
}
|
||||
if prov.Config == nil {
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty")
|
||||
}
|
||||
apiKey := strings.TrimSpace(prov.Config.ResolvedAPIKey())
|
||||
if apiKey == "" {
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "set api_key on ai_provider or add an enabled ai_key with secret for this provider")
|
||||
}
|
||||
return &resolvedUpstreamAPIKey{Secret: apiKey}, nil
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "add an enabled ai_key with secret for this provider")
|
||||
}
|
||||
|
||||
// RepickUpstreamAPIKey selects another ai_key for the same provider/model, excluding already tried ids.
|
||||
|
||||
@@ -16,14 +16,17 @@ package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
"yunion.io/x/sqlchemy"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/cloudcommon/db"
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
"yunion.io/x/onecloud/pkg/util/stringutils2"
|
||||
)
|
||||
@@ -35,7 +38,7 @@ type SAiProvider struct {
|
||||
// ProviderKey selects the upstream adapter implementation (e.g. openai, vllm, aliyun).
|
||||
// Multiple ai_provider rows may share the same provider_key with different config.
|
||||
ProviderKey string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
|
||||
// Config is a JSON snapshot of provider connectivity (base_url, optional api_key).
|
||||
// Config is a JSON snapshot of provider connectivity (base_url, api_mode).
|
||||
Config *api.SAiProviderConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"`
|
||||
// LlmDeploymentId and LlmId link this provider to an llm_deployment replica (set by llm sync).
|
||||
LlmDeploymentId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user" index:"true"`
|
||||
@@ -125,10 +128,25 @@ func (manager *SAiProviderManager) ValidateCreateData(
|
||||
input.ProviderKey = pk
|
||||
|
||||
input.Config = normalizeAiProviderConfig(input.Config)
|
||||
if err := validateAiProviderConfig(input.Config); err != nil {
|
||||
if err := validateAiProviderConfig(input.Config, input.ProviderKey); err != nil {
|
||||
return input, err
|
||||
}
|
||||
|
||||
input.Secret = strings.TrimSpace(input.Secret)
|
||||
|
||||
if api.IsCustomProviderKey(input.ProviderKey) {
|
||||
if input.Secret == "" {
|
||||
return input, errors.Wrap(httperrors.ErrInputParameter, "secret is required for provider_key custom")
|
||||
}
|
||||
if input.Config == nil || strings.TrimSpace(input.Config.ResolvedBaseURL()) == "" {
|
||||
return input, errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required for provider_key custom")
|
||||
}
|
||||
}
|
||||
|
||||
if input.Enabled == nil && input.Disabled == nil {
|
||||
input.SetEnabled()
|
||||
}
|
||||
|
||||
if strings.TrimSpace(input.Name) == "" {
|
||||
input.Name = pk
|
||||
}
|
||||
@@ -139,9 +157,97 @@ func (manager *SAiProviderManager) ValidateCreateData(
|
||||
return input, err
|
||||
}
|
||||
|
||||
if strings.TrimSpace(input.Secret) != "" {
|
||||
var err error
|
||||
input.ModelKeys, err = normalizeProviderModelKeys(input.ModelKeys)
|
||||
if err != nil {
|
||||
return input, err
|
||||
}
|
||||
if len(input.ModelKeys) == 0 {
|
||||
return input, errors.Wrap(httperrors.ErrInputParameter, "model_keys is required when secret is provided")
|
||||
}
|
||||
if err := probeProviderConnectivity(ctx, input.ProviderKey, input.Secret, input.Config); err != nil {
|
||||
return input, err
|
||||
}
|
||||
}
|
||||
|
||||
return input, nil
|
||||
}
|
||||
|
||||
func (p *SAiProvider) CustomizeCreate(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
ownerId mcclient.IIdentityProvider,
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) error {
|
||||
if err := rejectProviderConfigAPIKeyInJSON(data); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.SEnabledStatusStandaloneResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data)
|
||||
}
|
||||
|
||||
func (p *SAiProvider) PostCreate(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
ownerId mcclient.IIdentityProvider,
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) {
|
||||
p.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
|
||||
|
||||
input := api.AiProviderCreateInput{}
|
||||
if err := data.Unmarshal(&input); err != nil {
|
||||
log.Errorf("ai_provider PostCreate unmarshal: %v", err)
|
||||
return
|
||||
}
|
||||
if len(input.ModelKeys) > 0 {
|
||||
if err := createSelectedProviderModels(ctx, userCred, ownerId, p, input.ModelKeys); err != nil {
|
||||
log.Errorf("ai_provider %s create selected models: %v", p.Id, err)
|
||||
}
|
||||
} else if err := createCatalogModelsForUserProvider(ctx, userCred, ownerId, p); err != nil {
|
||||
log.Errorf("ai_provider %s create catalog models: %v", p.Id, err)
|
||||
}
|
||||
|
||||
secret := strings.TrimSpace(input.Secret)
|
||||
if secret == "" {
|
||||
secret, _ = data.GetString("secret")
|
||||
secret = strings.TrimSpace(secret)
|
||||
}
|
||||
if secret == "" {
|
||||
return
|
||||
}
|
||||
if err := createInitialAiKey(ctx, userCred, ownerId, p, secret); err != nil {
|
||||
log.Errorf("ai_provider %s create initial ai_key: %v", p.Id, err)
|
||||
}
|
||||
}
|
||||
|
||||
func createInitialAiKey(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
ownerId mcclient.IIdentityProvider,
|
||||
prov *SAiProvider,
|
||||
secret string,
|
||||
) error {
|
||||
if prov == nil || strings.TrimSpace(prov.Id) == "" {
|
||||
return errors.Error("ai_provider is nil or has no id")
|
||||
}
|
||||
secret = strings.TrimSpace(secret)
|
||||
if secret == "" {
|
||||
return nil
|
||||
}
|
||||
dataDict := jsonutils.NewDict()
|
||||
dataDict.Set("ai_provider_id", jsonutils.NewString(prov.Id))
|
||||
dataDict.Set("secret", jsonutils.NewString(secret))
|
||||
dataDict.Set("weight", jsonutils.NewInt(1))
|
||||
dataDict.Set("enabled", jsonutils.JSONTrue)
|
||||
dataDict.Set("generate_name", jsonutils.NewString(fmt.Sprintf("%s-key", prov.Name)))
|
||||
if _, err := db.DoCreate(AiKeyManager, ctx, userCred, nil, dataDict, ownerId); err != nil {
|
||||
return errors.Wrap(err, "create ai_key for provider")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *SAiProvider) ValidateUpdateData(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
@@ -164,7 +270,11 @@ func (p *SAiProvider) ValidateUpdateData(
|
||||
|
||||
if input.Config != nil {
|
||||
input.Config = normalizeAiProviderConfig(input.Config)
|
||||
if err := validateAiProviderConfig(input.Config); err != nil {
|
||||
pk := strings.TrimSpace(input.ProviderKey)
|
||||
if pk == "" {
|
||||
pk = p.ProviderKey
|
||||
}
|
||||
if err := validateAiProviderConfig(input.Config, pk); err != nil {
|
||||
return input, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
func TestRejectProviderConfigAPIKeyInJSON(t *testing.T) {
|
||||
obj, err := jsonutils.Parse([]byte(`{"config":{"api_key":"sk-test"}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
err = rejectProviderConfigAPIKeyInJSON(obj)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for config.api_key")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "config.api_key is not supported") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectProviderConfigAPIKeyInJSONAllowsValidConfig(t *testing.T) {
|
||||
obj, err := jsonutils.Parse([]byte(`{"config":{"base_url":"https://api.openai.com"}}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
err = rejectProviderConfigAPIKeyInJSON(obj)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAiProviderCreateInputDefaultsEnabled(t *testing.T) {
|
||||
input := api.AiProviderCreateInput{
|
||||
ProviderKey: "deepseek",
|
||||
Secret: "sk-test",
|
||||
}
|
||||
if input.Enabled != nil {
|
||||
t.Fatal("expected enabled unset before defaulting")
|
||||
}
|
||||
if input.Disabled != nil {
|
||||
t.Fatal("expected disabled unset before defaulting")
|
||||
}
|
||||
input.SetEnabled()
|
||||
if input.Enabled == nil || !*input.Enabled {
|
||||
t.Fatal("expected enabled=true after SetEnabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveUpstreamAPIKeyNilProvider(t *testing.T) {
|
||||
_, err := resolveUpstreamAPIKey(nil, "gpt-4o")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for nil provider")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAiProviderConfigCustomRequiresBaseURL(t *testing.T) {
|
||||
err := validateAiProviderConfig(nil, api.ProviderKeyCustom)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when custom has no config")
|
||||
}
|
||||
err = validateAiProviderConfig(&api.SAiProviderConfig{}, api.ProviderKeyCustom)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when custom base_url empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAiProviderConfigCustomAnthropic(t *testing.T) {
|
||||
cfg := &api.SAiProviderConfig{
|
||||
BaseURL: "https://llm.example.com/anthropic",
|
||||
APIMode: api.ProviderAPIModeAnthropic,
|
||||
}
|
||||
if err := validateAiProviderConfig(cfg, api.ProviderKeyCustom); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -109,6 +109,8 @@ func (manager *SAiRoutingModelManager) FetchCustomizeColumns(
|
||||
for i := range objs {
|
||||
rows[i].StandaloneResourceDetails = baseRows[i]
|
||||
rm := objs[i].(*SAiRoutingModel)
|
||||
rows[i].Id = rm.Id
|
||||
rows[i].Name = rm.Name
|
||||
rows[i].AiRoutingId = rm.AiRoutingId
|
||||
rows[i].AiProviderId = rm.AiProviderId
|
||||
rows[i].AiModelId = rm.AiModelId
|
||||
|
||||
@@ -140,6 +140,8 @@ func (manager *SAiRoutingManager) FetchCustomizeColumns(
|
||||
for j := range entries {
|
||||
e := entries[j]
|
||||
rows[i].RoutingModels[j] = api.AiRoutingModelDetails{
|
||||
Id: e.Id,
|
||||
Name: e.Name,
|
||||
AiRoutingId: e.AiRoutingId,
|
||||
AiProviderId: e.AiProviderId,
|
||||
AiModelId: e.AiModelId,
|
||||
@@ -367,3 +369,12 @@ func (routing *SAiRouting) PerformSetModels(
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (routing *SAiRouting) CustomizeDelete(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) error {
|
||||
return deleteAiRoutingModels(ctx, routing.Id)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
@@ -110,10 +111,23 @@ func catalogModelKeySlug(modelKey string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
func validateAiProviderConfig(cfg *api.SAiProviderConfig) error {
|
||||
func validateAiProviderConfig(cfg *api.SAiProviderConfig, providerKey string) error {
|
||||
pk := strings.ToLower(strings.TrimSpace(providerKey))
|
||||
if api.IsCustomProviderKey(pk) {
|
||||
if cfg == nil || cfg.IsZero() || cfg.ResolvedBaseURL() == "" {
|
||||
return errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required for provider_key custom")
|
||||
}
|
||||
}
|
||||
if cfg == nil || cfg.IsZero() {
|
||||
return nil
|
||||
}
|
||||
if !api.IsValidProviderAPIMode(cfg.APIMode) {
|
||||
return errors.Wrap(httperrors.ErrInputParameter, "config.api_mode must be openai or anthropic")
|
||||
}
|
||||
mode := cfg.ResolvedAPIMode()
|
||||
if mode == api.ProviderAPIModeAnthropic && !api.SupportsDualAPIMode(pk) {
|
||||
return errors.Wrapf(httperrors.ErrInputParameter, "config.api_mode=anthropic is not supported for provider_key %q", providerKey)
|
||||
}
|
||||
baseURL := cfg.ResolvedBaseURL()
|
||||
if baseURL == "" {
|
||||
return nil
|
||||
@@ -139,12 +153,34 @@ func normalizeAiProviderConfig(cfg *api.SAiProviderConfig) *api.SAiProviderConfi
|
||||
if base := cfg.ResolvedBaseURL(); base != "" {
|
||||
out.BaseURL = base
|
||||
}
|
||||
if key := cfg.ResolvedAPIKey(); key != "" {
|
||||
out.APIKey = key
|
||||
if mode := strings.TrimSpace(cfg.APIMode); mode != "" {
|
||||
out.APIMode = strings.ToLower(mode)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectProviderConfigAPIKeyInJSON(data jsonutils.JSONObject) error {
|
||||
if data == nil {
|
||||
return nil
|
||||
}
|
||||
dict, ok := data.(*jsonutils.JSONDict)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
cfgVal, err := dict.Get("config")
|
||||
if err != nil || cfgVal == nil {
|
||||
return nil
|
||||
}
|
||||
cfgDict, ok := cfgVal.(*jsonutils.JSONDict)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if cfgDict.Contains("api_key") {
|
||||
return errors.Wrap(httperrors.ErrInputParameter, "config.api_key is not supported, use secret and ai_keys")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureAiModelKeyUniquePerProvider(ctx context.Context, providerId, modelKey, excludeId string) error {
|
||||
q := AiModelManager.Query().Equals("ai_provider_id", providerId).Equals("model_key", modelKey)
|
||||
if excludeId != "" {
|
||||
|
||||
@@ -19,100 +19,21 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/apis"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/cloudcommon/db"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
// standardCatalogProviderKeys lists built-in provider_key values seeded at InitDB.
|
||||
var standardCatalogProviderKeys = []string{
|
||||
"anthropic",
|
||||
"azure",
|
||||
"bedrock",
|
||||
"cerebras",
|
||||
"cohere",
|
||||
"gemini",
|
||||
"groq",
|
||||
"mistral",
|
||||
"ollama",
|
||||
"openai",
|
||||
"parasail",
|
||||
"perplexity",
|
||||
"sgl",
|
||||
"vertex",
|
||||
"openrouter",
|
||||
"elevenlabs",
|
||||
"huggingface",
|
||||
"nebius",
|
||||
"xai",
|
||||
"replicate",
|
||||
"vllm",
|
||||
"runway",
|
||||
"fireworks",
|
||||
"aliyun",
|
||||
"baidu",
|
||||
"xiaomi",
|
||||
}
|
||||
|
||||
// defaultPublicBaseURL returns a well-known public API base for OpenAI-compatible upstreams.
|
||||
// Empty string means no default in catalog (user must set base_url in provider config).
|
||||
func defaultPublicBaseURL(providerKey string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(providerKey)) {
|
||||
case "openai":
|
||||
return "https://api.openai.com"
|
||||
case "anthropic":
|
||||
return "https://api.anthropic.com"
|
||||
case "azure", "bedrock", "sgl", "vertex":
|
||||
return ""
|
||||
case "cerebras":
|
||||
return "https://api.cerebras.ai"
|
||||
case "cohere":
|
||||
return "https://api.cohere.ai"
|
||||
case "gemini":
|
||||
return "https://generativelanguage.googleapis.com/v1beta"
|
||||
case "groq":
|
||||
return "https://api.groq.com/openai"
|
||||
case "mistral":
|
||||
return "https://api.mistral.ai"
|
||||
case "ollama":
|
||||
return "http://127.0.0.1:11434"
|
||||
case "vllm":
|
||||
return "http://127.0.0.1:8000"
|
||||
case "parasail":
|
||||
return "https://api.parasail.io"
|
||||
case "perplexity":
|
||||
return "https://api.perplexity.ai"
|
||||
case "openrouter":
|
||||
return "https://openrouter.ai/api"
|
||||
case "elevenlabs":
|
||||
return "https://api.elevenlabs.io"
|
||||
case "huggingface":
|
||||
return "https://router.huggingface.co"
|
||||
case "nebius":
|
||||
return "https://api.tokenfactory.nebius.com"
|
||||
case "xai":
|
||||
return "https://api.x.ai"
|
||||
case "replicate":
|
||||
return "https://api.replicate.com"
|
||||
case "runway":
|
||||
return "https://api.dev.runwayml.com"
|
||||
case "fireworks":
|
||||
return "https://api.fireworks.ai/inference"
|
||||
case "aliyun":
|
||||
return "https://dashscope.aliyuncs.com/compatible-mode"
|
||||
case "baidu":
|
||||
return "https://qianfan.baidubce.com/v2"
|
||||
case "xiaomi":
|
||||
return "https://api.xiaomimimo.com"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
var standardCatalogProviderKeys = api.StandardCatalogProviderKeys
|
||||
|
||||
func standardProviderConfig(providerKey string) *api.SAiProviderConfig {
|
||||
if u := defaultPublicBaseURL(providerKey); u != "" {
|
||||
if u := api.DefaultPublicBaseURL(providerKey); u != "" {
|
||||
return &api.SAiProviderConfig{BaseURL: u}
|
||||
}
|
||||
return nil
|
||||
@@ -216,6 +137,97 @@ func ensureSeedProvider(ctx context.Context, providerKey string) error {
|
||||
return ensureSeedModelsEntries(ctx, providerId, providerKey, catalogSeedModelsForProvider(providerKey))
|
||||
}
|
||||
|
||||
func providerModelExists(providerId, modelKey string) (bool, error) {
|
||||
cnt, err := AiModelManager.Query().
|
||||
Equals("ai_provider_id", providerId).
|
||||
Equals("model_key", modelKey).
|
||||
CountWithError()
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, "count ai_model for provider")
|
||||
}
|
||||
return cnt > 0, nil
|
||||
}
|
||||
|
||||
func createUserProviderModel(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
ownerId mcclient.IIdentityProvider,
|
||||
prov *SAiProvider,
|
||||
modelKey, description string,
|
||||
) error {
|
||||
if prov == nil || strings.TrimSpace(prov.Id) == "" {
|
||||
return errors.Error("ai_provider is nil or has no id")
|
||||
}
|
||||
modelKey = strings.TrimSpace(modelKey)
|
||||
if modelKey == "" {
|
||||
return errors.Error("model_key is empty")
|
||||
}
|
||||
exists, err := providerModelExists(prov.Id, modelKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
return nil
|
||||
}
|
||||
dataDict := jsonutils.NewDict()
|
||||
dataDict.Set("ai_provider_id", jsonutils.NewString(prov.Id))
|
||||
dataDict.Set("model_key", jsonutils.NewString(modelKey))
|
||||
dataDict.Set("enabled", jsonutils.JSONTrue)
|
||||
dataDict.Set("generate_name", jsonutils.NewString(defaultAiModelName(prov.Name, modelKey)))
|
||||
if desc := strings.TrimSpace(description); desc != "" {
|
||||
dataDict.Set("description", jsonutils.NewString(desc))
|
||||
}
|
||||
if _, err := db.DoCreate(AiModelManager, ctx, userCred, nil, dataDict, ownerId); err != nil {
|
||||
return errors.Wrapf(err, "create ai_model %q for provider %s", modelKey, prov.Id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSelectedProviderModels(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
ownerId mcclient.IIdentityProvider,
|
||||
prov *SAiProvider,
|
||||
modelKeys []string,
|
||||
) error {
|
||||
if prov == nil {
|
||||
return nil
|
||||
}
|
||||
for _, modelKey := range modelKeys {
|
||||
if err := createUserProviderModel(ctx, userCred, ownerId, prov, modelKey, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// createCatalogModelsForUserProvider inserts built-in catalog model rows for a newly created public SaaS provider.
|
||||
func createCatalogModelsForUserProvider(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
ownerId mcclient.IIdentityProvider,
|
||||
prov *SAiProvider,
|
||||
) error {
|
||||
if prov == nil {
|
||||
return nil
|
||||
}
|
||||
pk := strings.TrimSpace(prov.ProviderKey)
|
||||
if !api.HasDefaultPublicBaseURL(pk) {
|
||||
return nil
|
||||
}
|
||||
entries := catalogSeedModelsForProvider(pk)
|
||||
if len(entries) == 0 {
|
||||
return createUserProviderModel(ctx, userCred, ownerId, prov, placeholderCatalogModelKey,
|
||||
"Catalog seed placeholder; replace with concrete model_key values or use a provider with a built-in catalog.")
|
||||
}
|
||||
for i := range entries {
|
||||
if err := createUserProviderModel(ctx, userCred, ownerId, prov, entries[i].ModelKey, entries[i].Description); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SeedStandardCatalog inserts built-in ai_provider / ai_model catalog rows on first init only.
|
||||
// Existing rows are left unchanged so user config survives service restarts.
|
||||
func SeedStandardCatalog(ctx context.Context) error {
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
|
||||
package models
|
||||
|
||||
import api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
|
||||
// catalogSeedModel is one row to insert into ai_models when seeding a standard provider.
|
||||
// ModelKey is the id sent to the upstream API (no "provider/" prefix).
|
||||
type catalogSeedModel struct {
|
||||
@@ -26,7 +28,7 @@ type catalogSeedModel struct {
|
||||
// Providers without a list return nil and the seeder inserts model_key "default".
|
||||
func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
switch providerKey {
|
||||
case "anthropic":
|
||||
case api.ProviderKeyAnthropic:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "claude-opus-4-20250514", Description: "Anthropic Claude Opus 4"},
|
||||
{ModelKey: "claude-sonnet-4-20250514", Description: "Anthropic Claude Sonnet 4"},
|
||||
@@ -36,58 +38,69 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "claude-3-opus-20240229", Description: "Anthropic Claude 3 Opus"},
|
||||
{ModelKey: "claude-3-haiku-20240307", Description: "Anthropic Claude 3 Haiku"},
|
||||
}
|
||||
case "azure":
|
||||
// Azure OpenAI uses deployment names; these match common Azure OpenAI deployment ids.
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyAzure:
|
||||
// // Azure OpenAI uses deployment names; these match common Azure OpenAI deployment ids.
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "gpt-4o", Description: "Azure OpenAI GPT-4o deployment"},
|
||||
// {ModelKey: "gpt-4o-mini", Description: "Azure OpenAI GPT-4o mini deployment"},
|
||||
// {ModelKey: "gpt-4", Description: "Azure OpenAI GPT-4 deployment"},
|
||||
// {ModelKey: "gpt-35-turbo", Description: "Azure OpenAI GPT-3.5 Turbo deployment"},
|
||||
// {ModelKey: "o3-mini", Description: "Azure OpenAI o3-mini deployment"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyBedrock:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "anthropic.claude-3-5-sonnet-20241022-v2:0", Description: "Bedrock Claude 3.5 Sonnet"},
|
||||
// {ModelKey: "anthropic.claude-3-5-haiku-20241022-v1:0", Description: "Bedrock Claude 3.5 Haiku"},
|
||||
// {ModelKey: "anthropic.claude-3-opus-20240229-v1:0", Description: "Bedrock Claude 3 Opus"},
|
||||
// {ModelKey: "anthropic.claude-3-sonnet-20240229-v1:0", Description: "Bedrock Claude 3 Sonnet"},
|
||||
// {ModelKey: "anthropic.claude-3-haiku-20240307-v1:0", Description: "Bedrock Claude 3 Haiku"},
|
||||
// {ModelKey: "meta.llama3-70b-instruct-v1:0", Description: "Bedrock Llama 3 70B Instruct"},
|
||||
// {ModelKey: "meta.llama3-8b-instruct-v1:0", Description: "Bedrock Llama 3 8B Instruct"},
|
||||
// {ModelKey: "mistral.mistral-large-2402-v1:0", Description: "Bedrock Mistral Large"},
|
||||
// {ModelKey: "amazon.titan-text-express-v1", Description: "Bedrock Amazon Titan Text Express"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyCerebras:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "llama3.1-8b", Description: "Cerebras Llama 3.1 8B"},
|
||||
// {ModelKey: "llama3.1-70b", Description: "Cerebras Llama 3.1 70B"},
|
||||
// {ModelKey: "llama-3.3-70b", Description: "Cerebras Llama 3.3 70B"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyCohere:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "command-r-plus", Description: "Cohere Command R+"},
|
||||
// {ModelKey: "command-r", Description: "Cohere Command R"},
|
||||
// {ModelKey: "command-a", Description: "Cohere Command A"},
|
||||
// {ModelKey: "command", Description: "Cohere Command"},
|
||||
// {ModelKey: "command-light", Description: "Cohere Command Light"},
|
||||
// {ModelKey: "embed-english-v3.0", Description: "Cohere Embed English v3"},
|
||||
// {ModelKey: "embed-multilingual-v3.0", Description: "Cohere Embed Multilingual v3"},
|
||||
// }
|
||||
case api.ProviderKeyDeepseek:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "gpt-4o", Description: "Azure OpenAI GPT-4o deployment"},
|
||||
{ModelKey: "gpt-4o-mini", Description: "Azure OpenAI GPT-4o mini deployment"},
|
||||
{ModelKey: "gpt-4", Description: "Azure OpenAI GPT-4 deployment"},
|
||||
{ModelKey: "gpt-35-turbo", Description: "Azure OpenAI GPT-3.5 Turbo deployment"},
|
||||
{ModelKey: "o3-mini", Description: "Azure OpenAI o3-mini deployment"},
|
||||
{ModelKey: "deepseek-v4-flash", Description: "DeepSeek-V4-Flash; 1M context; high concurrency (2500); cost-efficient default"},
|
||||
{ModelKey: "deepseek-v4-pro", Description: "DeepSeek-V4-Pro; 1M context; frontier reasoning/coding/agents (500 concurrency)"},
|
||||
}
|
||||
case "bedrock":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "anthropic.claude-3-5-sonnet-20241022-v2:0", Description: "Bedrock Claude 3.5 Sonnet"},
|
||||
{ModelKey: "anthropic.claude-3-5-haiku-20241022-v1:0", Description: "Bedrock Claude 3.5 Haiku"},
|
||||
{ModelKey: "anthropic.claude-3-opus-20240229-v1:0", Description: "Bedrock Claude 3 Opus"},
|
||||
{ModelKey: "anthropic.claude-3-sonnet-20240229-v1:0", Description: "Bedrock Claude 3 Sonnet"},
|
||||
{ModelKey: "anthropic.claude-3-haiku-20240307-v1:0", Description: "Bedrock Claude 3 Haiku"},
|
||||
{ModelKey: "meta.llama3-70b-instruct-v1:0", Description: "Bedrock Llama 3 70B Instruct"},
|
||||
{ModelKey: "meta.llama3-8b-instruct-v1:0", Description: "Bedrock Llama 3 8B Instruct"},
|
||||
{ModelKey: "mistral.mistral-large-2402-v1:0", Description: "Bedrock Mistral Large"},
|
||||
{ModelKey: "amazon.titan-text-express-v1", Description: "Bedrock Amazon Titan Text Express"},
|
||||
}
|
||||
case "cerebras":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "llama3.1-8b", Description: "Cerebras Llama 3.1 8B"},
|
||||
{ModelKey: "llama3.1-70b", Description: "Cerebras Llama 3.1 70B"},
|
||||
{ModelKey: "llama-3.3-70b", Description: "Cerebras Llama 3.3 70B"},
|
||||
}
|
||||
case "cohere":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "command-r-plus", Description: "Cohere Command R+"},
|
||||
{ModelKey: "command-r", Description: "Cohere Command R"},
|
||||
{ModelKey: "command-a", Description: "Cohere Command A"},
|
||||
{ModelKey: "command", Description: "Cohere Command"},
|
||||
{ModelKey: "command-light", Description: "Cohere Command Light"},
|
||||
{ModelKey: "embed-english-v3.0", Description: "Cohere Embed English v3"},
|
||||
{ModelKey: "embed-multilingual-v3.0", Description: "Cohere Embed Multilingual v3"},
|
||||
}
|
||||
case "elevenlabs":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "eleven_multilingual_v2", Description: "ElevenLabs multilingual v2"},
|
||||
{ModelKey: "eleven_turbo_v2_5", Description: "ElevenLabs Turbo v2.5"},
|
||||
{ModelKey: "eleven_flash_v2_5", Description: "ElevenLabs Flash v2.5"},
|
||||
{ModelKey: "eleven_multilingual_v1", Description: "ElevenLabs multilingual v1"},
|
||||
}
|
||||
case "fireworks":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "accounts/fireworks/models/llama-v3p1-8b-instruct", Description: "Fireworks Llama 3.1 8B Instruct"},
|
||||
{ModelKey: "accounts/fireworks/models/llama-v3p1-70b-instruct", Description: "Fireworks Llama 3.1 70B Instruct"},
|
||||
{ModelKey: "accounts/fireworks/models/llama-v3p3-70b-instruct", Description: "Fireworks Llama 3.3 70B Instruct"},
|
||||
{ModelKey: "accounts/fireworks/models/mixtral-8x7b-instruct", Description: "Fireworks Mixtral 8x7B Instruct"},
|
||||
}
|
||||
case "gemini":
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyElevenlabs:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "eleven_multilingual_v2", Description: "ElevenLabs multilingual v2"},
|
||||
// {ModelKey: "eleven_turbo_v2_5", Description: "ElevenLabs Turbo v2.5"},
|
||||
// {ModelKey: "eleven_flash_v2_5", Description: "ElevenLabs Flash v2.5"},
|
||||
// {ModelKey: "eleven_multilingual_v1", Description: "ElevenLabs multilingual v1"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyFireworks:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "accounts/fireworks/models/llama-v3p1-8b-instruct", Description: "Fireworks Llama 3.1 8B Instruct"},
|
||||
// {ModelKey: "accounts/fireworks/models/llama-v3p1-70b-instruct", Description: "Fireworks Llama 3.1 70B Instruct"},
|
||||
// {ModelKey: "accounts/fireworks/models/llama-v3p3-70b-instruct", Description: "Fireworks Llama 3.3 70B Instruct"},
|
||||
// {ModelKey: "accounts/fireworks/models/mixtral-8x7b-instruct", Description: "Fireworks Mixtral 8x7B Instruct"},
|
||||
// }
|
||||
case api.ProviderKeyGemini:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "gemini-2.0-flash", Description: "Google Gemini 2.0 Flash"},
|
||||
{ModelKey: "gemini-2.0-flash-lite", Description: "Google Gemini 2.0 Flash-Lite"},
|
||||
@@ -96,7 +109,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "gemini-1.5-flash-8b", Description: "Google Gemini 1.5 Flash 8B"},
|
||||
{ModelKey: "gemini-embedding-001", Description: "Google Gemini Embedding 001"},
|
||||
}
|
||||
case "groq":
|
||||
case api.ProviderKeyGroq:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "llama-3.3-70b-versatile", Description: "Groq Llama 3.3 70B Versatile"},
|
||||
{ModelKey: "llama-3.1-8b-instant", Description: "Groq Llama 3.1 8B Instant"},
|
||||
@@ -104,14 +117,14 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "mixtral-8x7b-32768", Description: "Groq Mixtral 8x7B"},
|
||||
{ModelKey: "gemma2-9b-it", Description: "Groq Gemma2 9B IT"},
|
||||
}
|
||||
case "huggingface":
|
||||
case api.ProviderKeyHuggingface:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "HF Llama 3.1 8B Instruct"},
|
||||
{ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "HF Llama 3.1 70B Instruct"},
|
||||
{ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "HF Mistral 7B Instruct"},
|
||||
{ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "HF Qwen2.5 72B Instruct"},
|
||||
}
|
||||
case "mistral":
|
||||
case api.ProviderKeyMistral:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "mistral-large-latest", Description: "Mistral Large (latest)"},
|
||||
{ModelKey: "mistral-small-latest", Description: "Mistral Small (latest)"},
|
||||
@@ -122,13 +135,14 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "mixtral-8x22b", Description: "Mistral Mixtral 8x22B"},
|
||||
{ModelKey: "mixtral-8x7b", Description: "Mistral Mixtral 8x7B"},
|
||||
}
|
||||
case "nebius":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "deepseek-ai/DeepSeek-V3", Description: "Nebius DeepSeek V3"},
|
||||
{ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "Nebius Qwen2.5 72B Instruct"},
|
||||
{ModelKey: "meta-llama/Llama-3.3-70B-Instruct", Description: "Nebius Llama 3.3 70B Instruct"},
|
||||
}
|
||||
case "ollama":
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyNebius:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "deepseek-ai/DeepSeek-V3", Description: "Nebius DeepSeek V3"},
|
||||
// {ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "Nebius Qwen2.5 72B Instruct"},
|
||||
// {ModelKey: "meta-llama/Llama-3.3-70B-Instruct", Description: "Nebius Llama 3.3 70B Instruct"},
|
||||
// }
|
||||
case api.ProviderKeyOllama:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "llama3.2", Description: "Ollama Llama 3.2"},
|
||||
{ModelKey: "llama3.1", Description: "Ollama Llama 3.1"},
|
||||
@@ -137,7 +151,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "codellama", Description: "Ollama Code Llama"},
|
||||
{ModelKey: "phi3", Description: "Ollama Phi 3"},
|
||||
}
|
||||
case "vllm":
|
||||
case api.ProviderKeyVLLM:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "vLLM Llama 3.1 8B Instruct"},
|
||||
{ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "vLLM Llama 3.1 70B Instruct"},
|
||||
@@ -145,7 +159,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "vLLM Qwen2.5 72B Instruct"},
|
||||
{ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "vLLM Mistral 7B Instruct"},
|
||||
}
|
||||
case "openai":
|
||||
case api.ProviderKeyOpenAI:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "gpt-5-nano", Description: "OpenAI GPT-5 nano"},
|
||||
{ModelKey: "gpt-5-mini", Description: "OpenAI GPT-5 mini"},
|
||||
@@ -176,7 +190,7 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "text-embedding-3-large", Description: "OpenAI text-embedding-3-large"},
|
||||
{ModelKey: "text-embedding-ada-002", Description: "OpenAI text-embedding-ada-002"},
|
||||
}
|
||||
case "openrouter":
|
||||
case api.ProviderKeyOpenrouter:
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "openai/gpt-4o", Description: "OpenRouter OpenAI GPT-4o"},
|
||||
{ModelKey: "openai/gpt-4o-mini", Description: "OpenRouter OpenAI GPT-4o mini"},
|
||||
@@ -186,46 +200,53 @@ func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
|
||||
{ModelKey: "meta-llama/llama-3.3-70b-instruct", Description: "OpenRouter Llama 3.3 70B Instruct"},
|
||||
{ModelKey: "mistralai/mistral-large", Description: "OpenRouter Mistral Large"},
|
||||
}
|
||||
case "perplexity":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "sonar", Description: "Perplexity Sonar"},
|
||||
{ModelKey: "sonar-pro", Description: "Perplexity Sonar Pro"},
|
||||
{ModelKey: "sonar-reasoning", Description: "Perplexity Sonar Reasoning"},
|
||||
{ModelKey: "llama-3.1-sonar-small-128k-online", Description: "Perplexity Llama 3.1 Sonar Small online"},
|
||||
{ModelKey: "llama-3.1-sonar-large-128k-online", Description: "Perplexity Llama 3.1 Sonar Large online"},
|
||||
}
|
||||
case "replicate":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "meta/meta-llama-3-8b-instruct", Description: "Replicate Meta Llama 3 8B Instruct"},
|
||||
{ModelKey: "meta/meta-llama-3-70b-instruct", Description: "Replicate Meta Llama 3 70B Instruct"},
|
||||
{ModelKey: "mistralai/mixtral-8x7b-instruct-v0.1", Description: "Replicate Mixtral 8x7B Instruct"},
|
||||
}
|
||||
case "runway":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "gen3a_turbo", Description: "Runway Gen-3 Alpha Turbo"},
|
||||
{ModelKey: "gen3a", Description: "Runway Gen-3 Alpha"},
|
||||
{ModelKey: "gen4_aleph", Description: "Runway Gen-4 Aleph"},
|
||||
}
|
||||
case "vertex":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "gemini-2.0-flash", Description: "Vertex AI Gemini 2.0 Flash"},
|
||||
{ModelKey: "gemini-1.5-pro", Description: "Vertex AI Gemini 1.5 Pro"},
|
||||
{ModelKey: "gemini-1.5-flash", Description: "Vertex AI Gemini 1.5 Flash"},
|
||||
{ModelKey: "publishers/google/models/gemini-1.5-pro", Description: "Vertex publisher path Gemini 1.5 Pro"},
|
||||
}
|
||||
case "xai":
|
||||
return []catalogSeedModel{
|
||||
{ModelKey: "grok-3", Description: "xAI Grok 3"},
|
||||
{ModelKey: "grok-3-mini", Description: "xAI Grok 3 mini"},
|
||||
{ModelKey: "grok-2-latest", Description: "xAI Grok 2 latest"},
|
||||
{ModelKey: "grok-2-1212", Description: "xAI Grok 2 1212"},
|
||||
{ModelKey: "grok-beta", Description: "xAI Grok beta"},
|
||||
}
|
||||
case "aliyun":
|
||||
return aliyunQwenSeedModels()
|
||||
case "baidu":
|
||||
return baiduErnieSeedModels()
|
||||
case "xiaomi":
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyPerplexity:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "sonar", Description: "Perplexity Sonar"},
|
||||
// {ModelKey: "sonar-pro", Description: "Perplexity Sonar Pro"},
|
||||
// {ModelKey: "sonar-reasoning", Description: "Perplexity Sonar Reasoning"},
|
||||
// {ModelKey: "llama-3.1-sonar-small-128k-online", Description: "Perplexity Llama 3.1 Sonar Small online"},
|
||||
// {ModelKey: "llama-3.1-sonar-large-128k-online", Description: "Perplexity Llama 3.1 Sonar Large online"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyReplicate:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "meta/meta-llama-3-8b-instruct", Description: "Replicate Meta Llama 3 8B Instruct"},
|
||||
// {ModelKey: "meta/meta-llama-3-70b-instruct", Description: "Replicate Meta Llama 3 70B Instruct"},
|
||||
// {ModelKey: "mistralai/mixtral-8x7b-instruct-v0.1", Description: "Replicate Mixtral 8x7B Instruct"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyRunway:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "gen3a_turbo", Description: "Runway Gen-3 Alpha Turbo"},
|
||||
// {ModelKey: "gen3a", Description: "Runway Gen-3 Alpha"},
|
||||
// {ModelKey: "gen4_aleph", Description: "Runway Gen-4 Aleph"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyVertex:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "gemini-2.0-flash", Description: "Vertex AI Gemini 2.0 Flash"},
|
||||
// {ModelKey: "gemini-1.5-pro", Description: "Vertex AI Gemini 1.5 Pro"},
|
||||
// {ModelKey: "gemini-1.5-flash", Description: "Vertex AI Gemini 1.5 Flash"},
|
||||
// {ModelKey: "publishers/google/models/gemini-1.5-pro", Description: "Vertex publisher path Gemini 1.5 Pro"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyXai:
|
||||
// return []catalogSeedModel{
|
||||
// {ModelKey: "grok-3", Description: "xAI Grok 3"},
|
||||
// {ModelKey: "grok-3-mini", Description: "xAI Grok 3 mini"},
|
||||
// {ModelKey: "grok-2-latest", Description: "xAI Grok 2 latest"},
|
||||
// {ModelKey: "grok-2-1212", Description: "xAI Grok 2 1212"},
|
||||
// {ModelKey: "grok-beta", Description: "xAI Grok beta"},
|
||||
// }
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyAliyun:
|
||||
// return aliyunQwenSeedModels()
|
||||
// disabled: uncommon provider
|
||||
// case api.ProviderKeyBaidu:
|
||||
// return baiduErnieSeedModels()
|
||||
case api.ProviderKeyXiaomi:
|
||||
return xiaomiMimoSeedModels()
|
||||
default:
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
func TestCatalogSeedModelsForPublicProviders(t *testing.T) {
|
||||
publicKeys := []string{
|
||||
api.ProviderKeyOpenAI,
|
||||
api.ProviderKeyDeepseek,
|
||||
api.ProviderKeyAnthropic,
|
||||
api.ProviderKeyGemini,
|
||||
}
|
||||
for _, pk := range publicKeys {
|
||||
if !api.HasDefaultPublicBaseURL(pk) {
|
||||
t.Fatalf("%q should have default public base URL", pk)
|
||||
}
|
||||
entries := catalogSeedModelsForProvider(pk)
|
||||
if len(entries) == 0 {
|
||||
t.Fatalf("expected catalog models for public provider %q", pk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSeedModelsSkippedForSelfHostedProviders(t *testing.T) {
|
||||
selfHosted := []string{api.ProviderKeyOllama, api.ProviderKeyVLLM, api.ProviderKeySGLang}
|
||||
for _, pk := range selfHosted {
|
||||
if api.HasDefaultPublicBaseURL(pk) {
|
||||
t.Fatalf("%q should not be treated as public SaaS provider", pk)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,7 @@ type ChatUpstream struct {
|
||||
ProviderKey string
|
||||
AiProviderId string
|
||||
AiKeyId string
|
||||
APIMode string
|
||||
|
||||
// VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row.
|
||||
VirtualKeyId string
|
||||
@@ -212,7 +213,7 @@ func resolveCatalogModelFromRouting(
|
||||
// 1. ai_virtual_key (auth + project scope)
|
||||
// 2. ai_routing in that project (model_key exact match first, then model_pattern / optional proxy-node scope, priority)
|
||||
// 3. ai_routing_model -> ai_provider + ai_model
|
||||
// 4. ai_key rows for that provider matching the catalog model_key (weight), else provider.config api_key
|
||||
// 4. ai_key rows for that provider matching the catalog model_key (weight)
|
||||
func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, virtualKey string, body *jsonutils.JSONDict) (*ChatUpstream, error) {
|
||||
vk, err := loadEnabledVirtualKey(virtualKey)
|
||||
if err != nil {
|
||||
@@ -246,7 +247,7 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
|
||||
if prov.Config == nil {
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty")
|
||||
}
|
||||
baseURL := prov.Config.ResolvedBaseURL()
|
||||
baseURL := prov.Config.EffectiveBaseURL(prov.ProviderKey)
|
||||
if baseURL == "" {
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config must include base_url")
|
||||
}
|
||||
@@ -256,13 +257,15 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model.model_key is empty")
|
||||
}
|
||||
|
||||
apiMode := prov.Config.ResolvedAPIMode()
|
||||
|
||||
// Keys are scoped to ai_provider; routing on each ai_key matches the resolved catalog model_key.
|
||||
keyRes, err := resolveUpstreamAPIKey(prov, upstreamModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if keyRes == nil || keyRes.Secret == "" {
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no api_key for ai_provider and catalog model")
|
||||
return nil, errors.Wrap(httperrors.ErrInvalidStatus, "add an enabled ai_key with secret for this provider")
|
||||
}
|
||||
|
||||
up := &ChatUpstream{
|
||||
@@ -273,6 +276,7 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
|
||||
AiProviderId: prov.Id,
|
||||
AiKeyId: keyRes.AiKeyId,
|
||||
VirtualKeyId: vk.Id,
|
||||
APIMode: apiMode,
|
||||
}
|
||||
if vk.Limits != nil {
|
||||
up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/ft"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/upstream"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
)
|
||||
|
||||
const (
|
||||
providerCreateConnectivityTimeout = 15 * time.Second
|
||||
providerTestConnectivityTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
func normalizeProviderModelKeys(modelKeys []string) ([]string, error) {
|
||||
seen := make(map[string]struct{}, len(modelKeys))
|
||||
out := make([]string, 0, len(modelKeys))
|
||||
for _, key := range modelKeys {
|
||||
mk, err := validateAiModelKey(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := seen[mk]; ok {
|
||||
continue
|
||||
}
|
||||
seen[mk] = struct{}{}
|
||||
out = append(out, mk)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func catalogModelKeysForConnectivity(providerKey string) []string {
|
||||
entries := catalogSeedModelsForProvider(providerKey)
|
||||
if len(entries) == 0 {
|
||||
return []string{placeholderCatalogModelKey}
|
||||
}
|
||||
keys := make([]string, len(entries))
|
||||
for i := range entries {
|
||||
keys[i] = entries[i].ModelKey
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func probeModelForConnectivity(providerKey string) string {
|
||||
entries := catalogSeedModelsForProvider(providerKey)
|
||||
if len(entries) > 0 && strings.TrimSpace(entries[0].ModelKey) != "" {
|
||||
return entries[0].ModelKey
|
||||
}
|
||||
if model := strings.TrimSpace(ft.DefaultModelForProvider(providerKey)); model != "" {
|
||||
return model
|
||||
}
|
||||
return placeholderCatalogModelKey
|
||||
}
|
||||
|
||||
func shouldFallbackToChatFromListModels(uerr *upstream.Error) bool {
|
||||
if uerr == nil {
|
||||
return true
|
||||
}
|
||||
switch uerr.StatusCode {
|
||||
case http.StatusUnauthorized, http.StatusForbidden:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func probeChatConnectivity(ctx context.Context, providerKey, secret string, cfg *api.SAiProviderConfig) error {
|
||||
if cfg == nil {
|
||||
cfg = &api.SAiProviderConfig{}
|
||||
}
|
||||
effectiveURL := cfg.EffectiveBaseURL(providerKey)
|
||||
if effectiveURL == "" {
|
||||
return errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required (no default for this provider_key)")
|
||||
}
|
||||
apiMode := cfg.ResolvedAPIMode()
|
||||
probeModel := probeModelForConnectivity(providerKey)
|
||||
|
||||
userMsg := jsonutils.NewDict()
|
||||
userMsg.Set("role", jsonutils.NewString("user"))
|
||||
userMsg.Set("content", jsonutils.NewString("ping"))
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString(probeModel))
|
||||
body.Set("max_tokens", jsonutils.NewInt(1))
|
||||
body.Set("messages", jsonutils.NewArray(userMsg))
|
||||
|
||||
prov := providers.ChatProviderForUpstream(providerKey, apiMode)
|
||||
httpReq, err := prov.BuildUpstreamRequest(providers.ChatContextFromUpstream(
|
||||
providerKey, effectiveURL, secret, probeModel, apiMode,
|
||||
), body, false)
|
||||
if err != nil {
|
||||
return httperrors.NewInputParameterError("failed to build chat probe request: %s", err.Error())
|
||||
}
|
||||
_, uerr := upstream.ChatCompletion(ctx, providers.ToUpstreamRequest(httpReq, secret))
|
||||
if uerr != nil {
|
||||
return connectivityErrorFromUpstream(uerr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func listProviderModels(ctx context.Context, providerKey, secret string, cfg *api.SAiProviderConfig, timeout time.Duration) ([]string, bool, error) {
|
||||
pk, err := validateAiCatalogIdentifier("provider_key", providerKey, maxAiProviderKeyLen)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
secret = strings.TrimSpace(secret)
|
||||
if secret == "" {
|
||||
return nil, false, errors.Wrap(httperrors.ErrInputParameter, "secret is required for connectivity test")
|
||||
}
|
||||
cfg = normalizeAiProviderConfig(cfg)
|
||||
if err := validateAiProviderConfig(cfg, pk); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if cfg == nil {
|
||||
cfg = &api.SAiProviderConfig{}
|
||||
}
|
||||
effectiveURL := cfg.EffectiveBaseURL(pk)
|
||||
if effectiveURL == "" {
|
||||
return nil, false, errors.Wrap(httperrors.ErrInputParameter, "config.base_url is required (no default for this provider_key)")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = providerCreateConnectivityTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
resp, uerr := upstream.ListModels(ctx, effectiveURL, secret)
|
||||
if uerr == nil {
|
||||
modelKeys, err := upstream.ParseModelsListBody(resp.Body)
|
||||
if err == nil && len(modelKeys) > 0 {
|
||||
return modelKeys, false, nil
|
||||
}
|
||||
} else if !shouldFallbackToChatFromListModels(uerr) {
|
||||
return nil, false, connectivityErrorFromUpstream(uerr)
|
||||
}
|
||||
|
||||
if err := probeChatConnectivity(ctx, pk, secret, cfg); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return catalogModelKeysForConnectivity(pk), true, nil
|
||||
}
|
||||
|
||||
func probeProviderConnectivity(ctx context.Context, providerKey, secret string, cfg *api.SAiProviderConfig) error {
|
||||
_, _, err := listProviderModels(ctx, providerKey, secret, cfg, providerCreateConnectivityTimeout)
|
||||
return err
|
||||
}
|
||||
|
||||
func connectivityErrorFromUpstream(uerr *upstream.Error) error {
|
||||
if uerr == nil {
|
||||
return nil
|
||||
}
|
||||
msg := strings.TrimSpace(uerr.Message)
|
||||
if msg == "" {
|
||||
msg = uerr.Error()
|
||||
}
|
||||
switch uerr.StatusCode {
|
||||
case http.StatusUnauthorized, http.StatusForbidden:
|
||||
return httperrors.NewInputParameterError("invalid API key: %s", msg)
|
||||
case http.StatusNotFound:
|
||||
return httperrors.NewInputParameterError("API URL not found (check config.base_url): %s", msg)
|
||||
default:
|
||||
return httperrors.NewInputParameterError("upstream connectivity test failed: %s", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func providerUpstreamModels(modelKeys []string) []api.AiProviderUpstreamModel {
|
||||
out := make([]api.AiProviderUpstreamModel, len(modelKeys))
|
||||
for i, mk := range modelKeys {
|
||||
out[i] = api.AiProviderUpstreamModel{ModelKey: mk}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PerformTestConnectivity probes upstream list-models without persisting an ai_provider row.
|
||||
func (manager *SAiProviderManager) PerformTestConnectivity(
|
||||
ctx context.Context,
|
||||
userCred mcclient.TokenCredential,
|
||||
query jsonutils.JSONObject,
|
||||
input api.AiProviderTestConnectivityInput,
|
||||
) (api.AiProviderTestConnectivityOutput, error) {
|
||||
out := api.AiProviderTestConnectivityOutput{}
|
||||
modelKeys, fromCatalog, err := listProviderModels(ctx, input.ProviderKey, input.Secret, input.Config, providerTestConnectivityTimeout)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
out.Ok = true
|
||||
if fromCatalog {
|
||||
out.Message = "connectivity test passed (catalog models)"
|
||||
out.ModelsSource = api.AiProviderModelsSourceCatalog
|
||||
} else {
|
||||
out.Message = "connectivity test passed"
|
||||
out.ModelsSource = api.AiProviderModelsSourceUpstream
|
||||
}
|
||||
out.Models = providerUpstreamModels(modelKeys)
|
||||
return out, nil
|
||||
}
|
||||
@@ -25,6 +25,7 @@ type ChatContext struct {
|
||||
BaseURL string
|
||||
APIKey string
|
||||
UpstreamModel string
|
||||
APIMode string
|
||||
}
|
||||
|
||||
// HTTPRequest is the wire-format call sent to an upstream provider.
|
||||
@@ -82,3 +83,24 @@ type CompletionsProvider interface {
|
||||
NormalizeCompletionsResponse(body []byte) ([]byte, error)
|
||||
OpenAICompletionsStreamPassthrough() bool
|
||||
}
|
||||
|
||||
// AnthropicStreamChunk is one Anthropic Messages API SSE event.
|
||||
type AnthropicStreamChunk struct {
|
||||
Event string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// AnthropicStreamState carries per-stream metadata for Anthropic Messages streaming.
|
||||
type AnthropicStreamState struct {
|
||||
RequestModel string
|
||||
}
|
||||
|
||||
// MessagesAdapter converts Anthropic Messages API requests to upstream HTTP calls and
|
||||
// normalizes responses back to Anthropic format.
|
||||
type MessagesAdapter interface {
|
||||
BuildUpstreamRequest(ctx *ChatContext, body *jsonutils.JSONDict, stream bool) (*HTTPRequest, error)
|
||||
NormalizeResponse(prov Provider, body []byte) ([]byte, error)
|
||||
AnthropicStreamPassthrough() bool
|
||||
NewStreamState(requestModel string) interface{}
|
||||
ConvertStreamPayload(state interface{}, payload []byte, endOfStream bool) ([]AnthropicStreamChunk, error)
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
func patchEnableThinkingFalse(body *jsonutils.JSONDict, stream bool) {
|
||||
@@ -33,5 +34,5 @@ func patchEnableThinkingFalse(body *jsonutils.JSONDict, stream bool) {
|
||||
|
||||
// New returns the Aliyun (DashScope compatible-mode) provider adapter.
|
||||
func New() providerapi.Provider {
|
||||
return openai.NewCompat("aliyun", patchEnableThinkingFalse)
|
||||
return openai.NewCompat(api.ProviderKeyAliyun, patchEnableThinkingFalse)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
const apiVersion = "2023-06-01"
|
||||
@@ -36,7 +37,7 @@ func New() providerapi.Provider {
|
||||
}
|
||||
|
||||
func (p *provider) Key() string {
|
||||
return "anthropic"
|
||||
return api.ProviderKeyAnthropic
|
||||
}
|
||||
|
||||
func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package anthropic
|
||||
|
||||
import "yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
|
||||
// OpenAINativeBridge returns a Provider that converts OpenAI chat/completions to Anthropic Messages upstream.
|
||||
func OpenAINativeBridge() providerapi.Provider {
|
||||
return New()
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package anthropic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
)
|
||||
|
||||
func TestOpenAINativeBridgeBuildDeepseekAnthropicURL(t *testing.T) {
|
||||
p := OpenAINativeBridge()
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("deepseek-chat"))
|
||||
user := jsonutils.NewDict()
|
||||
user.Set("role", jsonutils.NewString("user"))
|
||||
user.Set("content", jsonutils.NewString("hello"))
|
||||
body.Set("messages", jsonutils.NewArray(user))
|
||||
|
||||
req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{
|
||||
BaseURL: "https://api.deepseek.com/anthropic",
|
||||
APIKey: "ds-key",
|
||||
UpstreamModel: "deepseek-chat",
|
||||
}, body, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "https://api.deepseek.com/anthropic/v1/messages" {
|
||||
t.Fatalf("url: %s", req.URL)
|
||||
}
|
||||
if req.Headers["x-api-key"] != "ds-key" {
|
||||
t.Fatal("missing x-api-key header")
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
type provider struct{}
|
||||
@@ -33,7 +34,7 @@ func New() providerapi.Provider {
|
||||
}
|
||||
|
||||
func (p *provider) Key() string {
|
||||
return "azure"
|
||||
return api.ProviderKeyAzure
|
||||
}
|
||||
|
||||
func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
type provider struct {
|
||||
@@ -29,11 +30,11 @@ type provider struct {
|
||||
|
||||
// New returns the Baidu Wenxin / Qianfan provider adapter.
|
||||
func New() providerapi.Provider {
|
||||
return &provider{v2: openai.NewCompat("baidu")}
|
||||
return &provider{v2: openai.NewCompat(api.ProviderKeyBaidu)}
|
||||
}
|
||||
|
||||
func (p *provider) Key() string {
|
||||
return "baidu"
|
||||
return api.ProviderKeyBaidu
|
||||
}
|
||||
|
||||
func (p *provider) useV2(ctx *providerapi.ChatContext) bool {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package providers
|
||||
|
||||
import (
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic"
|
||||
apapi "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
// ChatProviderForUpstream returns the provider adapter for chat/completions upstream calls.
|
||||
func ChatProviderForUpstream(providerKey, apiMode string) Provider {
|
||||
if apiMode == apapi.ProviderAPIModeAnthropic && apapi.SupportsDualAPIMode(providerKey) {
|
||||
return anthropic.OpenAINativeBridge()
|
||||
}
|
||||
return Get(providerKey)
|
||||
}
|
||||
|
||||
// ChatContextFromUpstream builds a provider ChatContext from resolved upstream fields.
|
||||
func ChatContextFromUpstream(providerKey, baseURL, apiKey, upstreamModel, apiMode string) *ChatContext {
|
||||
return &ChatContext{
|
||||
ProviderKey: providerKey,
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
UpstreamModel: upstreamModel,
|
||||
APIMode: apiMode,
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
type provider struct {
|
||||
@@ -32,7 +33,7 @@ type provider struct {
|
||||
|
||||
// New returns the Cohere provider adapter (OpenAI-compatible chat, native embeddings).
|
||||
func New() providerapi.Provider {
|
||||
return &provider{Compat: openai.NewCompat("cohere")}
|
||||
return &provider{Compat: openai.NewCompat(api.ProviderKeyCohere)}
|
||||
}
|
||||
|
||||
func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package providers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
)
|
||||
|
||||
func TestOpenAIEmbeddingsCompatBuild(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Add(jsonutils.NewString("text-embedding-3-small"), "model")
|
||||
body.Add(jsonutils.NewString("hello"), "input")
|
||||
|
||||
p := GetEmbeddings("openai")
|
||||
req, err := p.BuildEmbeddingsRequest(&ChatContext{
|
||||
BaseURL: "https://api.openai.com",
|
||||
APIKey: "sk-test",
|
||||
UpstreamModel: "text-embedding-3-small",
|
||||
}, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "https://api.openai.com/v1/embeddings" {
|
||||
t.Fatalf("unexpected url: %s", req.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiEmbeddingsBuildSingle(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Add(jsonutils.NewString("text-embedding-004"), "model")
|
||||
body.Add(jsonutils.NewString("hello world"), "input")
|
||||
|
||||
p := GetEmbeddings("gemini")
|
||||
req, err := p.BuildEmbeddingsRequest(&ChatContext{
|
||||
BaseURL: "https://generativelanguage.googleapis.com/v1beta",
|
||||
APIKey: "key",
|
||||
UpstreamModel: "text-embedding-004",
|
||||
}, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent" {
|
||||
t.Fatalf("unexpected url: %s", req.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiEmbeddingsNormalize(t *testing.T) {
|
||||
p := GetEmbeddings("gemini")
|
||||
out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embedding":{"values":[0.1,0.2]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(out, &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["object"] != "list" {
|
||||
t.Fatalf("unexpected object: %#v", resp["object"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCohereEmbeddingsBuild(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Add(jsonutils.NewString("embed-english-v3.0"), "model")
|
||||
body.Add(jsonutils.NewArray(jsonutils.NewString("a"), jsonutils.NewString("b")), "input")
|
||||
|
||||
p := GetEmbeddings("cohere")
|
||||
req, err := p.BuildEmbeddingsRequest(&ChatContext{
|
||||
BaseURL: "https://api.cohere.ai",
|
||||
APIKey: "key",
|
||||
UpstreamModel: "embed-english-v3.0",
|
||||
}, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "https://api.cohere.ai/v2/embed" {
|
||||
t.Fatalf("unexpected url: %s", req.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCohereEmbeddingsNormalize(t *testing.T) {
|
||||
p := GetEmbeddings("cohere")
|
||||
out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embeddings":{"float":[[0.1],[0.2]]}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var resp struct {
|
||||
Data []struct {
|
||||
Index int `json:"index"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(out, &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(resp.Data) != 2 {
|
||||
t.Fatalf("expected 2 embeddings, got %d", len(resp.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicEmbeddingsUnsupported(t *testing.T) {
|
||||
p := GetEmbeddings("anthropic")
|
||||
_, err := p.BuildEmbeddingsRequest(&ChatContext{ProviderKey: "anthropic"}, jsonutils.NewDict())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for anthropic embeddings")
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
type provider struct{}
|
||||
@@ -36,7 +37,7 @@ func New() providerapi.Provider {
|
||||
}
|
||||
|
||||
func (p *provider) Key() string {
|
||||
return "gemini"
|
||||
return api.ProviderKeyGemini
|
||||
}
|
||||
|
||||
func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
|
||||
|
||||
@@ -20,6 +20,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/azure"
|
||||
)
|
||||
|
||||
func TestOpenAIImagesCompatBuild(t *testing.T) {
|
||||
@@ -98,7 +100,10 @@ func TestAzureImagesBuild(t *testing.T) {
|
||||
body.Add(jsonutils.NewString("dall-e-3"), "model")
|
||||
body.Add(jsonutils.NewString("test"), "prompt")
|
||||
|
||||
p := GetImages("azure")
|
||||
p, ok := azure.New().(ImagesProvider)
|
||||
if !ok {
|
||||
t.Fatal("azure provider should implement ImagesProvider")
|
||||
}
|
||||
req, err := p.BuildImagesGenerationsRequest(&ChatContext{
|
||||
BaseURL: "https://example.openai.azure.com",
|
||||
APIKey: "key",
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
package messages // import "yunion.io/x/onecloud/pkg/aiproxy/providers/messages"
|
||||
@@ -0,0 +1,164 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package messages
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
)
|
||||
|
||||
func TestPassthroughAdapterBuild(t *testing.T) {
|
||||
adapter, err := GetAdapter("anthropic", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !adapter.AnthropicStreamPassthrough() {
|
||||
t.Fatal("expected passthrough stream")
|
||||
}
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("claude-sonnet-4-5"))
|
||||
body.Set("max_tokens", jsonutils.NewInt(100))
|
||||
user := jsonutils.NewDict()
|
||||
user.Set("role", jsonutils.NewString("user"))
|
||||
user.Set("content", jsonutils.NewString("hi"))
|
||||
body.Set("messages", jsonutils.NewArray(user))
|
||||
|
||||
req, err := adapter.BuildUpstreamRequest(testChatCtx("anthropic", "https://api.anthropic.com", "sk-ant", "claude-sonnet-4-5"), body, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "https://api.anthropic.com/v1/messages" {
|
||||
t.Fatalf("url: %s", req.URL)
|
||||
}
|
||||
if req.Headers["x-api-key"] != "sk-ant" {
|
||||
t.Fatal("missing x-api-key")
|
||||
}
|
||||
var wire map[string]interface{}
|
||||
if err := json.Unmarshal(req.Body, &wire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wire["model"] != "claude-sonnet-4-5" {
|
||||
t.Fatalf("model override: %#v", wire["model"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranslationAdapterBuildDeepSeek(t *testing.T) {
|
||||
adapter, err := GetAdapter("openai", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if adapter.AnthropicStreamPassthrough() {
|
||||
t.Fatal("expected translated stream")
|
||||
}
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("deepseek-chat"))
|
||||
body.Set("max_tokens", jsonutils.NewInt(256))
|
||||
user := jsonutils.NewDict()
|
||||
user.Set("role", jsonutils.NewString("user"))
|
||||
user.Set("content", jsonutils.NewString("hello"))
|
||||
body.Set("messages", jsonutils.NewArray(user))
|
||||
|
||||
req, err := adapter.BuildUpstreamRequest(testChatCtx("openai", "https://api.deepseek.com", "ds-key", "deepseek-chat"), body, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "https://api.deepseek.com/v1/chat/completions" {
|
||||
t.Fatalf("url: %s", req.URL)
|
||||
}
|
||||
var wire map[string]interface{}
|
||||
if err := json.Unmarshal(req.Body, &wire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wire["model"] != "deepseek-chat" {
|
||||
t.Fatalf("model: %#v", wire["model"])
|
||||
}
|
||||
if wire["stream"] != true {
|
||||
t.Fatalf("stream: %#v", wire["stream"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAdapterDeepseekAnthropicPassthrough(t *testing.T) {
|
||||
adapter, err := GetAdapter("deepseek", "anthropic")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !adapter.AnthropicStreamPassthrough() {
|
||||
t.Fatal("expected passthrough stream for deepseek anthropic mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAdapterDeepseekOpenAITranslation(t *testing.T) {
|
||||
adapter, err := GetAdapter("deepseek", "openai")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if adapter.AnthropicStreamPassthrough() {
|
||||
t.Fatal("expected translated stream for deepseek openai mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAdapterCustomAnthropicPassthrough(t *testing.T) {
|
||||
adapter, err := GetAdapter("custom", "anthropic")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !adapter.AnthropicStreamPassthrough() {
|
||||
t.Fatal("expected passthrough stream for custom anthropic mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAdapterBlocksGemini(t *testing.T) {
|
||||
if _, err := GetAdapter("gemini", ""); err == nil {
|
||||
t.Fatal("expected gemini to be unsupported")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAdapterVLLMTranslation(t *testing.T) {
|
||||
adapter, err := GetAdapter("vllm", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if adapter.AnthropicStreamPassthrough() {
|
||||
t.Fatal("expected translated stream for vllm")
|
||||
}
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("t-vllm"))
|
||||
body.Set("max_tokens", jsonutils.NewInt(256))
|
||||
user := jsonutils.NewDict()
|
||||
user.Set("role", jsonutils.NewString("user"))
|
||||
user.Set("content", jsonutils.NewString("hello"))
|
||||
body.Set("messages", jsonutils.NewArray(user))
|
||||
|
||||
req, err := adapter.BuildUpstreamRequest(testChatCtx("vllm", "http://127.0.0.1:8000/v1", "sk-test", "t-vllm"), body, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.URL != "http://127.0.0.1:8000/v1/chat/completions" {
|
||||
t.Fatalf("url: %s", req.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func testChatCtx(providerKey, baseURL, apiKey, model string) *providerapi.ChatContext {
|
||||
return &providerapi.ChatContext{
|
||||
ProviderKey: providerKey,
|
||||
BaseURL: baseURL,
|
||||
APIKey: apiKey,
|
||||
UpstreamModel: model,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package messages
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
)
|
||||
|
||||
const anthropicAPIVersion = "2023-06-01"
|
||||
|
||||
type passthroughAdapter struct{}
|
||||
|
||||
func (passthroughAdapter) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
|
||||
if ctx == nil {
|
||||
return nil, fmt.Errorf("nil chat context")
|
||||
}
|
||||
dup := jsonutils.NewDict()
|
||||
if body != nil {
|
||||
dup = body.Copy()
|
||||
}
|
||||
dup.Set("model", jsonutils.NewString(ctx.UpstreamModel))
|
||||
if stream {
|
||||
dup.Set("stream", jsonutils.JSONTrue)
|
||||
}
|
||||
base := strings.TrimSpace(ctx.BaseURL)
|
||||
if base == "" {
|
||||
base = "https://api.anthropic.com"
|
||||
}
|
||||
return &providerapi.HTTPRequest{
|
||||
Method: http.MethodPost,
|
||||
URL: openai.JoinURL(base, "/v1/messages"),
|
||||
Headers: map[string]string{
|
||||
"x-api-key": strings.TrimSpace(ctx.APIKey),
|
||||
"anthropic-version": anthropicAPIVersion,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
Body: []byte(dup.String()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (passthroughAdapter) NormalizeResponse(_ providerapi.Provider, body []byte) ([]byte, error) {
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (passthroughAdapter) AnthropicStreamPassthrough() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (passthroughAdapter) NewStreamState(string) interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (passthroughAdapter) ConvertStreamPayload(_ interface{}, _ []byte, _ bool) ([]providerapi.AnthropicStreamChunk, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package messages
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
var (
|
||||
passthrough = passthroughAdapter{}
|
||||
)
|
||||
|
||||
// GetAdapter returns the MessagesAdapter for a resolved catalog provider_key and api_mode.
|
||||
func GetAdapter(providerKey, apiMode string) (providerapi.MessagesAdapter, error) {
|
||||
key := strings.ToLower(strings.TrimSpace(providerKey))
|
||||
mode := strings.ToLower(strings.TrimSpace(apiMode))
|
||||
if mode == "" {
|
||||
mode = api.ProviderAPIModeOpenAI
|
||||
}
|
||||
if key == api.ProviderKeyAnthropic {
|
||||
return passthrough, nil
|
||||
}
|
||||
if api.IsNativeMessagesAdapterProvider(key) {
|
||||
return nil, fmt.Errorf("provider %q does not support anthropic messages API", providerKey)
|
||||
}
|
||||
if mode == api.ProviderAPIModeAnthropic && api.SupportsDualAPIMode(key) {
|
||||
return passthrough, nil
|
||||
}
|
||||
return translationAdapter{providerKey: providerKey}, nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package messages
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
)
|
||||
|
||||
type translationStreamState struct {
|
||||
conv *openai.AnthropicStreamConverter
|
||||
}
|
||||
|
||||
type translationAdapter struct {
|
||||
providerKey string
|
||||
}
|
||||
|
||||
func (a translationAdapter) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
|
||||
if ctx == nil {
|
||||
return nil, fmt.Errorf("nil chat context")
|
||||
}
|
||||
openaiBody, err := openai.AnthropicToChatCompletions(body, ctx.UpstreamModel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stream {
|
||||
openaiBody.Set("stream", jsonutils.JSONTrue)
|
||||
streamOpts := jsonutils.NewDict()
|
||||
streamOpts.Set("include_usage", jsonutils.JSONTrue)
|
||||
openaiBody.Set("stream_options", streamOpts)
|
||||
}
|
||||
prov := providers.Get(a.providerKey)
|
||||
return prov.BuildUpstreamRequest(&providerapi.ChatContext{
|
||||
ProviderKey: ctx.ProviderKey,
|
||||
BaseURL: ctx.BaseURL,
|
||||
APIKey: ctx.APIKey,
|
||||
UpstreamModel: ctx.UpstreamModel,
|
||||
}, openaiBody, stream)
|
||||
}
|
||||
|
||||
func (a translationAdapter) NormalizeResponse(prov providerapi.Provider, body []byte) ([]byte, error) {
|
||||
norm, err := prov.NormalizeResponse(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(norm) > 0 {
|
||||
body = norm
|
||||
}
|
||||
return openai.ChatCompletionToAnthropic(body)
|
||||
}
|
||||
|
||||
func (translationAdapter) AnthropicStreamPassthrough() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (translationAdapter) NewStreamState(requestModel string) interface{} {
|
||||
return &translationStreamState{
|
||||
conv: openai.NewAnthropicStreamConverter(requestModel),
|
||||
}
|
||||
}
|
||||
|
||||
func (translationAdapter) ConvertStreamPayload(state interface{}, payload []byte, endOfStream bool) ([]providerapi.AnthropicStreamChunk, error) {
|
||||
st, ok := state.(*translationStreamState)
|
||||
if !ok || st == nil || st.conv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
events, err := st.conv.Feed(payload, endOfStream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]providerapi.AnthropicStreamChunk, 0, len(events))
|
||||
for _, evt := range events {
|
||||
out = append(out, providerapi.AnthropicStreamChunk{
|
||||
Event: evt.Event,
|
||||
Data: evt.Data,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,508 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package openai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
)
|
||||
|
||||
// AnthropicToChatCompletions converts an Anthropic Messages request body to OpenAI chat/completions shape.
|
||||
func AnthropicToChatCompletions(body *jsonutils.JSONDict, upstreamModel string) (*jsonutils.JSONDict, error) {
|
||||
if body == nil {
|
||||
return nil, fmt.Errorf("nil request body")
|
||||
}
|
||||
out := jsonutils.NewDict()
|
||||
model := strings.TrimSpace(upstreamModel)
|
||||
if model == "" {
|
||||
if m, err := body.GetString("model"); err == nil {
|
||||
model = strings.TrimSpace(m)
|
||||
}
|
||||
}
|
||||
if model == "" {
|
||||
return nil, fmt.Errorf("missing model")
|
||||
}
|
||||
out.Set("model", jsonutils.NewString(model))
|
||||
|
||||
maxTokens, err := body.Int("max_tokens")
|
||||
if err != nil || maxTokens <= 0 {
|
||||
return nil, fmt.Errorf("max_tokens is required")
|
||||
}
|
||||
out.Set("max_tokens", jsonutils.NewInt(maxTokens))
|
||||
|
||||
if stream, _ := body.Bool("stream"); stream {
|
||||
out.Set("stream", jsonutils.JSONTrue)
|
||||
streamOpts := jsonutils.NewDict()
|
||||
streamOpts.Set("include_usage", jsonutils.JSONTrue)
|
||||
out.Set("stream_options", streamOpts)
|
||||
}
|
||||
if v, ok := FloatParam(body, "temperature"); ok {
|
||||
out.Set("temperature", jsonutils.NewFloat64(v))
|
||||
}
|
||||
if v, ok := FloatParam(body, "top_p"); ok {
|
||||
out.Set("top_p", jsonutils.NewFloat64(v))
|
||||
}
|
||||
if stops, err := body.Get("stop_sequences"); err == nil {
|
||||
out.Set("stop", stops)
|
||||
}
|
||||
|
||||
msgs, err := anthropicMessagesToOpenAI(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out.Set("messages", msgs)
|
||||
|
||||
if tools, toolChoice, err := anthropicToolsToOpenAI(body); err != nil {
|
||||
return nil, err
|
||||
} else if tools != nil && tools.Length() > 0 {
|
||||
out.Set("tools", tools)
|
||||
if toolChoice != nil {
|
||||
out.Set("tool_choice", toolChoice)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func anthropicMessagesToOpenAI(body *jsonutils.JSONDict) (*jsonutils.JSONArray, error) {
|
||||
var systemParts []string
|
||||
if sysRaw, err := body.Get("system"); err == nil {
|
||||
if sysText := anthropicSystemText(sysRaw); sysText != "" {
|
||||
systemParts = append(systemParts, sysText)
|
||||
}
|
||||
}
|
||||
rawMsgs, err := body.Get("messages")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("missing messages")
|
||||
}
|
||||
var messages []json.RawMessage
|
||||
if err := json.Unmarshal([]byte(rawMsgs.String()), &messages); err != nil {
|
||||
return nil, fmt.Errorf("invalid messages: %w", err)
|
||||
}
|
||||
converted := jsonutils.NewArray()
|
||||
for _, raw := range messages {
|
||||
var msg struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &msg); err != nil {
|
||||
return nil, fmt.Errorf("invalid message: %w", err)
|
||||
}
|
||||
role := strings.ToLower(strings.TrimSpace(msg.Role))
|
||||
switch role {
|
||||
case "system":
|
||||
sysText, err := anthropicMessageContentText(msg.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sysText != "" {
|
||||
systemParts = append(systemParts, sysText)
|
||||
}
|
||||
case "user":
|
||||
parts, tools, err := parseAnthropicUserContent(msg.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, tr := range tools {
|
||||
toolMsg := jsonutils.NewDict()
|
||||
toolMsg.Set("role", jsonutils.NewString("tool"))
|
||||
toolMsg.Set("tool_call_id", jsonutils.NewString(tr.ID))
|
||||
toolMsg.Set("content", jsonutils.NewString(tr.Content))
|
||||
converted.Add(toolMsg)
|
||||
}
|
||||
if parts != nil {
|
||||
userMsg := jsonutils.NewDict()
|
||||
userMsg.Set("role", jsonutils.NewString("user"))
|
||||
userMsg.Set("content", parts)
|
||||
converted.Add(userMsg)
|
||||
}
|
||||
case "assistant":
|
||||
assistant, err := parseAnthropicAssistantContent(msg.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if assistant == nil {
|
||||
continue
|
||||
}
|
||||
converted.Add(assistant)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported message role %q", role)
|
||||
}
|
||||
}
|
||||
if converted.Size() == 0 {
|
||||
return nil, fmt.Errorf("no convertible messages")
|
||||
}
|
||||
arr := jsonutils.NewArray()
|
||||
if len(systemParts) > 0 {
|
||||
sysMsg := jsonutils.NewDict()
|
||||
sysMsg.Set("role", jsonutils.NewString("system"))
|
||||
sysMsg.Set("content", jsonutils.NewString(strings.Join(systemParts, "\n\n")))
|
||||
arr.Add(sysMsg)
|
||||
}
|
||||
for i := 0; i < converted.Size(); i++ {
|
||||
obj, err := converted.GetAt(i)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
arr.Add(obj)
|
||||
}
|
||||
return arr, nil
|
||||
}
|
||||
|
||||
func anthropicMessageContentText(raw json.RawMessage) (string, error) {
|
||||
if len(raw) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
parsed, err := jsonutils.Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid message content: %w", err)
|
||||
}
|
||||
return anthropicSystemText(parsed), nil
|
||||
}
|
||||
|
||||
type anthropicToolResult struct {
|
||||
ID string
|
||||
Content string
|
||||
}
|
||||
|
||||
func anthropicSystemText(raw jsonutils.JSONObject) string {
|
||||
if raw == nil {
|
||||
return ""
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal([]byte(raw.String()), &s); err == nil {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
var blocks []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(raw.String()), &blocks); err == nil {
|
||||
var b strings.Builder
|
||||
for _, blk := range blocks {
|
||||
if blk.Type == "text" && blk.Text != "" {
|
||||
if b.Len() > 0 {
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(blk.Text)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseAnthropicUserContent(raw json.RawMessage) (jsonutils.JSONObject, []anthropicToolResult, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err == nil {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil, nil, nil
|
||||
}
|
||||
return jsonutils.NewString(s), nil, nil
|
||||
}
|
||||
var blocks []map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &blocks); err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid user content: %w", err)
|
||||
}
|
||||
var textParts []string
|
||||
var tools []anthropicToolResult
|
||||
for _, blk := range blocks {
|
||||
typ, _ := blk["type"].(string)
|
||||
switch typ {
|
||||
case "text":
|
||||
if t, _ := blk["text"].(string); t != "" {
|
||||
textParts = append(textParts, t)
|
||||
}
|
||||
case "tool_result":
|
||||
id, _ := blk["tool_use_id"].(string)
|
||||
content := anthropicBlockContentText(blk["content"])
|
||||
tools = append(tools, anthropicToolResult{ID: id, Content: content})
|
||||
}
|
||||
}
|
||||
if len(textParts) == 0 {
|
||||
return nil, tools, nil
|
||||
}
|
||||
if len(textParts) == 1 {
|
||||
return jsonutils.NewString(textParts[0]), tools, nil
|
||||
}
|
||||
parts := jsonutils.NewArray()
|
||||
for _, p := range textParts {
|
||||
blk := jsonutils.NewDict()
|
||||
blk.Set("type", jsonutils.NewString("text"))
|
||||
blk.Set("text", jsonutils.NewString(p))
|
||||
parts.Add(blk)
|
||||
}
|
||||
return parts, tools, nil
|
||||
}
|
||||
|
||||
func anthropicBlockContentText(v interface{}) string {
|
||||
switch c := v.(type) {
|
||||
case string:
|
||||
return c
|
||||
case []interface{}:
|
||||
var parts []string
|
||||
for _, item := range c {
|
||||
if m, ok := item.(map[string]interface{}); ok {
|
||||
if t, _ := m["text"].(string); t != "" {
|
||||
parts = append(parts, t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "\n")
|
||||
default:
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
}
|
||||
|
||||
func parseAnthropicAssistantContent(raw json.RawMessage) (*jsonutils.JSONDict, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var s string
|
||||
if err := json.Unmarshal(raw, &s); err == nil {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
msg := jsonutils.NewDict()
|
||||
msg.Set("role", jsonutils.NewString("assistant"))
|
||||
msg.Set("content", jsonutils.NewString(s))
|
||||
return msg, nil
|
||||
}
|
||||
var blocks []AnthropicBlock
|
||||
if err := json.Unmarshal(raw, &blocks); err != nil {
|
||||
return nil, fmt.Errorf("invalid assistant content: %w", err)
|
||||
}
|
||||
assistant := AnthropicBlocksToAssistant(blocks)
|
||||
msg := jsonutils.NewDict()
|
||||
msg.Set("role", jsonutils.NewString("assistant"))
|
||||
if assistant.Content != "" {
|
||||
msg.Set("content", jsonutils.NewString(assistant.Content))
|
||||
}
|
||||
if len(assistant.ToolCalls) > 0 {
|
||||
calls := jsonutils.NewArray()
|
||||
for _, tc := range assistant.ToolCalls {
|
||||
call := jsonutils.NewDict()
|
||||
call.Set("id", jsonutils.NewString(tc.ID))
|
||||
call.Set("type", jsonutils.NewString("function"))
|
||||
fn := jsonutils.NewDict()
|
||||
fn.Set("name", jsonutils.NewString(tc.Function.Name))
|
||||
fn.Set("arguments", jsonutils.NewString(tc.Function.Arguments))
|
||||
call.Set("function", fn)
|
||||
calls.Add(call)
|
||||
}
|
||||
msg.Set("tool_calls", calls)
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
func anthropicToolsToOpenAI(body *jsonutils.JSONDict) (*jsonutils.JSONArray, jsonutils.JSONObject, error) {
|
||||
rawTools, err := body.Get("tools")
|
||||
if err != nil {
|
||||
return nil, nil, nil
|
||||
}
|
||||
var toolsIn []struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
InputSchema json.RawMessage `json:"input_schema"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(rawTools.String()), &toolsIn); err != nil {
|
||||
return nil, nil, fmt.Errorf("invalid tools: %w", err)
|
||||
}
|
||||
if len(toolsIn) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
out := jsonutils.NewArray()
|
||||
for _, t := range toolsIn {
|
||||
name := strings.TrimSpace(t.Name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
tool := jsonutils.NewDict()
|
||||
tool.Set("type", jsonutils.NewString("function"))
|
||||
fn := jsonutils.NewDict()
|
||||
fn.Set("name", jsonutils.NewString(name))
|
||||
if desc := strings.TrimSpace(t.Description); desc != "" {
|
||||
fn.Set("description", jsonutils.NewString(desc))
|
||||
}
|
||||
if len(t.InputSchema) > 0 && string(t.InputSchema) != "null" {
|
||||
if params, err := jsonutils.Parse(t.InputSchema); err == nil {
|
||||
fn.Set("parameters", params)
|
||||
}
|
||||
}
|
||||
tool.Set("function", fn)
|
||||
out.Add(tool)
|
||||
}
|
||||
var toolChoice jsonutils.JSONObject
|
||||
if tcRaw, err := body.Get("tool_choice"); err == nil {
|
||||
toolChoice = anthropicToolChoiceToOpenAI(tcRaw)
|
||||
}
|
||||
return out, toolChoice, nil
|
||||
}
|
||||
|
||||
func anthropicToolChoiceToOpenAI(raw jsonutils.JSONObject) jsonutils.JSONObject {
|
||||
var obj map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(raw.String()), &obj); err != nil {
|
||||
return nil
|
||||
}
|
||||
typ, _ := obj["type"].(string)
|
||||
switch strings.ToLower(strings.TrimSpace(typ)) {
|
||||
case "auto", "":
|
||||
return jsonutils.NewString("auto")
|
||||
case "none":
|
||||
return jsonutils.NewString("none")
|
||||
case "any":
|
||||
return jsonutils.NewString("required")
|
||||
case "tool":
|
||||
name, _ := obj["name"].(string)
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return jsonutils.NewString("required")
|
||||
}
|
||||
choice := jsonutils.NewDict()
|
||||
choice.Set("type", jsonutils.NewString("function"))
|
||||
fn := jsonutils.NewDict()
|
||||
fn.Set("name", jsonutils.NewString(strings.TrimSpace(name)))
|
||||
choice.Set("function", fn)
|
||||
return choice
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ChatCompletionToAnthropic converts an OpenAI chat.completion JSON body to Anthropic Messages response.
|
||||
func ChatCompletionToAnthropic(body []byte) ([]byte, error) {
|
||||
var resp struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
ToolCalls []ToolCall `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil, fmt.Errorf("invalid OpenAI response: %w", err)
|
||||
}
|
||||
if len(resp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("empty OpenAI choices")
|
||||
}
|
||||
choice := resp.Choices[0]
|
||||
blocks := make([]map[string]interface{}, 0, 1+len(choice.Message.ToolCalls))
|
||||
if text := MessageTextContent(choice.Message.Content); text != "" {
|
||||
blocks = append(blocks, map[string]interface{}{
|
||||
"type": "text",
|
||||
"text": text,
|
||||
})
|
||||
}
|
||||
for _, tc := range choice.Message.ToolCalls {
|
||||
input := map[string]interface{}{}
|
||||
if args := strings.TrimSpace(tc.Function.Arguments); args != "" {
|
||||
_ = json.Unmarshal([]byte(args), &input)
|
||||
}
|
||||
id := strings.TrimSpace(tc.ID)
|
||||
if id == "" {
|
||||
id = "toolu_" + strings.TrimSpace(tc.Function.Name)
|
||||
}
|
||||
blocks = append(blocks, map[string]interface{}{
|
||||
"type": "tool_use",
|
||||
"id": id,
|
||||
"name": strings.TrimSpace(tc.Function.Name),
|
||||
"input": input,
|
||||
})
|
||||
}
|
||||
stopReason := openAIFinishReasonToAnthropic(choice.FinishReason)
|
||||
out := map[string]interface{}{
|
||||
"id": resp.ID,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": resp.Model,
|
||||
"content": blocks,
|
||||
"stop_reason": stopReason,
|
||||
"usage": map[string]interface{}{
|
||||
"input_tokens": resp.Usage.PromptTokens,
|
||||
"output_tokens": resp.Usage.CompletionTokens,
|
||||
},
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
|
||||
func openAIFinishReasonToAnthropic(reason string) string {
|
||||
switch strings.TrimSpace(reason) {
|
||||
case "stop":
|
||||
return "end_turn"
|
||||
case "length":
|
||||
return "max_tokens"
|
||||
case "tool_calls", "function_call":
|
||||
return "tool_use"
|
||||
default:
|
||||
if reason == "" {
|
||||
return "end_turn"
|
||||
}
|
||||
return reason
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAIErrorToAnthropic converts an OpenAI-style error JSON body to Anthropic error format.
|
||||
func OpenAIErrorToAnthropic(body []byte, statusCode int) []byte {
|
||||
msg := "upstream request failed"
|
||||
var wrap struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(body, &wrap) == nil && wrap.Error.Message != "" {
|
||||
msg = wrap.Error.Message
|
||||
}
|
||||
errType := "api_error"
|
||||
if statusCode == 400 {
|
||||
errType = "invalid_request_error"
|
||||
} else if statusCode == 401 {
|
||||
errType = "authentication_error"
|
||||
} else if statusCode == 429 {
|
||||
errType = "rate_limit_error"
|
||||
}
|
||||
out, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "error",
|
||||
"error": map[string]interface{}{
|
||||
"type": errType,
|
||||
"message": msg,
|
||||
},
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// NewAnthropicErrorBody builds an Anthropic-style error response body.
|
||||
func NewAnthropicErrorBody(errType, message string) []byte {
|
||||
out, _ := json.Marshal(map[string]interface{}{
|
||||
"type": "error",
|
||||
"error": map[string]interface{}{
|
||||
"type": errType,
|
||||
"message": message,
|
||||
},
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package openai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
)
|
||||
|
||||
func TestAnthropicToChatCompletionsSystemInMessages(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("deepseek-chat"))
|
||||
body.Set("max_tokens", jsonutils.NewInt(512))
|
||||
sysMsg := jsonutils.NewDict()
|
||||
sysMsg.Set("role", jsonutils.NewString("system"))
|
||||
sysMsg.Set("content", jsonutils.NewString("You are Claude Code."))
|
||||
userMsg := jsonutils.NewDict()
|
||||
userMsg.Set("role", jsonutils.NewString("user"))
|
||||
userMsg.Set("content", jsonutils.NewString("hi"))
|
||||
body.Set("messages", jsonutils.NewArray(sysMsg, userMsg))
|
||||
|
||||
out, err := AnthropicToChatCompletions(body, "deepseek-chat")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
msgs, err := out.Get("messages")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arr, ok := msgs.(*jsonutils.JSONArray)
|
||||
if !ok || arr.Length() != 2 {
|
||||
t.Fatalf("expected system+user messages, got %#v", msgs)
|
||||
}
|
||||
role0, _ := arr.GetAt(0)
|
||||
if got, _ := role0.(*jsonutils.JSONDict).GetString("role"); got != "system" {
|
||||
t.Fatalf("first message role: got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicToChatCompletionsBasic(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("deepseek-chat"))
|
||||
body.Set("max_tokens", jsonutils.NewInt(512))
|
||||
body.Set("system", jsonutils.NewString("You are helpful."))
|
||||
userMsg := jsonutils.NewDict()
|
||||
userMsg.Set("role", jsonutils.NewString("user"))
|
||||
userMsg.Set("content", jsonutils.NewString("Hello"))
|
||||
body.Set("messages", jsonutils.NewArray(userMsg))
|
||||
|
||||
out, err := AnthropicToChatCompletions(body, "deepseek-chat")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := out.GetString("model"); got != "deepseek-chat" {
|
||||
t.Fatalf("model: got %q", got)
|
||||
}
|
||||
msgs, err := out.Get("messages")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arr, ok := msgs.(*jsonutils.JSONArray)
|
||||
if !ok || arr.Length() != 2 {
|
||||
t.Fatalf("expected system+user messages, got %#v", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatCompletionToAnthropicBasic(t *testing.T) {
|
||||
raw := []byte(`{
|
||||
"id":"chatcmpl-1",
|
||||
"model":"deepseek-chat",
|
||||
"choices":[{"message":{"role":"assistant","content":"Hi there"},"finish_reason":"stop"}],
|
||||
"usage":{"prompt_tokens":5,"completion_tokens":3}
|
||||
}`)
|
||||
out, err := ChatCompletionToAnthropic(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(out, &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["type"] != "message" {
|
||||
t.Fatalf("unexpected type: %#v", resp["type"])
|
||||
}
|
||||
if resp["stop_reason"] != "end_turn" {
|
||||
t.Fatalf("unexpected stop_reason: %#v", resp["stop_reason"])
|
||||
}
|
||||
content := resp["content"].([]interface{})
|
||||
if len(content) != 1 {
|
||||
t.Fatalf("expected one content block, got %#v", content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicStreamConverterMultipleTools(t *testing.T) {
|
||||
conv := NewAnthropicStreamConverter("deepseek-chat")
|
||||
textChunk, _ := json.Marshal(NewStreamChunk("deepseek-chat", "chatcmpl-1", 0, "Checking.", ""))
|
||||
if _, err := conv.Feed(textChunk, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tool1Start, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{
|
||||
ID: "call_0", Type: "function", Function: ToolFunction{Name: "Bash"},
|
||||
}, ""))
|
||||
events, err := conv.Feed(tool1Start, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := blockIndexFromEvent(t, events[1]); got != 1 {
|
||||
t.Fatalf("tool1 start index: got %d", got)
|
||||
}
|
||||
|
||||
tool1End, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{
|
||||
Function: ToolFunction{Arguments: `{"command":"ls"}`},
|
||||
}, ""))
|
||||
if _, err := conv.Feed(tool1End, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tool2Start, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 1, ToolCall{
|
||||
ID: "call_1", Type: "function", Function: ToolFunction{Name: "Bash"},
|
||||
}, ""))
|
||||
events, err = conv.Feed(tool2Start, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) != 2 {
|
||||
t.Fatalf("expected stop+start for tool2, got %v", eventNames(events))
|
||||
}
|
||||
if events[0].Event != "content_block_stop" {
|
||||
t.Fatalf("expected stop before tool2, got %v", eventNames(events))
|
||||
}
|
||||
if got := blockIndexFromEvent(t, events[0]); got != 1 {
|
||||
t.Fatalf("tool1 stop index: got %d want 1", got)
|
||||
}
|
||||
if got := blockIndexFromEvent(t, events[1]); got != 2 {
|
||||
t.Fatalf("tool2 start index: got %d want 2", got)
|
||||
}
|
||||
|
||||
finishChunk, _ := json.Marshal(map[string]interface{}{
|
||||
"id": "chatcmpl-1",
|
||||
"choices": []map[string]interface{}{
|
||||
{"delta": map[string]interface{}{}, "finish_reason": "tool_calls"},
|
||||
},
|
||||
"usage": map[string]interface{}{
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
},
|
||||
})
|
||||
events, err = conv.Feed(finishChunk, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var sawStop2, sawMessageStop bool
|
||||
for _, evt := range events {
|
||||
switch evt.Event {
|
||||
case "content_block_stop":
|
||||
if blockIndexFromEvent(t, evt) == 2 {
|
||||
sawStop2 = true
|
||||
}
|
||||
case "message_stop":
|
||||
sawMessageStop = true
|
||||
}
|
||||
}
|
||||
if !sawStop2 {
|
||||
t.Fatalf("expected content_block_stop for tool2, got %v", eventNames(events))
|
||||
}
|
||||
if !sawMessageStop {
|
||||
t.Fatalf("expected message_stop, got %v", eventNames(events))
|
||||
}
|
||||
if conv.outputTokens != 50 {
|
||||
t.Fatalf("output tokens: got %d want 50", conv.outputTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func blockIndexFromEvent(t *testing.T, evt AnthropicStreamEvent) int {
|
||||
t.Helper()
|
||||
var wrap map[string]interface{}
|
||||
if err := json.Unmarshal(evt.Data, &wrap); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idx, ok := wrap["index"].(float64)
|
||||
if !ok {
|
||||
t.Fatalf("missing index in %s: %#v", evt.Event, wrap)
|
||||
}
|
||||
return int(idx)
|
||||
}
|
||||
|
||||
func TestAnthropicStreamConverterTextToTool(t *testing.T) {
|
||||
conv := NewAnthropicStreamConverter("deepseek-chat")
|
||||
textChunk, _ := json.Marshal(NewStreamChunk("deepseek-chat", "chatcmpl-1", 0, "Let me check.", ""))
|
||||
events, err := conv.Feed(textChunk, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) < 3 {
|
||||
t.Fatalf("expected message_start + block start + delta, got %d events", len(events))
|
||||
}
|
||||
|
||||
toolChunk, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{
|
||||
ID: "call_1",
|
||||
Type: "function",
|
||||
Function: ToolFunction{
|
||||
Name: "Glob",
|
||||
},
|
||||
}, ""))
|
||||
events, err = conv.Feed(toolChunk, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) != 2 {
|
||||
t.Fatalf("expected content_block_stop + content_block_start, got %d events: %v", len(events), eventNames(events))
|
||||
}
|
||||
if events[0].Event != "content_block_stop" || events[1].Event != "content_block_start" {
|
||||
t.Fatalf("unexpected events: %v", eventNames(events))
|
||||
}
|
||||
var start map[string]interface{}
|
||||
if err := json.Unmarshal(events[1].Data, &start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(start["index"].(float64)) != 1 {
|
||||
t.Fatalf("tool block index: %#v", start["index"])
|
||||
}
|
||||
|
||||
argsChunk, _ := json.Marshal(NewStreamChunkToolDelta("deepseek-chat", "chatcmpl-1", 0, ToolCall{
|
||||
Function: ToolFunction{Arguments: `{"pattern":"**/*.go"}`},
|
||||
}, ""))
|
||||
events, err = conv.Feed(argsChunk, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) != 1 || events[0].Event != "content_block_delta" {
|
||||
t.Fatalf("expected tool args delta, got %v", eventNames(events))
|
||||
}
|
||||
var delta map[string]interface{}
|
||||
if err := json.Unmarshal(events[0].Data, &delta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(delta["index"].(float64)) != 1 {
|
||||
t.Fatalf("tool delta index: %#v", delta["index"])
|
||||
}
|
||||
}
|
||||
|
||||
func eventNames(events []AnthropicStreamEvent) []string {
|
||||
out := make([]string, len(events))
|
||||
for i, evt := range events {
|
||||
out[i] = evt.Event
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnthropicStreamConverterText(t *testing.T) {
|
||||
conv := NewAnthropicStreamConverter("deepseek-chat")
|
||||
chunk, _ := json.Marshal(NewStreamChunk("deepseek-chat", "chatcmpl-1", 0, "Hello", ""))
|
||||
events, err := conv.Feed(chunk, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(events) == 0 {
|
||||
t.Fatal("expected stream events")
|
||||
}
|
||||
if events[0].Event != "message_start" {
|
||||
t.Fatalf("first event: %s", events[0].Event)
|
||||
}
|
||||
finishChunk, _ := json.Marshal(map[string]interface{}{
|
||||
"id": "chatcmpl-1",
|
||||
"choices": []map[string]interface{}{
|
||||
{"delta": map[string]interface{}{}, "finish_reason": "stop"},
|
||||
},
|
||||
"usage": map[string]interface{}{
|
||||
"prompt_tokens": 3,
|
||||
"completion_tokens": 2,
|
||||
},
|
||||
})
|
||||
events, err = conv.Feed(finishChunk, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
foundStop := false
|
||||
for _, evt := range events {
|
||||
if evt.Event == "message_stop" {
|
||||
foundStop = true
|
||||
}
|
||||
}
|
||||
if !foundStop {
|
||||
t.Fatal("expected message_stop event")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicToolRoundTrip(t *testing.T) {
|
||||
body := jsonutils.NewDict()
|
||||
body.Set("model", jsonutils.NewString("claude-3-5-sonnet"))
|
||||
body.Set("max_tokens", jsonutils.NewInt(1024))
|
||||
userMsg := jsonutils.NewDict()
|
||||
userMsg.Set("role", jsonutils.NewString("user"))
|
||||
userMsg.Set("content", jsonutils.NewString("Weather?"))
|
||||
body.Set("messages", jsonutils.NewArray(userMsg))
|
||||
tool := jsonutils.NewDict()
|
||||
tool.Set("name", jsonutils.NewString("get_weather"))
|
||||
tool.Set("description", jsonutils.NewString("Get weather"))
|
||||
tool.Set("input_schema", jsonutils.NewDict())
|
||||
body.Set("tools", jsonutils.NewArray(tool))
|
||||
|
||||
openaiBody, err := AnthropicToChatCompletions(body, "claude-3-5-sonnet")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
toolsObj, err := openaiBody.Get("tools")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
toolsArr, ok := toolsObj.(*jsonutils.JSONArray)
|
||||
if !ok || toolsArr.Length() != 1 {
|
||||
t.Fatalf("expected tools in OpenAI body: %#v", openaiBody)
|
||||
}
|
||||
|
||||
raw := []byte(`{
|
||||
"id":"msg_1",
|
||||
"model":"claude-3-5-sonnet",
|
||||
"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Boston\"}"}}]},"finish_reason":"tool_calls"}],
|
||||
"usage":{"prompt_tokens":10,"completion_tokens":5}
|
||||
}`)
|
||||
out, err := ChatCompletionToAnthropic(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var resp map[string]interface{}
|
||||
if err := json.Unmarshal(out, &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp["stop_reason"] != "tool_use" {
|
||||
t.Fatalf("stop_reason: %#v", resp["stop_reason"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package openai
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// AnthropicStreamEvent is one Anthropic Messages SSE event for upstream clients.
|
||||
type AnthropicStreamEvent struct {
|
||||
Event string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// AnthropicStreamConverter converts OpenAI chat.completion.chunk SSE payloads to Anthropic SSE events.
|
||||
type AnthropicStreamConverter struct {
|
||||
requestModel string
|
||||
messageStarted bool
|
||||
hasOpenBlock bool
|
||||
openBlockIndex int
|
||||
closingEmitted bool
|
||||
messageID string
|
||||
model string
|
||||
stopReason string
|
||||
inputTokens int
|
||||
outputTokens int
|
||||
blockIndex int
|
||||
activeTools map[int]*streamToolState
|
||||
}
|
||||
|
||||
type streamToolState struct {
|
||||
blockIdx int
|
||||
id string
|
||||
name string
|
||||
}
|
||||
|
||||
// NewAnthropicStreamConverter creates stream conversion state for one Anthropic Messages response.
|
||||
func NewAnthropicStreamConverter(requestModel string) *AnthropicStreamConverter {
|
||||
return &AnthropicStreamConverter{
|
||||
requestModel: requestModel,
|
||||
activeTools: make(map[int]*streamToolState),
|
||||
openBlockIndex: -1,
|
||||
}
|
||||
}
|
||||
|
||||
// Feed processes one OpenAI SSE data payload (without the "data:" prefix).
|
||||
func (s *AnthropicStreamConverter) Feed(payload []byte, endOfStream bool) ([]AnthropicStreamEvent, error) {
|
||||
if len(payload) == 0 {
|
||||
if endOfStream && !s.closingEmitted {
|
||||
return s.emitClosing()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if string(payload) == "[DONE]" {
|
||||
if !s.closingEmitted {
|
||||
return s.emitClosing()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
var chunk struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Delta struct {
|
||||
Content *string `json:"content"`
|
||||
ToolCalls []ToolCall `json:"tool_calls"`
|
||||
} `json:"delta"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage *struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &chunk); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
if chunk.ID != "" && s.messageID == "" {
|
||||
s.messageID = chunk.ID
|
||||
}
|
||||
if chunk.Model != "" && s.model == "" {
|
||||
s.model = chunk.Model
|
||||
}
|
||||
if chunk.Usage != nil {
|
||||
s.inputTokens = chunk.Usage.PromptTokens
|
||||
s.outputTokens = chunk.Usage.CompletionTokens
|
||||
}
|
||||
var out []AnthropicStreamEvent
|
||||
if chunk.Usage != nil && len(chunk.Choices) == 0 {
|
||||
closing, err := s.emitClosing()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return closing, nil
|
||||
}
|
||||
if len(chunk.Choices) == 0 {
|
||||
if endOfStream && !s.closingEmitted {
|
||||
return s.emitClosing()
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
choice := chunk.Choices[0]
|
||||
if !s.messageStarted {
|
||||
events, err := s.emitMessageStart()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, events...)
|
||||
}
|
||||
if choice.Delta.Content != nil && *choice.Delta.Content != "" {
|
||||
if !s.hasOpenBlock {
|
||||
events, err := s.emitTextBlockStart()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, events...)
|
||||
}
|
||||
events, err := s.emitTextDelta(*choice.Delta.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, events...)
|
||||
}
|
||||
for _, tc := range choice.Delta.ToolCalls {
|
||||
events, err := s.handleToolDelta(tc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, events...)
|
||||
}
|
||||
if choice.FinishReason != "" {
|
||||
s.stopReason = openAIFinishReasonToAnthropic(choice.FinishReason)
|
||||
}
|
||||
if endOfStream && !s.closingEmitted {
|
||||
closing, err := s.emitClosing()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, closing...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *AnthropicStreamConverter) emitMessageStart() ([]AnthropicStreamEvent, error) {
|
||||
s.messageStarted = true
|
||||
model := s.model
|
||||
if model == "" {
|
||||
model = s.requestModel
|
||||
}
|
||||
data, err := json.Marshal(map[string]interface{}{
|
||||
"type": "message_start",
|
||||
"message": map[string]interface{}{
|
||||
"id": s.messageID,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": []interface{}{},
|
||||
"stop_reason": nil,
|
||||
"stop_sequence": nil,
|
||||
"usage": map[string]interface{}{
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []AnthropicStreamEvent{{Event: "message_start", Data: data}}, nil
|
||||
}
|
||||
|
||||
func (s *AnthropicStreamConverter) emitTextBlockStart() ([]AnthropicStreamEvent, error) {
|
||||
s.hasOpenBlock = true
|
||||
s.openBlockIndex = s.blockIndex
|
||||
data, err := json.Marshal(map[string]interface{}{
|
||||
"type": "content_block_start",
|
||||
"index": s.blockIndex,
|
||||
"content_block": map[string]interface{}{"type": "text", "text": ""},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []AnthropicStreamEvent{{Event: "content_block_start", Data: data}}, nil
|
||||
}
|
||||
|
||||
func (s *AnthropicStreamConverter) emitTextDelta(text string) ([]AnthropicStreamEvent, error) {
|
||||
data, err := json.Marshal(map[string]interface{}{
|
||||
"type": "content_block_delta",
|
||||
"index": s.blockIndex,
|
||||
"delta": map[string]interface{}{"type": "text_delta", "text": text},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []AnthropicStreamEvent{{Event: "content_block_delta", Data: data}}, nil
|
||||
}
|
||||
|
||||
func (s *AnthropicStreamConverter) handleToolDelta(tc ToolCall) ([]AnthropicStreamEvent, error) {
|
||||
var out []AnthropicStreamEvent
|
||||
idx := tc.Index
|
||||
st, ok := s.activeTools[idx]
|
||||
if !ok {
|
||||
if s.hasOpenBlock {
|
||||
events, err := s.closeOpenBlock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, events...)
|
||||
}
|
||||
st = &streamToolState{blockIdx: s.blockIndex}
|
||||
s.activeTools[idx] = st
|
||||
id := strings.TrimSpace(tc.ID)
|
||||
name := strings.TrimSpace(tc.Function.Name)
|
||||
if id != "" {
|
||||
st.id = id
|
||||
}
|
||||
if name != "" {
|
||||
st.name = name
|
||||
}
|
||||
if st.id == "" {
|
||||
st.id = "toolu_" + st.name
|
||||
}
|
||||
data, err := json.Marshal(map[string]interface{}{
|
||||
"type": "content_block_start",
|
||||
"index": st.blockIdx,
|
||||
"content_block": map[string]interface{}{
|
||||
"type": "tool_use",
|
||||
"id": st.id,
|
||||
"name": st.name,
|
||||
"input": map[string]interface{}{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.hasOpenBlock = true
|
||||
s.openBlockIndex = st.blockIdx
|
||||
s.blockIndex++
|
||||
out = append(out, AnthropicStreamEvent{Event: "content_block_start", Data: data})
|
||||
}
|
||||
if tc.Function.Name != "" {
|
||||
st.name = tc.Function.Name
|
||||
}
|
||||
if tc.ID != "" {
|
||||
st.id = tc.ID
|
||||
}
|
||||
if tc.Function.Arguments == "" {
|
||||
return out, nil
|
||||
}
|
||||
data, err := json.Marshal(map[string]interface{}{
|
||||
"type": "content_block_delta",
|
||||
"index": st.blockIdx,
|
||||
"delta": map[string]interface{}{
|
||||
"type": "input_json_delta",
|
||||
"partial_json": tc.Function.Arguments,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AnthropicStreamEvent{Event: "content_block_delta", Data: data})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *AnthropicStreamConverter) closeOpenBlock() ([]AnthropicStreamEvent, error) {
|
||||
if !s.hasOpenBlock {
|
||||
return nil, nil
|
||||
}
|
||||
idx := s.openBlockIndex
|
||||
if idx < 0 {
|
||||
idx = s.blockIndex
|
||||
}
|
||||
data, err := json.Marshal(map[string]interface{}{
|
||||
"type": "content_block_stop",
|
||||
"index": idx,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.hasOpenBlock = false
|
||||
s.openBlockIndex = -1
|
||||
events := []AnthropicStreamEvent{{Event: "content_block_stop", Data: data}}
|
||||
if s.blockIndex <= idx {
|
||||
s.blockIndex = idx + 1
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
func (s *AnthropicStreamConverter) emitClosing() ([]AnthropicStreamEvent, error) {
|
||||
if s.closingEmitted {
|
||||
return nil, nil
|
||||
}
|
||||
s.closingEmitted = true
|
||||
var out []AnthropicStreamEvent
|
||||
if s.hasOpenBlock {
|
||||
events, err := s.closeOpenBlock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, events...)
|
||||
}
|
||||
stopReason := s.stopReason
|
||||
if stopReason == "" {
|
||||
stopReason = "end_turn"
|
||||
}
|
||||
deltaData, err := json.Marshal(map[string]interface{}{
|
||||
"type": "message_delta",
|
||||
"delta": map[string]interface{}{
|
||||
"stop_reason": stopReason,
|
||||
"stop_sequence": nil,
|
||||
},
|
||||
"usage": map[string]interface{}{
|
||||
"output_tokens": s.outputTokens,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AnthropicStreamEvent{Event: "message_delta", Data: deltaData})
|
||||
|
||||
stopData, err := json.Marshal(map[string]interface{}{"type": "message_stop"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AnthropicStreamEvent{Event: "message_stop", Data: stopData})
|
||||
return out, nil
|
||||
}
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun"
|
||||
)
|
||||
|
||||
func TestAliyunProviderEnableThinkingPatch(t *testing.T) {
|
||||
@@ -26,7 +28,7 @@ func TestAliyunProviderEnableThinkingPatch(t *testing.T) {
|
||||
body.Add(jsonutils.NewString("qwen-turbo"), "model")
|
||||
body.Add(jsonutils.NewArray(jsonutils.NewDict()), "messages")
|
||||
|
||||
p := Get("aliyun")
|
||||
p := aliyun.New()
|
||||
req, err := p.BuildUpstreamRequest(&ChatContext{
|
||||
ProviderKey: "aliyun",
|
||||
BaseURL: "https://dashscope.aliyuncs.com/compatible-mode",
|
||||
|
||||
@@ -19,11 +19,12 @@ import (
|
||||
"sync"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
// "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun" // uncommon
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/azure"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/baidu"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/cohere"
|
||||
// "yunion.io/x/onecloud/pkg/aiproxy/providers/azure" // uncommon
|
||||
// "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu" // uncommon
|
||||
// "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere" // uncommon
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/gemini"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/vllm"
|
||||
@@ -35,36 +36,20 @@ var (
|
||||
defaultP providerapi.Provider
|
||||
)
|
||||
|
||||
var openAICompatKeys = []string{
|
||||
"openai",
|
||||
"groq",
|
||||
"mistral",
|
||||
"cerebras",
|
||||
"perplexity",
|
||||
"openrouter",
|
||||
"fireworks",
|
||||
"nebius",
|
||||
"xai",
|
||||
"parasail",
|
||||
"sgl",
|
||||
"huggingface",
|
||||
"ollama",
|
||||
"xiaomi",
|
||||
}
|
||||
|
||||
func init() {
|
||||
defaultP = openai.NewCompat("")
|
||||
register(defaultP)
|
||||
for _, key := range openAICompatKeys {
|
||||
for _, key := range api.OpenAICompatProviderKeys {
|
||||
register(openai.NewCompat(key))
|
||||
}
|
||||
register(cohere.New())
|
||||
register(aliyun.New())
|
||||
register(baidu.New())
|
||||
// register(cohere.New()) // uncommon
|
||||
// register(aliyun.New()) // uncommon
|
||||
// register(baidu.New()) // uncommon
|
||||
register(anthropic.New())
|
||||
register(gemini.New())
|
||||
register(azure.New())
|
||||
// register(azure.New()) // uncommon
|
||||
register(vllm.New())
|
||||
register(openai.NewCompat(api.ProviderKeyCustom))
|
||||
}
|
||||
|
||||
// Register adds or replaces a provider implementation for its Key().
|
||||
|
||||
@@ -19,14 +19,16 @@ import (
|
||||
)
|
||||
|
||||
type (
|
||||
ChatContext = api.ChatContext
|
||||
HTTPRequest = api.HTTPRequest
|
||||
StreamChunk = api.StreamChunk
|
||||
StreamState = api.StreamState
|
||||
Provider = api.Provider
|
||||
EmbeddingsProvider = api.EmbeddingsProvider
|
||||
ImagesProvider = api.ImagesProvider
|
||||
CompletionsProvider = api.CompletionsProvider
|
||||
ChatContext = api.ChatContext
|
||||
HTTPRequest = api.HTTPRequest
|
||||
StreamChunk = api.StreamChunk
|
||||
StreamState = api.StreamState
|
||||
Provider = api.Provider
|
||||
EmbeddingsProvider = api.EmbeddingsProvider
|
||||
ImagesProvider = api.ImagesProvider
|
||||
CompletionsProvider = api.CompletionsProvider
|
||||
MessagesAdapter = api.MessagesAdapter
|
||||
AnthropicStreamChunk = api.AnthropicStreamChunk
|
||||
)
|
||||
|
||||
type ContextualStreamPassthrough = api.ContextualStreamPassthrough
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
|
||||
"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
|
||||
api "yunion.io/x/onecloud/pkg/apis/aiproxy"
|
||||
)
|
||||
|
||||
type provider struct {
|
||||
@@ -39,7 +40,7 @@ func patchVLLMRequest(body *jsonutils.JSONDict, stream bool) {
|
||||
func New() providerapi.Provider {
|
||||
patches := []openai.PatchFunc{patchVLLMRequest}
|
||||
return &provider{
|
||||
Compat: openai.NewCompat("vllm", patches...),
|
||||
Compat: openai.NewCompat(api.ProviderKeyVLLM, patches...),
|
||||
completions: openai.NewCompletionsCompat(patches...),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -84,6 +85,27 @@ func ChatCompletionsURL(baseURL string) string {
|
||||
return base + "/v1/chat/completions"
|
||||
}
|
||||
|
||||
// ModelsURL builds the OpenAI-compatible models list endpoint from a provider base URL.
|
||||
func ModelsURL(baseURL string) string {
|
||||
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
if strings.HasSuffix(base, "/v1") {
|
||||
return base + "/models"
|
||||
}
|
||||
if hasAPIVersionPathSuffix(base) {
|
||||
return base + "/models"
|
||||
}
|
||||
return base + "/v1/models"
|
||||
}
|
||||
|
||||
func hasAPIVersionPathSuffix(base string) bool {
|
||||
idx := strings.LastIndex(base, "/")
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
seg := base[idx+1:]
|
||||
return len(seg) >= 2 && seg[0] == 'v' && seg[1] >= '0' && seg[1] <= '9'
|
||||
}
|
||||
|
||||
var (
|
||||
httpClient *http.Client
|
||||
httpClientOnce sync.Once
|
||||
@@ -182,6 +204,136 @@ func ChatCompletion(ctx context.Context, req *Request) (*Response, *Error) {
|
||||
return &Response{StatusCode: resp.StatusCode, Body: body}, nil
|
||||
}
|
||||
|
||||
// ListModels performs a GET on the upstream models list endpoint.
|
||||
func ListModels(ctx context.Context, baseURL, apiKey string) (*Response, *Error) {
|
||||
req := &Request{
|
||||
BaseURL: strings.TrimSpace(baseURL),
|
||||
URL: ModelsURL(baseURL),
|
||||
APIKey: strings.TrimSpace(apiKey),
|
||||
}
|
||||
httpReq, err := newUpstreamGETRequest(ctx, req)
|
||||
if err != nil {
|
||||
return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
|
||||
}
|
||||
resp, err := sharedHTTPClient().Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
|
||||
}
|
||||
body, err := readResponseBody(resp, 4<<20)
|
||||
if err != nil {
|
||||
return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, errorFromResponse(resp, body)
|
||||
}
|
||||
if err := validateModelsListBody(body); err != nil {
|
||||
return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
|
||||
}
|
||||
if _, err := ParseModelsListBody(body); err != nil {
|
||||
return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
|
||||
}
|
||||
return &Response{StatusCode: resp.StatusCode, Body: body}, nil
|
||||
}
|
||||
|
||||
// ParseModelsListBody extracts upstream model ids from a list-models JSON body.
|
||||
func ParseModelsListBody(body []byte) ([]string, error) {
|
||||
if err := validateModelsListBody(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, fmt.Errorf("invalid models response JSON")
|
||||
}
|
||||
keys := make([]string, 0)
|
||||
if dataRaw, ok := raw["data"]; ok {
|
||||
var items []struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(dataRaw, &items); err != nil {
|
||||
return nil, fmt.Errorf("invalid models data array")
|
||||
}
|
||||
for _, item := range items {
|
||||
if id := strings.TrimSpace(item.ID); id != "" {
|
||||
keys = append(keys, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if modelsRaw, ok := raw["models"]; ok {
|
||||
var items []struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := json.Unmarshal(modelsRaw, &items); err != nil {
|
||||
return nil, fmt.Errorf("invalid models array")
|
||||
}
|
||||
for _, item := range items {
|
||||
name := strings.TrimSpace(item.Name)
|
||||
name = strings.TrimPrefix(name, "models/")
|
||||
if name != "" {
|
||||
keys = append(keys, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return []string{}, nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
uniq := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
uniq = append(uniq, key)
|
||||
}
|
||||
sort.Strings(uniq)
|
||||
return uniq, nil
|
||||
}
|
||||
|
||||
func newUpstreamGETRequest(ctx context.Context, req *Request) (*http.Request, error) {
|
||||
if req == nil {
|
||||
return nil, fmt.Errorf("nil upstream request")
|
||||
}
|
||||
url := strings.TrimSpace(req.URL)
|
||||
if url == "" {
|
||||
url = ModelsURL(req.BaseURL)
|
||||
}
|
||||
apiKey := strings.TrimSpace(req.APIKey)
|
||||
if url == "" {
|
||||
return nil, fmt.Errorf("empty upstream URL")
|
||||
}
|
||||
if apiKey == "" && len(req.Headers) == 0 {
|
||||
return nil, fmt.Errorf("empty API key")
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range req.Headers {
|
||||
httpReq.Header.Set(k, v)
|
||||
}
|
||||
if apiKey != "" && httpReq.Header.Get("Authorization") == "" && httpReq.Header.Get("x-api-key") == "" && httpReq.Header.Get("api-key") == "" && httpReq.Header.Get("x-goog-api-key") == "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
func validateModelsListBody(body []byte) error {
|
||||
if len(body) == 0 {
|
||||
return fmt.Errorf("empty models response")
|
||||
}
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return fmt.Errorf("invalid models response JSON")
|
||||
}
|
||||
if _, ok := raw["data"]; ok {
|
||||
return nil
|
||||
}
|
||||
if _, ok := raw["models"]; ok {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("models response missing data or models field")
|
||||
}
|
||||
|
||||
// ChatCompletionStream opens a streaming chat completions request and returns SSE data chunks.
|
||||
func ChatCompletionStream(ctx context.Context, req *Request) (<-chan StreamChunk, *Error) {
|
||||
_, resp, uerr := openChatCompletionStream(ctx, req)
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package upstream
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestModelsURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
base string
|
||||
want string
|
||||
}{
|
||||
{"https://api.openai.com", "https://api.openai.com/v1/models"},
|
||||
{"https://api.openai.com/v1", "https://api.openai.com/v1/models"},
|
||||
{"https://generativelanguage.googleapis.com/v1beta", "https://generativelanguage.googleapis.com/v1beta/models"},
|
||||
{"https://api.deepseek.com/anthropic", "https://api.deepseek.com/anthropic/v1/models"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := ModelsURL(tc.base); got != tc.want {
|
||||
t.Fatalf("ModelsURL(%q) = %q, want %q", tc.base, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateModelsListBody(t *testing.T) {
|
||||
if err := validateModelsListBody([]byte(`{"object":"list","data":[]}`)); err != nil {
|
||||
t.Fatalf("expected valid data field: %v", err)
|
||||
}
|
||||
if err := validateModelsListBody([]byte(`{"models":[]}`)); err != nil {
|
||||
t.Fatalf("expected valid models field: %v", err)
|
||||
}
|
||||
if err := validateModelsListBody([]byte(`{"object":"list"}`)); err == nil {
|
||||
t.Fatal("expected error for missing data/models")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseModelsListBodyOpenAI(t *testing.T) {
|
||||
keys, err := ParseModelsListBody([]byte(`{"object":"list","data":[{"id":"gpt-4o-mini"},{"id":"gpt-4o"}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(keys) != 2 || keys[0] != "gpt-4o" || keys[1] != "gpt-4o-mini" {
|
||||
t.Fatalf("keys: %#v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseModelsListBodyGemini(t *testing.T) {
|
||||
keys, err := ParseModelsListBody([]byte(`{"models":[{"name":"models/gemini-2.0-flash"},{"name":"models/gemini-pro"}]}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(keys) != 2 || keys[0] != "gemini-2.0-flash" || keys[1] != "gemini-pro" {
|
||||
t.Fatalf("keys: %#v", keys)
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ package aiproxy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/apis"
|
||||
@@ -24,7 +25,29 @@ import (
|
||||
// SAiProviderConfig holds JSON-serialized provider connectivity settings for an ai_provider row.
|
||||
type SAiProviderConfig struct {
|
||||
BaseURL string `json:"base_url,omitempty"`
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
APIMode string `json:"api_mode,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON rejects legacy config.api_key and decodes supported fields only.
|
||||
func (c *SAiProviderConfig) UnmarshalJSON(data []byte) error {
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return err
|
||||
}
|
||||
if msg, ok := raw["api_key"]; ok {
|
||||
var key string
|
||||
_ = json.Unmarshal(msg, &key)
|
||||
if strings.TrimSpace(key) != "" {
|
||||
return errors.New("config.api_key is not supported, use secret and ai_keys")
|
||||
}
|
||||
}
|
||||
type cfgAlias SAiProviderConfig
|
||||
var alias cfgAlias
|
||||
if err := json.Unmarshal(data, &alias); err != nil {
|
||||
return err
|
||||
}
|
||||
*c = SAiProviderConfig(alias)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResolvedBaseURL returns config.base_url.
|
||||
@@ -35,12 +58,39 @@ func (c *SAiProviderConfig) ResolvedBaseURL() string {
|
||||
return strings.TrimSpace(c.BaseURL)
|
||||
}
|
||||
|
||||
// ResolvedAPIKey returns config.api_key.
|
||||
func (c *SAiProviderConfig) ResolvedAPIKey() string {
|
||||
// ResolvedAPIMode returns config.api_mode (default openai).
|
||||
func (c *SAiProviderConfig) ResolvedAPIMode() string {
|
||||
if c == nil {
|
||||
return ProviderAPIModeOpenAI
|
||||
}
|
||||
mode := strings.ToLower(strings.TrimSpace(c.APIMode))
|
||||
if mode == "" {
|
||||
return ProviderAPIModeOpenAI
|
||||
}
|
||||
return mode
|
||||
}
|
||||
|
||||
// EffectiveBaseURL returns the upstream base URL adjusted for api_mode and provider_key.
|
||||
func (c *SAiProviderConfig) EffectiveBaseURL(providerKey string) string {
|
||||
base := c.ResolvedBaseURL()
|
||||
if base == "" {
|
||||
base = DefaultPublicBaseURL(providerKey)
|
||||
}
|
||||
if base == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(c.APIKey)
|
||||
if c.ResolvedAPIMode() != ProviderAPIModeAnthropic {
|
||||
return base
|
||||
}
|
||||
pk := strings.ToLower(strings.TrimSpace(providerKey))
|
||||
if pk != ProviderKeyDeepseek {
|
||||
return base
|
||||
}
|
||||
base = strings.TrimRight(base, "/")
|
||||
if strings.HasSuffix(strings.ToLower(base), "/anthropic") {
|
||||
return base
|
||||
}
|
||||
return base + "/anthropic"
|
||||
}
|
||||
|
||||
// String implements gotypes.ISerializable for sqlchemy JSON/compound columns.
|
||||
@@ -60,7 +110,7 @@ func (c *SAiProviderConfig) IsZero() bool {
|
||||
if c == nil {
|
||||
return true
|
||||
}
|
||||
return c.ResolvedBaseURL() == "" && c.ResolvedAPIKey() == ""
|
||||
return c.ResolvedBaseURL() == "" && strings.TrimSpace(c.APIMode) == ""
|
||||
}
|
||||
|
||||
type AiProviderListInput struct {
|
||||
@@ -76,6 +126,8 @@ type AiProviderCreateInput struct {
|
||||
|
||||
ProviderKey string `json:"provider_key"`
|
||||
Config *SAiProviderConfig `json:"config"`
|
||||
Secret string `json:"secret"`
|
||||
ModelKeys []string `json:"model_keys"`
|
||||
LlmDeploymentId string `json:"llm_deployment_id"`
|
||||
LlmId string `json:"llm_id"`
|
||||
}
|
||||
@@ -98,3 +150,25 @@ type AiProviderDetails struct {
|
||||
LlmDeploymentId string `json:"llm_deployment_id"`
|
||||
LlmId string `json:"llm_id"`
|
||||
}
|
||||
|
||||
type AiProviderTestConnectivityInput struct {
|
||||
ProviderKey string `json:"provider_key"`
|
||||
Secret string `json:"secret"`
|
||||
Config *SAiProviderConfig `json:"config"`
|
||||
}
|
||||
|
||||
const (
|
||||
AiProviderModelsSourceUpstream = "upstream"
|
||||
AiProviderModelsSourceCatalog = "catalog"
|
||||
)
|
||||
|
||||
type AiProviderUpstreamModel struct {
|
||||
ModelKey string `json:"model_key"`
|
||||
}
|
||||
|
||||
type AiProviderTestConnectivityOutput struct {
|
||||
Ok bool `json:"ok"`
|
||||
Message string `json:"message"`
|
||||
ModelsSource string `json:"models_source"`
|
||||
Models []AiProviderUpstreamModel `json:"models"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package aiproxy
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestResolvedAPIModeDefault(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{}
|
||||
if got := cfg.ResolvedAPIMode(); got != ProviderAPIModeOpenAI {
|
||||
t.Fatalf("ResolvedAPIMode() = %q, want %q", got, ProviderAPIModeOpenAI)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveBaseURLDeepseekAnthropic(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{
|
||||
BaseURL: "https://api.deepseek.com",
|
||||
APIMode: ProviderAPIModeAnthropic,
|
||||
}
|
||||
got := cfg.EffectiveBaseURL(ProviderKeyDeepseek)
|
||||
want := "https://api.deepseek.com/anthropic"
|
||||
if got != want {
|
||||
t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveBaseURLDeepseekOpenAI(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{
|
||||
BaseURL: "https://api.deepseek.com/anthropic",
|
||||
APIMode: ProviderAPIModeOpenAI,
|
||||
}
|
||||
got := cfg.EffectiveBaseURL(ProviderKeyDeepseek)
|
||||
want := "https://api.deepseek.com/anthropic"
|
||||
if got != want {
|
||||
t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsDualAPIMode(t *testing.T) {
|
||||
if !SupportsDualAPIMode(ProviderKeyDeepseek) {
|
||||
t.Fatal("deepseek should support dual api mode")
|
||||
}
|
||||
if !SupportsDualAPIMode(ProviderKeyCustom) {
|
||||
t.Fatal("custom should support dual api mode")
|
||||
}
|
||||
if SupportsDualAPIMode(ProviderKeyOpenAI) {
|
||||
t.Fatal("openai should not support dual api mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCustomProviderKey(t *testing.T) {
|
||||
if !IsCustomProviderKey(ProviderKeyCustom) {
|
||||
t.Fatal("custom key should match")
|
||||
}
|
||||
if IsCustomProviderKey(ProviderKeyOpenAI) {
|
||||
t.Fatal("openai should not be custom")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveBaseURLFallbackOpenAI(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{}
|
||||
got := cfg.EffectiveBaseURL(ProviderKeyOpenAI)
|
||||
want := "https://api.openai.com"
|
||||
if got != want {
|
||||
t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveBaseURLFallbackDeepseekAnthropic(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{APIMode: ProviderAPIModeAnthropic}
|
||||
got := cfg.EffectiveBaseURL(ProviderKeyDeepseek)
|
||||
want := "https://api.deepseek.com/anthropic"
|
||||
if got != want {
|
||||
t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveBaseURLCustomNoAnthropicSuffix(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{
|
||||
BaseURL: "https://llm.example.com/v1",
|
||||
APIMode: ProviderAPIModeAnthropic,
|
||||
}
|
||||
got := cfg.EffectiveBaseURL(ProviderKeyCustom)
|
||||
want := "https://llm.example.com/v1"
|
||||
if got != want {
|
||||
t.Fatalf("EffectiveBaseURL() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasDefaultPublicBaseURL(t *testing.T) {
|
||||
if !HasDefaultPublicBaseURL(ProviderKeyOpenAI) {
|
||||
t.Fatal("openai should have default base url")
|
||||
}
|
||||
if HasDefaultPublicBaseURL(ProviderKeyAzure) {
|
||||
t.Fatal("azure should not have default base url")
|
||||
}
|
||||
if HasDefaultPublicBaseURL(ProviderKeyCustom) {
|
||||
t.Fatal("custom should not have default base url")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package aiproxy
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
)
|
||||
|
||||
func TestSAiProviderConfigRejectsAPIKey(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{}
|
||||
obj, err := jsonutils.Parse([]byte(`{"base_url":"https://api.openai.com","api_key":"sk-test"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
err = obj.Unmarshal(cfg)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for config.api_key")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "config.api_key is not supported") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSAiProviderConfigAllowsWithoutAPIKey(t *testing.T) {
|
||||
cfg := &SAiProviderConfig{}
|
||||
obj, err := jsonutils.Parse([]byte(`{"base_url":"https://api.openai.com","api_mode":"openai"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if err := obj.Unmarshal(cfg); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if cfg.ResolvedBaseURL() != "https://api.openai.com" {
|
||||
t.Fatalf("base_url = %q", cfg.ResolvedBaseURL())
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,11 @@ type AiRoutingModelUpdateInput struct {
|
||||
type AiRoutingModelDetails struct {
|
||||
apis.StandaloneResourceDetails
|
||||
|
||||
// Id and Name are set explicitly for nested routing_models on ai_routing details
|
||||
// (list responses also merge the full model row, which includes these fields).
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
|
||||
AiRoutingId string `json:"ai_routing_id"`
|
||||
AiProviderId string `json:"ai_provider_id"`
|
||||
AiModelId string `json:"ai_model_id"`
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package aiproxy
|
||||
|
||||
import "strings"
|
||||
|
||||
// DefaultPublicBaseURL returns a well-known public API base for catalog providers.
|
||||
// Empty string means no default (user must set config.base_url).
|
||||
func DefaultPublicBaseURL(providerKey string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(providerKey)) {
|
||||
case ProviderKeyOpenAI:
|
||||
return "https://api.openai.com"
|
||||
case ProviderKeyAnthropic:
|
||||
return "https://api.anthropic.com"
|
||||
case ProviderKeyAzure, ProviderKeyBedrock, ProviderKeySGLang, ProviderKeyOllama, ProviderKeyVLLM, ProviderKeyAliyun, ProviderKeyBaidu:
|
||||
return ""
|
||||
case ProviderKeyDeepseek:
|
||||
return "https://api.deepseek.com"
|
||||
case ProviderKeyGemini:
|
||||
return "https://generativelanguage.googleapis.com/v1beta"
|
||||
case ProviderKeyGroq:
|
||||
return "https://api.groq.com/openai"
|
||||
case ProviderKeyMistral:
|
||||
return "https://api.mistral.ai"
|
||||
case ProviderKeyOpenrouter:
|
||||
return "https://openrouter.ai/api"
|
||||
case ProviderKeyHuggingface:
|
||||
return "https://router.huggingface.co"
|
||||
case ProviderKeyXiaomi:
|
||||
return "https://api.xiaomimimo.com"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// HasDefaultPublicBaseURL reports whether provider_key has a built-in public base URL.
|
||||
func HasDefaultPublicBaseURL(providerKey string) bool {
|
||||
return DefaultPublicBaseURL(providerKey) != ""
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package aiproxy
|
||||
|
||||
import "strings"
|
||||
|
||||
// Built-in catalog provider_key values (seeded at InitDB and registered in providers).
|
||||
const (
|
||||
ProviderKeyAliyun = "aliyun"
|
||||
ProviderKeyAnthropic = "anthropic"
|
||||
ProviderKeyAzure = "azure"
|
||||
ProviderKeyBaidu = "baidu"
|
||||
ProviderKeyBedrock = "bedrock"
|
||||
ProviderKeyCerebras = "cerebras"
|
||||
ProviderKeyCohere = "cohere"
|
||||
ProviderKeyCustom = "custom"
|
||||
ProviderKeyDeepseek = "deepseek"
|
||||
ProviderKeyElevenlabs = "elevenlabs"
|
||||
ProviderKeyFireworks = "fireworks"
|
||||
ProviderKeyGemini = "gemini"
|
||||
ProviderKeyGroq = "groq"
|
||||
ProviderKeyHuggingface = "huggingface"
|
||||
ProviderKeyMistral = "mistral"
|
||||
ProviderKeyNebius = "nebius"
|
||||
ProviderKeyOllama = "ollama"
|
||||
ProviderKeyOpenAI = "openai"
|
||||
ProviderKeyOpenrouter = "openrouter"
|
||||
ProviderKeyParasail = "parasail"
|
||||
ProviderKeyPerplexity = "perplexity"
|
||||
ProviderKeyReplicate = "replicate"
|
||||
ProviderKeyRunway = "runway"
|
||||
ProviderKeySGLang = "sglang"
|
||||
ProviderKeyVertex = "vertex"
|
||||
ProviderKeyVLLM = "vllm"
|
||||
ProviderKeyXai = "xai"
|
||||
ProviderKeyXiaomi = "xiaomi"
|
||||
)
|
||||
|
||||
// StandardCatalogProviderKeys lists built-in provider_key values seeded at InitDB.
|
||||
var StandardCatalogProviderKeys = []string{
|
||||
ProviderKeyAnthropic,
|
||||
// ProviderKeyAzure, // uncommon
|
||||
// ProviderKeyBedrock, // uncommon
|
||||
// ProviderKeyCerebras, // uncommon
|
||||
// ProviderKeyCohere, // uncommon
|
||||
ProviderKeyDeepseek,
|
||||
ProviderKeyGemini,
|
||||
ProviderKeyGroq,
|
||||
ProviderKeyMistral,
|
||||
ProviderKeyOllama,
|
||||
ProviderKeyOpenAI,
|
||||
// ProviderKeyParasail, // uncommon
|
||||
// ProviderKeyPerplexity, // uncommon
|
||||
ProviderKeySGLang,
|
||||
// ProviderKeyVertex, // uncommon
|
||||
ProviderKeyOpenrouter,
|
||||
// ProviderKeyElevenlabs, // uncommon
|
||||
ProviderKeyHuggingface,
|
||||
// ProviderKeyNebius, // uncommon
|
||||
// ProviderKeyXai, // uncommon
|
||||
// ProviderKeyReplicate, // uncommon
|
||||
ProviderKeyVLLM,
|
||||
// ProviderKeyRunway, // uncommon
|
||||
// ProviderKeyFireworks, // uncommon
|
||||
// ProviderKeyAliyun, // uncommon
|
||||
// ProviderKeyBaidu, // uncommon
|
||||
ProviderKeyXiaomi,
|
||||
}
|
||||
|
||||
// OpenAICompatProviderKeys are catalog keys routed through openai.NewCompat.
|
||||
var OpenAICompatProviderKeys = []string{
|
||||
ProviderKeyOpenAI,
|
||||
ProviderKeyGroq,
|
||||
ProviderKeyMistral,
|
||||
// ProviderKeyCerebras, // uncommon
|
||||
ProviderKeyDeepseek,
|
||||
// ProviderKeyPerplexity, // uncommon
|
||||
ProviderKeyOpenrouter,
|
||||
// ProviderKeyFireworks, // uncommon
|
||||
// ProviderKeyNebius, // uncommon
|
||||
// ProviderKeyXai, // uncommon
|
||||
// ProviderKeyParasail, // uncommon
|
||||
ProviderKeySGLang,
|
||||
ProviderKeyHuggingface,
|
||||
ProviderKeyOllama,
|
||||
ProviderKeyXiaomi,
|
||||
}
|
||||
|
||||
var nativeMessagesAdapterProviderKeys = map[string]struct{}{
|
||||
ProviderKeyGemini: {},
|
||||
// ProviderKeyCohere: {}, // uncommon
|
||||
// ProviderKeyBaidu: {}, // uncommon
|
||||
// ProviderKeyAliyun: {}, // uncommon
|
||||
}
|
||||
|
||||
// IsNativeMessagesAdapterProvider reports whether provider_key uses a dedicated
|
||||
// non-OpenAI-chat upstream API that cannot be reached via Anthropic-to-OpenAI translation.
|
||||
func IsNativeMessagesAdapterProvider(providerKey string) bool {
|
||||
key := strings.ToLower(strings.TrimSpace(providerKey))
|
||||
_, ok := nativeMessagesAdapterProviderKeys[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
const (
|
||||
ProviderAPIModeOpenAI = "openai"
|
||||
ProviderAPIModeAnthropic = "anthropic"
|
||||
)
|
||||
|
||||
// DualAPIProviderKeys lists provider_key values that support openai and anthropic upstream APIs.
|
||||
var DualAPIProviderKeys = map[string]struct{}{
|
||||
ProviderKeyCustom: {},
|
||||
ProviderKeyDeepseek: {},
|
||||
}
|
||||
|
||||
// IsCustomProviderKey reports whether provider_key is the user-defined custom gateway type.
|
||||
func IsCustomProviderKey(providerKey string) bool {
|
||||
return strings.ToLower(strings.TrimSpace(providerKey)) == ProviderKeyCustom
|
||||
}
|
||||
|
||||
// SupportsDualAPIMode reports whether provider_key may use config.api_mode.
|
||||
func SupportsDualAPIMode(providerKey string) bool {
|
||||
key := strings.ToLower(strings.TrimSpace(providerKey))
|
||||
_, ok := DualAPIProviderKeys[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
// IsValidProviderAPIMode reports whether mode is a supported api_mode value.
|
||||
func IsValidProviderAPIMode(mode string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(mode)) {
|
||||
case "", ProviderAPIModeOpenAI, ProviderAPIModeAnthropic:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
"yunion.io/x/pkg/util/printutils"
|
||||
)
|
||||
|
||||
const aiproxyPlaceholderAPIKey = "unused"
|
||||
|
||||
func aiproxyAdminSession(ctx context.Context) *mcclient.ClientSession {
|
||||
return auth.GetAdminSession(ctx, options.Options.Region)
|
||||
}
|
||||
@@ -34,7 +32,7 @@ func mapLLMTypeToProviderKey(llmType string) (string, bool) {
|
||||
case string(api.LLM_CONTAINER_OLLAMA):
|
||||
return "ollama", true
|
||||
case string(api.LLM_CONTAINER_SGLANG):
|
||||
return "sgl", true
|
||||
return "sglang", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
@@ -300,7 +298,6 @@ func upsertAiProvider(
|
||||
}
|
||||
cfg := jsonutils.Marshal(&apapi.SAiProviderConfig{
|
||||
BaseURL: baseURL,
|
||||
APIKey: aiproxyPlaceholderAPIKey,
|
||||
})
|
||||
params := jsonutils.NewDict()
|
||||
params.Set("provider_key", jsonutils.NewString(providerKey))
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/llm"
|
||||
)
|
||||
|
||||
func TestMapLLMTypeToProviderKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
key string
|
||||
ok bool
|
||||
}{
|
||||
{string(api.LLM_CONTAINER_VLLM), "vllm", true},
|
||||
{string(api.LLM_CONTAINER_OLLAMA), "ollama", true},
|
||||
{string(api.LLM_CONTAINER_SGLANG), "sgl", true},
|
||||
{"dify", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
key, ok := mapLLMTypeToProviderKey(c.in)
|
||||
if ok != c.ok || key != c.key {
|
||||
t.Fatalf("mapLLMTypeToProviderKey(%q) = (%q, %v), want (%q, %v)", c.in, key, ok, c.key, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugModelKey(t *testing.T) {
|
||||
if got := slugModelKey("Qwen/Qwen2.5-7B-Instruct"); got != "qwen-qwen2-5-7b-instruct" {
|
||||
t.Fatalf("slugModelKey got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentClientModelAlias(t *testing.T) {
|
||||
dep := &SLLMDeployment{}
|
||||
dep.Name = "my-qwen"
|
||||
if got := deploymentClientModelAlias(dep, "Qwen3-0.6B"); got != "my-qwen-Qwen3-0.6B" {
|
||||
t.Fatalf("deploymentClientModelAlias got %q", got)
|
||||
}
|
||||
depEmpty := &SLLMDeployment{}
|
||||
depEmpty.Id = "dep-id-1"
|
||||
if got := deploymentClientModelAlias(depEmpty, ""); got != "dep-id-1" {
|
||||
t.Fatalf("deploymentClientModelAlias without model_key got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentRoutingModelKey(t *testing.T) {
|
||||
dep := &SLLMDeployment{}
|
||||
dep.Name = "my-qwen"
|
||||
if got := deploymentRoutingModelKey(dep, "Qwen3-0.6B"); got != "my-qwen-Qwen3-0.6B" {
|
||||
t.Fatalf("deploymentRoutingModelKey got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAiproxyResourceNames(t *testing.T) {
|
||||
dep := &SLLMDeployment{}
|
||||
dep.Name = "My-Qwen"
|
||||
dep.Id = "dep-id-1"
|
||||
if got := aiRoutingNameForDeployment(dep); got != "llm-dep-my-qwen" {
|
||||
t.Fatalf("aiRoutingNameForDeployment got %q", got)
|
||||
}
|
||||
|
||||
depEmpty := &SLLMDeployment{}
|
||||
depEmpty.Id = "dep-id-2"
|
||||
if got := aiRoutingNameForDeployment(depEmpty); got != "llm-dep-dep-id-2" {
|
||||
t.Fatalf("aiRoutingNameForDeployment empty name got %q", got)
|
||||
}
|
||||
|
||||
llm := &SLLM{}
|
||||
llm.Name = "my-qwen-0"
|
||||
llm.Id = "llm-id-1"
|
||||
if got := aiProviderNameForLlm(llm); got != "llm-my-qwen-0" {
|
||||
t.Fatalf("aiProviderNameForLlm got %q", got)
|
||||
}
|
||||
|
||||
llmEmpty := &SLLM{}
|
||||
llmEmpty.Id = "llm-id-2"
|
||||
if got := aiProviderNameForLlm(llmEmpty); got != "llm-llm-id-2" {
|
||||
t.Fatalf("aiProviderNameForLlm empty name got %q", got)
|
||||
}
|
||||
|
||||
if got := aiModelNameForLlm(llm, "Qwen/Qwen3-0.6B"); got != "llm-my-qwen-0-qwen-qwen3-0-6b" {
|
||||
t.Fatalf("aiModelNameForLlm got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearDeploymentAiproxyRegistrationState(t *testing.T) {
|
||||
dep := &SLLMDeployment{}
|
||||
dep.AutoRegisterAiproxy = true
|
||||
dep.AiproxyRoutingId = "routing-1"
|
||||
dep.AiproxyBindings = &api.AiproxyBindings{{LlmId: "llm-1"}}
|
||||
dep.AiproxySyncStatus = api.AIPROXY_SYNC_STATUS_SYNCED
|
||||
|
||||
clearDeploymentAiproxyRegistrationState(dep)
|
||||
|
||||
if dep.AutoRegisterAiproxy {
|
||||
t.Fatal("AutoRegisterAiproxy should be false")
|
||||
}
|
||||
if dep.AiproxyRoutingId != "" {
|
||||
t.Fatalf("AiproxyRoutingId should be empty, got %q", dep.AiproxyRoutingId)
|
||||
}
|
||||
if dep.AiproxyBindings != nil {
|
||||
t.Fatal("AiproxyBindings should be nil")
|
||||
}
|
||||
if dep.AiproxySyncStatus != api.AIPROXY_SYNC_STATUS_DISABLED {
|
||||
t.Fatalf("AiproxySyncStatus should be disabled, got %q", dep.AiproxySyncStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAiproxySyncStatusAfterReconcile(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
result aiproxyBindingSyncResult
|
||||
wantStat string
|
||||
}{
|
||||
{
|
||||
name: "fully synced",
|
||||
result: aiproxyBindingSyncSynced,
|
||||
wantStat: api.AIPROXY_SYNC_STATUS_SYNCED,
|
||||
},
|
||||
{
|
||||
name: "binding partial failure",
|
||||
result: aiproxyBindingSyncPartial,
|
||||
wantStat: api.AIPROXY_SYNC_STATUS_PARTIAL,
|
||||
},
|
||||
{
|
||||
name: "all bindings failed",
|
||||
result: aiproxyBindingSyncFailed,
|
||||
wantStat: api.AIPROXY_SYNC_STATUS_FAILED,
|
||||
},
|
||||
{
|
||||
name: "pending",
|
||||
result: aiproxyBindingSyncPending,
|
||||
wantStat: api.AIPROXY_SYNC_STATUS_PENDING,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := resolveAiproxySyncStatusAfterReconcile(c.result)
|
||||
if got != c.wantStat {
|
||||
t.Fatalf("%s: got %q want %q", c.name, got, c.wantStat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAiproxySyncFailureReason(t *testing.T) {
|
||||
dep := &SLLMDeployment{}
|
||||
dep.AiproxyBindings = &api.AiproxyBindings{
|
||||
{LlmId: "llm-1", SyncStatus: api.AIPROXY_BINDING_SYNC_SYNCED},
|
||||
{LlmId: "llm-2", SyncStatus: api.AIPROXY_BINDING_SYNC_FAILED, LastError: "provider upsert failed"},
|
||||
}
|
||||
got := AiproxySyncFailureReason(dep)
|
||||
want := "llm llm-2: provider upsert failed"
|
||||
if got != want {
|
||||
t.Fatalf("AiproxySyncFailureReason() = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
msg := aiproxySyncStatusMessage(dep, aiproxyBindingSyncFailed)
|
||||
if msg != want {
|
||||
t.Fatalf("aiproxySyncStatusMessage(failed) = %q, want %q", msg, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamModelKeyForBackend(t *testing.T) {
|
||||
cases := []struct {
|
||||
llmType string
|
||||
modelName string
|
||||
modelTag string
|
||||
want string
|
||||
}{
|
||||
{string(api.LLM_CONTAINER_VLLM), "Qwen/Qwen3-0.6B", "main", "Qwen3-0.6B"},
|
||||
{string(api.LLM_CONTAINER_SGLANG), "Qwen/Qwen2.5-7B-Instruct", "main", "Qwen2.5-7B-Instruct"},
|
||||
{string(api.LLM_CONTAINER_VLLM), "Qwen3-0.6B", "main", "Qwen3-0.6B"},
|
||||
{string(api.LLM_CONTAINER_OLLAMA), "qwen3", "8b", "qwen3:8b"},
|
||||
{string(api.LLM_CONTAINER_OLLAMA), "qwen3", "", "qwen3"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := upstreamModelKeyForBackend(c.llmType, c.modelName, c.modelTag)
|
||||
if got != c.want {
|
||||
t.Fatalf("upstreamModelKeyForBackend(%q, %q, %q) = %q, want %q",
|
||||
c.llmType, c.modelName, c.modelTag, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,21 @@ func (o *AiProviderCreateOptions) Params() (jsonutils.JSONObject, error) {
|
||||
return params, nil
|
||||
}
|
||||
|
||||
type AiProviderTestConnectivityOptions struct {
|
||||
ProviderKey string `help:"provider key" json:"provider_key"`
|
||||
Secret string `help:"API secret" json:"secret"`
|
||||
Config string `help:"provider config as JSON object string" json:"-"`
|
||||
}
|
||||
|
||||
func (o *AiProviderTestConnectivityOptions) Params() (jsonutils.JSONObject, error) {
|
||||
params := jsonutils.Marshal(o).(*jsonutils.JSONDict)
|
||||
params.Remove("config")
|
||||
if err := mergeJSONStringField(params, "config", o.Config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return params, nil
|
||||
}
|
||||
|
||||
type AiProviderUpdateOptions struct {
|
||||
ID string `help:"ID or name" json:"-"`
|
||||
Name string `json:"name,omitempty"`
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test climc ai-provider-create: create a custom ai_provider and verify with show/list.
|
||||
#
|
||||
# Usage:
|
||||
# source /etc/yunion/rcadmin
|
||||
# bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh
|
||||
#
|
||||
# Non-interactive:
|
||||
# export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1
|
||||
# export AIPROXY_PROVIDER_FT_NAME=my-custom-provider
|
||||
# export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-custom-key
|
||||
# export AIPROXY_PROVIDER_FT_BASE_URL=https://api.example.com/v1
|
||||
# bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh
|
||||
#
|
||||
# Or pass full config JSON:
|
||||
# export AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"https://api.example.com"}'
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=scripts/test/aiproxy/aiproxy-functional-test-common.sh
|
||||
source "${SCRIPT_DIR}/aiproxy-functional-test-common.sh"
|
||||
|
||||
CLIMC_OUTPUT_FORMAT="${CLIMC_OUTPUT_FORMAT:-json}"
|
||||
export CLIMC_OUTPUT_FORMAT
|
||||
|
||||
aiproxy_ft_need_cmds
|
||||
|
||||
PROVIDER_RESOURCE_NAME="${AIPROXY_PROVIDER_FT_NAME:-}"
|
||||
PROVIDER_KEY="${AIPROXY_PROVIDER_FT_PROVIDER_KEY:-}"
|
||||
BASE_URL="${AIPROXY_PROVIDER_FT_BASE_URL:-}"
|
||||
CONFIG_JSON="${AIPROXY_PROVIDER_FT_CONFIG:-}"
|
||||
ENABLED_FLAG="${AIPROXY_PROVIDER_FT_ENABLED:-true}"
|
||||
DELETE_IF_EXISTS="${AIPROXY_PROVIDER_FT_DELETE_EXISTING:-}"
|
||||
|
||||
prompt_line() {
|
||||
local prompt="$1" default="${2:-}" varname="$3"
|
||||
local value
|
||||
if [[ -n "$default" ]]; then
|
||||
echo -n "${prompt} [${default}]: " >&2
|
||||
else
|
||||
echo -n "${prompt}: " >&2
|
||||
fi
|
||||
if [[ ! -t 0 ]]; then
|
||||
value="$default"
|
||||
else
|
||||
read -r value </dev/tty
|
||||
fi
|
||||
value="${value:-$default}"
|
||||
[[ -n "$value" ]] || die "empty input for ${varname}"
|
||||
printf -v "$varname" '%s' "$value"
|
||||
}
|
||||
|
||||
prompt_yes_no() {
|
||||
local prompt="$1" default_yes="${2:-1}"
|
||||
local ans
|
||||
if [[ "${AIPROXY_PROVIDER_FT_NONINTERACTIVE:-}" == "1" ]]; then
|
||||
[[ "$default_yes" == "1" ]]
|
||||
return
|
||||
fi
|
||||
if [[ ! -t 0 ]]; then
|
||||
[[ "$default_yes" == "1" ]]
|
||||
return
|
||||
fi
|
||||
echo -n "${prompt} [Y/n]: " >&2
|
||||
read -r ans </dev/tty
|
||||
case "$ans" in
|
||||
n|N|no|No|NO) return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
build_config_json() {
|
||||
if [[ -n "$CONFIG_JSON" ]]; then
|
||||
echo "$CONFIG_JSON" | jq -c .
|
||||
return
|
||||
fi
|
||||
[[ -n "$BASE_URL" ]] || die "set AIPROXY_PROVIDER_FT_BASE_URL or AIPROXY_PROVIDER_FT_CONFIG"
|
||||
jq -nc --arg u "$BASE_URL" '{base_url:$u}'
|
||||
}
|
||||
|
||||
collect_inputs() {
|
||||
local suffix
|
||||
suffix="$(date +%Y%m%d%H%M%S)"
|
||||
|
||||
if [[ "${AIPROXY_PROVIDER_FT_NONINTERACTIVE:-}" == "1" ]]; then
|
||||
PROVIDER_RESOURCE_NAME="${PROVIDER_RESOURCE_NAME:-aiproxy-provider-ft-${suffix}}"
|
||||
PROVIDER_KEY="${PROVIDER_KEY:-custom-ft-${suffix}}"
|
||||
[[ -n "$CONFIG_JSON" || -n "$BASE_URL" ]] || die "non-interactive mode requires BASE_URL or CONFIG"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "=== ai_provider 创建测试 ===" >&2
|
||||
echo "将创建自定义 ai_provider（provider_key 不可与 catalog 重复）。" >&2
|
||||
echo
|
||||
|
||||
if [[ -z "$PROVIDER_RESOURCE_NAME" ]]; then
|
||||
prompt_line "资源名称 (climc 第一个参数 NAME)" "aiproxy-provider-ft-${suffix}" PROVIDER_RESOURCE_NAME
|
||||
fi
|
||||
if [[ -z "$PROVIDER_KEY" ]]; then
|
||||
prompt_line "provider_key (唯一标识)" "${PROVIDER_RESOURCE_NAME}" PROVIDER_KEY
|
||||
fi
|
||||
if [[ -z "$CONFIG_JSON" && -z "$BASE_URL" ]]; then
|
||||
prompt_line "config.base_url (OpenAI 兼容上游)" "https://api.openai.com" BASE_URL
|
||||
fi
|
||||
if prompt_yes_no "创建后启用 (--enabled)?" 1; then
|
||||
ENABLED_FLAG=true
|
||||
else
|
||||
ENABLED_FLAG=false
|
||||
fi
|
||||
}
|
||||
|
||||
delete_existing_provider() {
|
||||
local name="$1"
|
||||
if ! climc_json ai-provider-show "$name" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "$DELETE_IF_EXISTS" != "1" ]]; then
|
||||
die "ai_provider $name already exists; set AIPROXY_PROVIDER_FT_DELETE_EXISTING=1 to delete first"
|
||||
fi
|
||||
echo "deleting existing ai_provider $name" >&2
|
||||
climc ai-provider-delete "$name"
|
||||
}
|
||||
|
||||
create_provider() {
|
||||
local config enabled_args=()
|
||||
config="$(build_config_json)"
|
||||
if [[ "$ENABLED_FLAG" == "true" ]]; then
|
||||
enabled_args=(--enabled)
|
||||
fi
|
||||
climc ai-provider-create \
|
||||
"$PROVIDER_RESOURCE_NAME" \
|
||||
--provider-key "$PROVIDER_KEY" \
|
||||
--config "$config" \
|
||||
"${enabled_args[@]}"
|
||||
}
|
||||
|
||||
verify_provider() {
|
||||
local row pk base enabled
|
||||
row="$(climc_json ai-provider-show "$PROVIDER_RESOURCE_NAME")"
|
||||
pk="$(echo "$row" | jq -r '.provider_key // empty')"
|
||||
base="$(echo "$row" | jq -r '.config.base_url // empty')"
|
||||
enabled="$(echo "$row" | jq -r '.enabled // false')"
|
||||
[[ "$pk" == "$PROVIDER_KEY" ]] || die "provider_key mismatch: got $pk want $PROVIDER_KEY"
|
||||
if [[ -n "$BASE_URL" ]]; then
|
||||
[[ "$base" == "$BASE_URL" ]] || die "base_url mismatch: got $base want $BASE_URL"
|
||||
fi
|
||||
[[ "$enabled" == "true" ]] || [[ "$ENABLED_FLAG" != "true" ]] || die "expected enabled=true"
|
||||
echo "$row" | jq '{id, name, provider_key, enabled, config}'
|
||||
}
|
||||
|
||||
collect_inputs
|
||||
delete_existing_provider "$PROVIDER_RESOURCE_NAME"
|
||||
|
||||
aiproxy_ft_step "create ai_provider"
|
||||
create_provider
|
||||
|
||||
aiproxy_ft_step "verify ai-provider-show"
|
||||
verify_provider
|
||||
|
||||
aiproxy_ft_step "verify ai-provider-list filter"
|
||||
cnt="$(climc_json ai-provider-list --provider-key "$PROVIDER_KEY" \
|
||||
| jq --arg n "$PROVIDER_RESOURCE_NAME" '[.data[] | select(.name == $n)] | length')"
|
||||
[[ "$cnt" -ge 1 ]] || die "ai-provider-list --provider-key did not return created row"
|
||||
|
||||
echo
|
||||
echo "OK: ai_provider create test passed."
|
||||
echo " name: $PROVIDER_RESOURCE_NAME"
|
||||
echo " provider_key: $PROVIDER_KEY"
|
||||
echo "Cleanup:"
|
||||
echo " climc ai-provider-delete $PROVIDER_RESOURCE_NAME"
|
||||
@@ -1,396 +0,0 @@
|
||||
# Shared helpers for aiproxy functional test scripts (source only, do not execute directly).
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
need_cmd() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "missing command: $1"
|
||||
}
|
||||
|
||||
aiproxy_ft_need_cmds() {
|
||||
need_cmd climc
|
||||
need_cmd curl
|
||||
need_cmd jq
|
||||
}
|
||||
|
||||
climc_json() {
|
||||
climc --output-format json "$@"
|
||||
}
|
||||
|
||||
aiproxy_ft_step() { echo; echo "==> $*"; }
|
||||
|
||||
catalog_model_id() {
|
||||
local provider="$1" model_key="$2"
|
||||
echo "${provider}-${model_key}"
|
||||
}
|
||||
|
||||
default_model_for_provider() {
|
||||
case "$1" in
|
||||
aliyun) echo "qwen-turbo" ;;
|
||||
xiaomi) echo "mimo-v2-flash" ;;
|
||||
openai) echo "gpt-4o-mini" ;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
default_prompt_for_provider() {
|
||||
case "$1" in
|
||||
aliyun) echo "用一句话介绍通义千问" ;;
|
||||
xiaomi) echo "用一句话介绍小米 MiMo" ;;
|
||||
*) echo "用一句话介绍这个模型" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_aiproxy_url() {
|
||||
if [[ -n "${AIPROXY_URL:-}" ]]; then
|
||||
echo "${AIPROXY_URL%/}"
|
||||
return
|
||||
fi
|
||||
local url
|
||||
url="$(climc_json endpoint-list --service aiproxy --interface public --limit 1 \
|
||||
| jq -r '.data[0].url // empty')"
|
||||
[[ -n "$url" ]] || die "cannot resolve aiproxy public URL; set AIPROXY_URL"
|
||||
echo "${url%/}"
|
||||
}
|
||||
|
||||
list_catalog_provider_keys() {
|
||||
climc_json ai-provider-list --limit 500 \
|
||||
| jq -r '.data[] | .provider_key // empty' | sed '/^$/d' | sort -u
|
||||
}
|
||||
|
||||
list_catalog_model_keys() {
|
||||
local provider_key="$1"
|
||||
climc_json ai-model-list --ai-provider-id "$provider_key" --limit 500 \
|
||||
| jq -r '.data[] | .model_key // empty' \
|
||||
| sed '/^$/d' | grep -vx 'default' | sort -u
|
||||
}
|
||||
|
||||
# Resolve API key from env (generic or provider-specific legacy names).
|
||||
resolve_api_key_from_env() {
|
||||
local provider_key="$1"
|
||||
if [[ -n "${AIPROXY_FT_API_KEY:-}" ]]; then
|
||||
echo "$AIPROXY_FT_API_KEY"
|
||||
return
|
||||
fi
|
||||
case "$provider_key" in
|
||||
aliyun)
|
||||
[[ -n "${DASHSCOPE_API_KEY:-}" ]] && echo "$DASHSCOPE_API_KEY" && return
|
||||
;;
|
||||
xiaomi)
|
||||
[[ -n "${MIMO_API_KEY:-}" ]] && echo "$MIMO_API_KEY" && return
|
||||
;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
prompt_api_key() {
|
||||
local provider_key="$1" key
|
||||
if key="$(resolve_api_key_from_env "$provider_key")"; then
|
||||
echo "使用环境变量中的 API Key（未回显）" >&2
|
||||
echo "$key"
|
||||
return
|
||||
fi
|
||||
if [[ ! -t 0 ]]; then
|
||||
die "未设置 API Key：export AIPROXY_FT_API_KEY 或 ${provider_key} 对应的环境变量，或使用交互式终端"
|
||||
fi
|
||||
echo -n "请输入 ${provider_key} 的 API Key（不回显）: " >&2
|
||||
read -r -s key </dev/tty
|
||||
echo >&2
|
||||
[[ -n "$key" ]] || die "API Key 不能为空"
|
||||
echo "$key"
|
||||
}
|
||||
|
||||
# Populates global AIPROXY_FT_LINES[] (bash 3.2 compatible).
|
||||
_load_lines_into_array() {
|
||||
AIPROXY_FT_LINES=()
|
||||
while IFS= read -r line; do
|
||||
[[ -n "$line" ]] && AIPROXY_FT_LINES+=("$line")
|
||||
done
|
||||
}
|
||||
|
||||
prompt_select_provider() {
|
||||
local -a keys=()
|
||||
local k i choice
|
||||
_load_lines_into_array < <(list_catalog_provider_keys)
|
||||
keys=("${AIPROXY_FT_LINES[@]}")
|
||||
[[ ${#keys[@]} -gt 0 ]] || die "catalog 中无 ai_provider，请先执行 aiproxy master InitDB"
|
||||
|
||||
if [[ -n "${AIPROXY_FT_PROVIDER:-}" ]]; then
|
||||
for k in "${keys[@]}"; do
|
||||
[[ "$k" == "${AIPROXY_FT_PROVIDER}" ]] && echo "${AIPROXY_FT_PROVIDER}" && return
|
||||
done
|
||||
die "ai_provider ${AIPROXY_FT_PROVIDER} 不在 catalog 中"
|
||||
fi
|
||||
|
||||
if [[ ! -t 0 ]]; then
|
||||
die "请设置 AIPROXY_FT_PROVIDER 或在交互式终端运行"
|
||||
fi
|
||||
|
||||
echo "可用模型提供商 (catalog):" >&2
|
||||
for i in "${!keys[@]}"; do
|
||||
printf ' [%d] %s\n' "$((i + 1))" "${keys[$i]}" >&2
|
||||
done
|
||||
while true; do
|
||||
echo -n "请选择序号 [1-${#keys[@]}] 或直接输入 provider_key: " >&2
|
||||
read -r choice </dev/tty
|
||||
choice="${choice//[[:space:]]/}"
|
||||
[[ -z "$choice" ]] && continue
|
||||
if [[ "$choice" =~ ^[0-9]+$ ]] && ((choice >= 1 && choice <= ${#keys[@]})); then
|
||||
echo "${keys[$((choice - 1))]}"
|
||||
return
|
||||
fi
|
||||
for k in "${keys[@]}"; do
|
||||
[[ "$k" == "$choice" ]] && echo "$choice" && return
|
||||
done
|
||||
echo "无效选择，请重试。" >&2
|
||||
done
|
||||
}
|
||||
|
||||
prompt_select_model() {
|
||||
local provider_key="$1"
|
||||
local -a models=()
|
||||
local m i choice default_m found
|
||||
_load_lines_into_array < <(list_catalog_model_keys "$provider_key")
|
||||
models=("${AIPROXY_FT_LINES[@]}")
|
||||
[[ ${#models[@]} -gt 0 ]] || die "provider ${provider_key} 下无可用 model_key（catalog 未 seed？）"
|
||||
|
||||
if [[ -n "${AIPROXY_FT_MODEL:-}" ]]; then
|
||||
for m in "${models[@]}"; do
|
||||
[[ "$m" == "${AIPROXY_FT_MODEL}" ]] && echo "${AIPROXY_FT_MODEL}" && return
|
||||
done
|
||||
die "model_key ${AIPROXY_FT_MODEL} 不在 provider ${provider_key} 的 catalog 中"
|
||||
fi
|
||||
|
||||
default_m="$(default_model_for_provider "$provider_key")"
|
||||
found=0
|
||||
if [[ -n "$default_m" ]]; then
|
||||
for m in "${models[@]}"; do
|
||||
if [[ "$m" == "$default_m" ]]; then
|
||||
found=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
[[ "$found" -eq 1 ]] || default_m="${models[0]}"
|
||||
|
||||
if [[ ! -t 0 ]]; then
|
||||
echo "$default_m"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "提供商 ${provider_key} 的模型:" >&2
|
||||
for i in "${!models[@]}"; do
|
||||
printf ' [%d] %s\n' "$((i + 1))" "${models[$i]}" >&2
|
||||
done
|
||||
while true; do
|
||||
echo -n "请选择序号 [1-${#models[@]}] 或输入 model_key [默认: ${default_m}]: " >&2
|
||||
read -r choice </dev/tty
|
||||
choice="${choice//[[:space:]]/}"
|
||||
if [[ -z "$choice" ]]; then
|
||||
echo "$default_m"
|
||||
return
|
||||
fi
|
||||
if [[ "$choice" =~ ^[0-9]+$ ]] && ((choice >= 1 && choice <= ${#models[@]})); then
|
||||
echo "${models[$((choice - 1))]}"
|
||||
return
|
||||
fi
|
||||
for m in "${models[@]}"; do
|
||||
[[ "$m" == "$choice" ]] && echo "$choice" && return
|
||||
done
|
||||
echo "无效选择，请重试。" >&2
|
||||
done
|
||||
}
|
||||
|
||||
prompt_run_stream() {
|
||||
if [[ "${AIPROXY_FT_SKIP_STREAM:-}" == "1" ]]; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "${AIPROXY_FT_SKIP_STREAM:-}" == "0" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ ! -t 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
local ans
|
||||
echo -n "是否执行流式测试 (stream=true)? [Y/n]: " >&2
|
||||
read -r ans </dev/tty
|
||||
case "$ans" in
|
||||
n|N|no|No|NO) return 1 ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ensure_ai_key_enabled() {
|
||||
local key_name="$1"
|
||||
local enabled
|
||||
enabled="$(climc_json ai-key-show "$key_name" | jq -r '.enabled // false')"
|
||||
if [[ "$enabled" != "true" ]]; then
|
||||
echo "ai_key $key_name is disabled, enabling"
|
||||
climc ai-key-enable "$key_name"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_ai_key() {
|
||||
local provider_key="$1" key_name="$2" api_secret="$3"
|
||||
local provider_id
|
||||
provider_id="$(climc_json ai-provider-show "$provider_key" | jq -r '.id // empty')"
|
||||
[[ -n "$provider_id" ]] || die "ai_provider $provider_key not found"
|
||||
|
||||
if climc_json ai-key-show "$key_name" >/dev/null 2>&1; then
|
||||
echo "ai_key $key_name exists, syncing secret and ai_provider_id"
|
||||
climc ai-key-update "$key_name" \
|
||||
--ai-provider-id "$provider_id" \
|
||||
--secret "$api_secret" \
|
||||
--weight 10
|
||||
else
|
||||
climc ai-key-create \
|
||||
"$key_name" \
|
||||
--ai-provider-id "$provider_id" \
|
||||
--secret "$api_secret" \
|
||||
--weight 10 \
|
||||
--enabled
|
||||
fi
|
||||
ensure_ai_key_enabled "$key_name"
|
||||
}
|
||||
|
||||
verify_ai_key_for_provider() {
|
||||
local provider_key="$1"
|
||||
local provider_id count
|
||||
provider_id="$(climc_json ai-provider-show "$provider_key" | jq -r '.id // empty')"
|
||||
[[ -n "$provider_id" ]] || die "ai_provider $provider_key not found"
|
||||
count="$(climc_json ai-key-list --ai-provider-id "$provider_id" | jq '[.data[] | select(.enabled == true)] | length')"
|
||||
[[ "$count" -gt 0 ]] || die "no enabled ai_key bound to ai_provider_id=$provider_id"
|
||||
echo "enabled ai_key rows for $provider_id: $count"
|
||||
}
|
||||
|
||||
ensure_virtual_key() {
|
||||
local vk_name="$1"
|
||||
if climc_json ai-virtual-key-show "$vk_name" >/dev/null 2>&1; then
|
||||
echo "virtual key $vk_name already exists"
|
||||
return
|
||||
fi
|
||||
climc ai-virtual-key-create "$vk_name"
|
||||
}
|
||||
|
||||
ensure_routing() {
|
||||
local routing_name="$1" provider_key="$2" catalog_model_id="$3"
|
||||
if climc_json ai-routing-show "$routing_name" >/dev/null 2>&1; then
|
||||
echo "routing $routing_name already exists"
|
||||
return
|
||||
fi
|
||||
climc ai-routing-create \
|
||||
"$routing_name" \
|
||||
--priority 10 \
|
||||
--models "[{\"ai_provider_id\":\"${provider_key}\",\"ai_model_id\":\"${catalog_model_id}\",\"priority\":1}]"
|
||||
}
|
||||
|
||||
verify_stream_chat() {
|
||||
local base_url="$1" vk="$2" model="$3" out="$4" prompt="$5"
|
||||
local http_code aggregated delta payload
|
||||
|
||||
http_code="$(curl -k -sS -N -o "$out" -w '%{http_code}' \
|
||||
"${base_url%/}/openai/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${vk}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"${model}\",\"stream\":true,\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rn --arg t "$prompt" '$t')}],\"max_tokens\":64}")"
|
||||
|
||||
echo "HTTP $http_code (stream)"
|
||||
[[ "$http_code" == "200" ]] || {
|
||||
echo "--- stream body (first 40 lines) ---" >&2
|
||||
head -n 40 "$out" >&2 || true
|
||||
die "stream chat failed with HTTP $http_code"
|
||||
}
|
||||
|
||||
aggregated=""
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
[[ "$line" == data:* ]] || continue
|
||||
payload="${line#data: }"
|
||||
payload="${payload//$'\r'/}"
|
||||
[[ -z "$payload" ]] && continue
|
||||
[[ "$payload" == "[DONE]" ]] && continue
|
||||
if echo "$payload" | jq -e '.error // .message' >/dev/null 2>&1; then
|
||||
echo "upstream error chunk: $payload" >&2
|
||||
die "stream returned error event"
|
||||
fi
|
||||
delta="$(echo "$payload" | jq -r '.choices[0].delta.content // empty' 2>/dev/null || true)"
|
||||
aggregated+="$delta"
|
||||
done <"$out"
|
||||
|
||||
[[ -n "$aggregated" ]] || {
|
||||
echo "--- stream body ---" >&2
|
||||
cat "$out" >&2
|
||||
die "empty aggregated stream content (no choices[0].delta.content)"
|
||||
}
|
||||
|
||||
echo "stream content (${#aggregated} chars): ${aggregated:0:120}..."
|
||||
}
|
||||
|
||||
# aiproxy_ft_run executes the full functional test for one provider/model/api key.
|
||||
aiproxy_ft_run() {
|
||||
local provider_key="$1" chat_model="$2" api_secret="$3" chat_prompt="$4"
|
||||
local run_stream="${5:-1}"
|
||||
|
||||
local key_name vk_name routing_name catalog_mid
|
||||
local chat_resp chat_stream_resp aiproxy_url vk http_code content
|
||||
|
||||
key_name="${AIPROXY_FT_KEY_NAME:-aiproxy-ft-${provider_key}}"
|
||||
vk_name="${AIPROXY_FT_VK_NAME:-aiproxy-ft-${provider_key}-vk}"
|
||||
routing_name="${AIPROXY_FT_ROUTING_NAME:-aiproxy-ft-${provider_key}-routing}"
|
||||
chat_resp="${AIPROXY_FT_CHAT_RESP:-/tmp/aiproxy-ft-${provider_key}-chat.json}"
|
||||
chat_stream_resp="${AIPROXY_FT_STREAM_RESP:-/tmp/aiproxy-ft-${provider_key}-chat-stream.sse}"
|
||||
catalog_mid="$(catalog_model_id "$provider_key" "$chat_model")"
|
||||
|
||||
echo
|
||||
echo "=== aiproxy 功能测试 ==="
|
||||
echo "provider: ${provider_key} model: ${chat_model} catalog_id: ${catalog_mid}"
|
||||
echo "ai_key: ${key_name} virtual_key: ${vk_name} routing: ${routing_name}"
|
||||
echo
|
||||
|
||||
aiproxy_ft_step "1. Keystone aiproxy public endpoint"
|
||||
aiproxy_url="$(resolve_aiproxy_url)"
|
||||
echo "AIPROXY_URL=$aiproxy_url"
|
||||
|
||||
aiproxy_ft_step "2. Catalog ${provider_key} / ${chat_model}"
|
||||
climc ai-provider-show "$provider_key" >/dev/null \
|
||||
|| die "ai_provider $provider_key missing; run aiproxy master InitDB first"
|
||||
climc ai-model-show "$catalog_mid" >/dev/null \
|
||||
|| die "ai_model $catalog_mid not in catalog (re-run aiproxy master InitDB)"
|
||||
|
||||
aiproxy_ft_step "3. ai_key"
|
||||
ensure_ai_key "$provider_key" "$key_name" "$api_secret"
|
||||
verify_ai_key_for_provider "$provider_key"
|
||||
|
||||
aiproxy_ft_step "4. ai_virtual_key"
|
||||
ensure_virtual_key "$vk_name"
|
||||
vk="$(climc_json ai-virtual-key-show "$vk_name" | jq -r '.virtual_key')"
|
||||
[[ -n "$vk" ]] || die "empty virtual_key from ai-virtual-key-show"
|
||||
echo "virtual_key=${vk:0:12}..."
|
||||
|
||||
aiproxy_ft_step "5. ai_routing"
|
||||
ensure_routing "$routing_name" "$provider_key" "$catalog_mid"
|
||||
|
||||
aiproxy_ft_step "6. POST /openai/v1/chat/completions"
|
||||
http_code="$(curl -k -sS -o "$chat_resp" -w '%{http_code}' \
|
||||
"${aiproxy_url}/openai/v1/chat/completions" \
|
||||
-H "Authorization: Bearer ${vk}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"${chat_model}\",\"messages\":[{\"role\":\"user\",\"content\":$(jq -Rn --arg t "$chat_prompt" '$t')}],\"max_tokens\":128}")"
|
||||
|
||||
echo "HTTP $http_code"
|
||||
jq . <"$chat_resp"
|
||||
[[ "$http_code" == "200" ]] || die "chat request failed with HTTP $http_code"
|
||||
|
||||
content="$(jq -r '.choices[0].message.content // empty' <"$chat_resp")"
|
||||
[[ -n "$content" ]] || die "empty choices[0].message.content"
|
||||
|
||||
if [[ "$run_stream" == "1" ]]; then
|
||||
aiproxy_ft_step "7. POST /openai/v1/chat/completions (stream=true)"
|
||||
verify_stream_chat "$aiproxy_url" "$vk" "$chat_model" "$chat_stream_resp" "$chat_prompt"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "OK: aiproxy functional test passed for ${provider_key}/${chat_model} (non-stream$([[ "$run_stream" == "1" ]] && echo ' + stream' || echo ''))."
|
||||
echo "Cleanup (optional):"
|
||||
echo " climc ai-routing-delete $routing_name"
|
||||
echo " climc ai-virtual-key-delete $vk_name"
|
||||
echo " climc ai-key-delete $key_name"
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Legacy entry: pre-select xiaomi provider, then run interactive / env-based test.
|
||||
# Prefer: bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export AIPROXY_FT_PROVIDER="${AIPROXY_FT_PROVIDER:-xiaomi}"
|
||||
[[ -n "${MIMO_API_KEY:-}" && -z "${AIPROXY_FT_API_KEY:-}" ]] && export AIPROXY_FT_API_KEY="${MIMO_API_KEY}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec bash "${SCRIPT_DIR}/aiproxy-functional-test.sh" "$@"
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Legacy entry: pre-select aliyun (通义千问) provider, then run interactive / env-based test.
|
||||
# Prefer: bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export AIPROXY_FT_PROVIDER="${AIPROXY_FT_PROVIDER:-aliyun}"
|
||||
[[ -n "${DASHSCOPE_API_KEY:-}" && -z "${AIPROXY_FT_API_KEY:-}" ]] && export AIPROXY_FT_API_KEY="${DASHSCOPE_API_KEY}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec bash "${SCRIPT_DIR}/aiproxy-functional-test.sh" "$@"
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# aiproxy interactive functional test: select catalog provider/model, enter API key, run chat + stream.
|
||||
#
|
||||
# Usage:
|
||||
# source /etc/yunion/rcadmin
|
||||
# bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
#
|
||||
# Non-interactive (CI):
|
||||
# export AIPROXY_FT_PROVIDER=aliyun
|
||||
# export AIPROXY_FT_MODEL=qwen-turbo
|
||||
# export AIPROXY_FT_API_KEY='...'
|
||||
# export AIPROXY_FT_NONINTERACTIVE=1
|
||||
# bash scripts/test/aiproxy/aiproxy-functional-test.sh
|
||||
#
|
||||
# Legacy env (still supported):
|
||||
# DASHSCOPE_API_KEY + AIPROXY_FT_PROVIDER=aliyun
|
||||
# MIMO_API_KEY + AIPROXY_FT_PROVIDER=xiaomi
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=scripts/test/aiproxy/aiproxy-functional-test-common.sh
|
||||
source "${SCRIPT_DIR}/aiproxy-functional-test-common.sh"
|
||||
|
||||
CLIMC_OUTPUT_FORMAT="${CLIMC_OUTPUT_FORMAT:-json}"
|
||||
export CLIMC_OUTPUT_FORMAT
|
||||
|
||||
aiproxy_ft_need_cmds
|
||||
|
||||
if [[ "${AIPROXY_FT_NONINTERACTIVE:-}" == "1" && -z "${AIPROXY_FT_PROVIDER:-}" ]]; then
|
||||
die "AIPROXY_FT_NONINTERACTIVE=1 时需设置 AIPROXY_FT_PROVIDER"
|
||||
fi
|
||||
|
||||
PROVIDER_KEY="$(prompt_select_provider)"
|
||||
CHAT_MODEL="$(prompt_select_model "$PROVIDER_KEY")"
|
||||
API_SECRET="$(prompt_api_key "$PROVIDER_KEY")"
|
||||
CHAT_PROMPT="${AIPROXY_FT_PROMPT:-$(default_prompt_for_provider "$PROVIDER_KEY")}"
|
||||
|
||||
RUN_STREAM=1
|
||||
if prompt_run_stream; then
|
||||
RUN_STREAM=1
|
||||
else
|
||||
RUN_STREAM=0
|
||||
fi
|
||||
|
||||
aiproxy_ft_run "$PROVIDER_KEY" "$CHAT_MODEL" "$API_SECRET" "$CHAT_PROMPT" "$RUN_STREAM"
|
||||
Reference in New Issue
Block a user